@narumitw/pi-sync 0.30.0 → 0.34.0

package/src/snapshot.ts

@@ -10,20 +10,25 @@ export { isDeniedPath } from "./paths.js";
 
 import {
 	canonicalSnapshotPathForConfig,
-	DEFAULT_SYNC_FILES,
-	extraFilePathsByLower,
+	customIncludePathsByLower,
+	DEFAULT_SYNC_INCLUDE,
+	includeFromSelectionConfig,
 	isConfiguredSnapshotPath,
 	isPreservableUnmanagedSnapshotPath,
 	normalizeExtraFiles,
 	normalizeSyncFiles,
+	type SyncSelectionConfig,
+	syncIncludeSelection,
 } from "./sync-policy.js";
-import type { Snapshot, SnapshotFile, SnapshotOptions, SyncConfig } from "./types.js";
+import type { Snapshot, SnapshotFile, SnapshotOptions } from "./types.js";
 
 export { canonicalSnapshotPathForConfig, isConfiguredSnapshotPath } from "./sync-policy.js";
 
 const VERSION = 1;
-const TOP_LEVEL_FILES: readonly string[] = DEFAULT_SYNC_FILES.filter((name) => name.includes("."));
-const TOP_LEVEL_DIRS = new Set<string>(DEFAULT_SYNC_FILES.filter((name) => !name.includes(".")));
+const TOP_LEVEL_FILES: readonly string[] = DEFAULT_SYNC_INCLUDE.filter((name) =>
+	name.includes("."),
+);
+const TOP_LEVEL_DIRS = new Set<string>(DEFAULT_SYNC_INCLUDE.filter((name) => !name.includes(".")));
 const SECRET_PATTERNS = [
 	/AWS_SECRET_ACCESS_KEY\s*[=:]\s*['"]?[A-Za-z0-9/+]{35,}/i,
 	/(ANTHROPIC|OPENAI|GEMINI|GOOGLE|FIRECRAWL|GITHUB|CLOUDFLARE|R2|S3)_[A-Z0-9_]*(KEY|TOKEN|SECRET)\s*[=:]\s*['"]?[^\s'"]{12,}/i,
@@ -78,16 +83,24 @@ function snapshotId() {
 	return `${new Date().toISOString().replace(/[:.]/g, "-")}-${randomUUID().slice(0, 8)}`;
 }
 
+export function regenerateSnapshotIdentity(snapshot: Snapshot): Snapshot {
+	return {
+		...snapshot,
+		id: snapshotId(),
+		createdAt: new Date().toISOString(),
+		machine: os.hostname(),
+	};
+}
+
 export async function createSnapshot(
 	profile: string,
 	options: SnapshotOptions = {},
 ): Promise<Snapshot> {
-	const syncSessions = Boolean(options.syncSessions);
+	const include = effectiveInclude(options);
+	const syncSessions = include.includes("sessions");
 	const files = await collectFiles(agentDir(), {
-		syncFiles: options.syncFiles,
-		syncSessions,
+		include,
 		sessionDir: options.sessionDir ?? (await configuredSessionDir()),
-		extraFiles: options.extraFiles,
 	});
 	return {
 		version: VERSION,
@@ -100,13 +113,23 @@ export async function createSnapshot(
 	};
 }
 
+function effectiveInclude(options: SnapshotOptions) {
+	if (options.include) return options.include;
+	return [
+		...normalizeSyncFiles(options.syncFiles),
+		...normalizeExtraFiles(options.extraFiles),
+		...(options.syncSessions ? ["sessions"] : []),
+	];
+}
+
 export async function collectFiles(
 	root: string,
 	options: SnapshotOptions = {},
 ): Promise<SnapshotFile[]> {
 	const results: SnapshotFile[] = [];
 	const entries = await fs.readdir(root, { withFileTypes: true });
-	const selectedFiles = new Set<string>(normalizeSyncFiles(options.syncFiles));
+	const selection = syncIncludeSelection(effectiveInclude(options));
+	const selectedFiles = new Set<string>(selection.builtIns);
 	for (const entry of entries) {
 		if (entry.isDirectory() && TOP_LEVEL_DIRS.has(entry.name) && selectedFiles.has(entry.name)) {
 			await collectDirectory(results, root, entry.name);
@@ -117,11 +140,10 @@ export async function collectFiles(
 		const entry = selectTopLevelFileEntry(entries, fileName);
 		if (entry) await addFile(results, root, entry.name, fileName);
 	}
-	for (const extraFileName of normalizeExtraFiles(options.extraFiles)) {
-		const entry = selectTopLevelFileEntry(entries, extraFileName);
-		if (entry) await addFile(results, root, entry.name, extraFileName);
+	for (const relativePath of selection.custom) {
+		await collectIncludedPath(results, root, relativePath);
 	}
-	if (options.syncSessions) {
+	if (selection.sessions) {
 		try {
 			await collectDirectory(results, sessionStorageRoot(root, options.sessionDir), "", {
 				sessionsOnly: true,
@@ -134,6 +156,22 @@ export async function collectFiles(
 	return results.sort((left, right) => left.path.localeCompare(right.path));
 }
 
+async function collectIncludedPath(results: SnapshotFile[], root: string, relativePath: string) {
+	const absolutePath = safeJoin(root, relativePath);
+	try {
+		const stat = await fs.lstat(absolutePath);
+		if (stat.isFile()) await addFile(results, root, relativePath);
+		else if (stat.isDirectory()) await collectDirectory(results, root, relativePath);
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+		if (!relativePath.includes("/")) {
+			const entries = await fs.readdir(root, { withFileTypes: true });
+			const entry = selectTopLevelFileEntry(entries, relativePath);
+			if (entry) await addFile(results, root, entry.name, relativePath);
+		}
+	}
+}
+
 async function collectDirectory(
 	results: SnapshotFile[],
 	root: string,
@@ -216,15 +254,15 @@ export function snapshotIncludesSessions(snapshot: Snapshot) {
 
 export function filterSnapshotForConfigPolicy(
 	snapshot: Snapshot,
-	config: Pick<SyncConfig, "syncFiles" | "syncSessions" | "extraFiles">,
+	config: SyncSelectionConfig,
 	options: { regenerateId?: boolean } = {},
 ) {
-	const extraFilePaths = extraFilePathsByLower(config.extraFiles);
-	const extraFiles = new Set(extraFilePaths.keys());
+	const include = includeFromSelectionConfig(config);
+	const includePaths = customIncludePathsByLower(include);
 	const filtered = {
 		...snapshot,
-		syncSessions: config.syncSessions ? snapshot.syncSessions : false,
-		files: canonicalizeSnapshotFilesForConfig(snapshot.files, config, extraFiles, extraFilePaths),
+		syncSessions: include.includes("sessions") ? snapshot.syncSessions : false,
+		files: canonicalizeSnapshotFilesForConfig(snapshot.files, config, includePaths),
 	};
 	if (!options.regenerateId || snapshotsMatch(snapshot, filtered)) return filtered;
 	return {
@@ -237,9 +275,8 @@ export function filterSnapshotForConfigPolicy(
 
 function canonicalizeSnapshotFilesForConfig(
 	files: SnapshotFile[],
-	config: Pick<SyncConfig, "syncFiles" | "syncSessions">,
-	extraFiles: Set<string>,
-	extraFilePaths: Map<string, string>,
+	config: SyncSelectionConfig,
+	includePaths: Map<string, string>,
 ) {
 	const configuredFiles: SnapshotFile[] = [];
 	const extraCandidates = new Map<
@@ -248,17 +285,14 @@ function canonicalizeSnapshotFilesForConfig(
 	>();
 	for (const file of files) {
 		const normalized = toPosix(file.path);
-		if (
-			!isSafeSnapshotPath(file.path) ||
-			!isConfiguredSnapshotPath(normalized, config, extraFiles)
-		) {
+		if (!isSafeSnapshotPath(file.path) || !isConfiguredSnapshotPath(normalized, config)) {
 			continue;
 		}
 		if (normalized.includes("/")) {
 			configuredFiles.push(normalized === file.path ? file : { ...file, path: normalized });
 			continue;
 		}
-		const topLevelPath = canonicalSnapshotPathForConfig(normalized, extraFilePaths);
+		const topLevelPath = canonicalSnapshotPathForConfig(normalized, includePaths);
 		const candidate = {
 			exact: normalized === topLevelPath,
 			file: { ...file, path: topLevelPath },
@@ -315,12 +349,9 @@ export function scanSnapshot(snapshot: Snapshot) {
 export function mergeRemotePreservedFiles(
 	local: Snapshot,
 	remote: Snapshot,
-	config: Pick<SyncConfig, "syncFiles" | "syncSessions" | "extraFiles">,
+	config: SyncSelectionConfig,
 ) {
 	const localPathNames = new Set(local.files.map((file) => file.path.toLowerCase()));
-	const configuredExtraFiles = new Set(
-		normalizeExtraFiles(config.extraFiles).map((file) => file.toLowerCase()),
-	);
 	const preservedPathNames = new Set<string>();
 	const preserved = remote.files.filter((file) => {
 		const normalized = toPosix(file.path);
@@ -329,7 +360,7 @@ export function mergeRemotePreservedFiles(
 			localPathNames.has(lower) ||
 			preservedPathNames.has(lower) ||
 			!isSafeSnapshotPath(file.path) ||
-			isConfiguredSnapshotPath(normalized, config, configuredExtraFiles) ||
+			isConfiguredSnapshotPath(normalized, config) ||
 			!isPreservableUnmanagedSnapshotPath(normalized)
 		) {
 			return false;

package/src/storage-connections-ui.ts

@@ -0,0 +1,308 @@
+import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
+import { isCloudflareR2Endpoint, readLocalConfigObject } from "./config.js";
+import { showAddGitStorageProfile, showEditGitStorageProfile } from "./git-ui.js";
+import { errorMessage, ownRecord, requiredInput, safeTerminalText } from "./manager-helpers.js";
+import {
+	applyS3CredentialUpdate,
+	chooseS3Credentials,
+	chooseS3CredentialUpdate,
+} from "./s3-credentials-ui.js";
+import {
+	addStorageConnection,
+	removeStorageConnection,
+	updateStorageConnection,
+} from "./settings-management.js";
+import { showAddWebDavStorageProfile, showEditWebDavStorageProfile } from "./webdav-ui.js";
+
+const BACK = "Back";
+
+export async function showStorageConnections(ctx: ExtensionCommandContext, signal?: AbortSignal) {
+	while (!signal?.aborted) {
+		const raw = await readLocalConfigObject();
+		if (signal?.aborted) return;
+		if (raw?.version !== 3) {
+			ctx.ui.notify("Create version 3 settings before managing storage connections.", "info");
+			return;
+		}
+		const profiles = ownRecord(raw.storageConnections) ?? {};
+		const labels = new Map(
+			Object.keys(profiles)
+				.sort((left, right) => left.localeCompare(right))
+				.map((name) => [safeTerminalText(name), name]),
+		);
+		const selected = await ctx.ui.select(
+			"Storage connections",
+			["Add storage connection", ...labels.keys(), BACK],
+			{ signal },
+		);
+		if (signal?.aborted || !selected || selected === BACK) return;
+		if (selected === "Add storage connection") {
+			try {
+				await showAddStorageConnection(ctx, signal);
+			} catch (error) {
+				if (signal?.aborted) return;
+				ctx.ui.notify(
+					`Storage connection was not added: ${safeTerminalText(errorMessage(error))} Retry from Add storage connection.`,
+					"error",
+				);
+			}
+			continue;
+		}
+		const name = labels.get(selected);
+		if (name) await showStorageConnectionDetail(ctx, name, signal);
+	}
+}
+
+async function showStorageConnectionDetail(
+	ctx: ExtensionCommandContext,
+	name: string,
+	signal?: AbortSignal,
+) {
+	while (!signal?.aborted) {
+		const raw = await readLocalConfigObject();
+		if (signal?.aborted) return;
+		const profiles = ownRecord(raw?.storageConnections) ?? {};
+		const profile = ownRecord(profiles[name]);
+		if (!profile) {
+			ctx.ui.notify(`Storage connection “${safeTerminalText(name)}” no longer exists.`, "warning");
+			return;
+		}
+		const usedBy = referencingSetups(raw, name);
+		const selected = await ctx.ui.select(
+			[
+				`Storage connection “${safeTerminalText(name)}”`,
+				"",
+				`Type: ${connectionType(profile)}`,
+				`Endpoint: ${connectionEndpoint(profile)}`,
+				`Credentials: ${credentialSource(profile)}`,
+				`Used by: ${usedBy.length > 0 ? usedBy.map(safeTerminalText).join(", ") : "No sync setups"}`,
+				...(usedBy.length > 0
+					? ["Remove unavailable: edit or remove the listed sync setups first."]
+					: []),
+			].join("\n"),
+			[
+				"Edit storage connection…",
+				...(usedBy.length === 0 ? ["Remove storage connection…"] : []),
+				BACK,
+			],
+			{ signal },
+		);
+		if (signal?.aborted || !selected || selected === BACK) return;
+		try {
+			if (selected === "Remove storage connection…") {
+				const confirmed = await ctx.ui.confirm(
+					"Remove storage connection?",
+					`Remove local storage connection “${safeTerminalText(name)}”? Remote data and history are not deleted.`,
+					{ signal },
+				);
+				if (!confirmed || signal?.aborted) continue;
+				await removeStorageConnection(name, signal);
+				ctx.ui.notify(`Removed storage connection “${safeTerminalText(name)}”.`, "info");
+				return;
+			}
+			await editStorageConnection(ctx, name, profile, usedBy, signal);
+		} catch (error) {
+			if (signal?.aborted) return;
+			ctx.ui.notify(
+				`Storage connection “${safeTerminalText(name)}” was not changed: ${safeTerminalText(errorMessage(error))} Reopen it and retry.`,
+				"error",
+			);
+		}
+	}
+}
+
+async function editStorageConnection(
+	ctx: ExtensionCommandContext,
+	name: string,
+	profile: Record<string, unknown>,
+	usedBy: string[],
+	signal?: AbortSignal,
+) {
+	if (ctx.mode !== "tui") {
+		ctx.ui.notify(
+			"Editing storage connections requires TUI mode for safe credential handling. Edit the private version 3 settings file instead.",
+			"warning",
+		);
+		return;
+	}
+	if (profile.type === "webdav") {
+		await showEditWebDavStorageProfile(
+			ctx,
+			name,
+			{ ...profile, kind: "webdav", ...(ownRecord(profile.credentials) ?? {}) },
+			signal,
+			usedBy,
+		);
+		return;
+	}
+	if (profile.type === "git") {
+		await showEditGitStorageProfile(ctx, name, { ...profile, kind: "git" }, signal, usedBy);
+		return;
+	}
+	const endpoint = await requiredInput(
+		ctx,
+		"Endpoint",
+		String(profile.endpoint ?? "https://s3.example.com"),
+		signal,
+	);
+	if (!endpoint || signal?.aborted) return;
+	const region = await requiredInput(ctx, "Region", String(profile.region ?? "auto"), signal);
+	if (!region || signal?.aborted) return;
+	const storedCredentials = ownRecord(profile.credentials) ?? {};
+	const credentials = await chooseS3CredentialUpdate(
+		ctx,
+		{ ...profile, ...storedCredentials },
+		signal,
+	);
+	if (!credentials || signal?.aborted) return;
+	const save = await ctx.ui.select(
+		[
+			"Review storage connection",
+			"",
+			`Storage connection: ${safeTerminalText(name)}`,
+			`Endpoint: ${safeTerminalText(String(profile.endpoint ?? "missing"))} → ${safeTerminalText(endpoint)}`,
+			`Region: ${safeTerminalText(String(profile.region ?? "auto"))} → ${safeTerminalText(region)}`,
+			`Credentials: ${safeTerminalText(credentials.summary)}`,
+			`Affected sync setups: ${usedBy.length > 0 ? usedBy.map(safeTerminalText).join(", ") : "None"}`,
+			"Saving changes future storage access for every affected setup; it does not move remote data.",
+		].join("\n"),
+		["Save storage connection", "Cancel"],
+		{ signal },
+	);
+	if (save !== "Save storage connection" || signal?.aborted) return;
+	await updateStorageConnection(
+		name,
+		(current) => {
+			if (current.type !== "s3") {
+				throw new Error("Storage connection type changed; reopen it.");
+			}
+			return {
+				...current,
+				endpoint,
+				region,
+				credentials: applyS3CredentialUpdate(
+					current.credentials,
+					credentials,
+				) as typeof current.credentials,
+			};
+		},
+		usedBy,
+		signal,
+	);
+	if (signal?.aborted) return;
+	ctx.ui.notify(`Saved storage connection “${safeTerminalText(name)}”.`, "info");
+}
+
+export async function showAddStorageConnection(ctx: ExtensionCommandContext, signal?: AbortSignal) {
+	if (ctx.mode !== "tui") {
+		ctx.ui.notify(
+			"Adding storage connections requires TUI mode for safe credential handling. Edit the private version 3 settings file instead.",
+			"warning",
+		);
+		return false;
+	}
+	const preset = await ctx.ui.select(
+		"Storage type",
+		["Cloudflare R2", "Other S3-compatible storage", "WebDAV", "Git", "Cancel"],
+		{ signal },
+	);
+	if (signal?.aborted || !preset || preset === "Cancel") return false;
+	if (preset === "WebDAV") return showAddWebDavStorageProfile(ctx, signal);
+	if (preset === "Git") return showAddGitStorageProfile(ctx, signal);
+	const name = await requiredInput(
+		ctx,
+		"Name this storage connection",
+		preset === "Cloudflare R2" ? "r2" : "s3",
+		signal,
+	);
+	if (!name || signal?.aborted) return false;
+	const endpoint = await requiredInput(
+		ctx,
+		"Endpoint",
+		preset === "Cloudflare R2"
+			? "https://<account-id>.r2.cloudflarestorage.com"
+			: "https://s3.example.com",
+		signal,
+	);
+	if (!endpoint || signal?.aborted) return false;
+	const region =
+		preset === "Cloudflare R2" ? "auto" : await requiredInput(ctx, "Region", "us-east-1", signal);
+	if (!region || signal?.aborted) return false;
+	const credentials = await chooseS3Credentials(ctx, signal);
+	if (!credentials || signal?.aborted) return false;
+	const save = await ctx.ui.select(
+		[
+			"Review storage connection",
+			"",
+			`Name: ${safeTerminalText(name)}`,
+			`Type: ${preset}`,
+			`Endpoint: ${safeTerminalText(endpoint)}`,
+			`Region: ${safeTerminalText(region)}`,
+			`Credentials: ${safeTerminalText(credentials.summary)}`,
+			"Adding a connection does not contact remote storage or start syncing.",
+		].join("\n"),
+		["Add storage connection", "Cancel"],
+		{ signal },
+	);
+	if (save !== "Add storage connection" || signal?.aborted) return false;
+	await addStorageConnection(
+		name,
+		{
+			type: "s3",
+			endpoint,
+			region,
+			credentials: {
+				accessKeyId: credentials.profileFields.accessKeyId ?? "",
+				secretAccessKey: credentials.profileFields.secretAccessKey ?? "",
+			},
+		},
+		signal,
+	);
+	if (signal?.aborted) return true;
+	ctx.ui.notify(`Added storage connection “${safeTerminalText(name)}”.`, "info");
+	return true;
+}
+
+function referencingSetups(raw: Record<string, unknown> | undefined, connection: string) {
+	return Object.entries(ownRecord(raw?.syncSetups) ?? {})
+		.filter(([, value]) => ownRecord(ownRecord(value)?.storage)?.connection === connection)
+		.map(([name]) => name)
+		.sort((left, right) => left.localeCompare(right));
+}
+
+function connectionType(profile: Record<string, unknown>) {
+	if (profile.type === "git") return "Git";
+	if (profile.type === "webdav") return "WebDAV";
+	if (
+		profile.type === "s3" &&
+		typeof profile.endpoint === "string" &&
+		isCloudflareR2Endpoint(profile.endpoint)
+	) {
+		return "Cloudflare R2";
+	}
+	return "S3-compatible";
+}
+
+function connectionEndpoint(profile: Record<string, unknown>) {
+	const value =
+		profile.type === "git"
+			? profile.remote
+			: profile.type === "webdav"
+				? profile.url
+				: profile.endpoint;
+	if (typeof value !== "string" || value.length === 0) return "Missing";
+	if (profile.type === "git") return safeTerminalText(value);
+	try {
+		return safeTerminalText(new URL(value).host);
+	} catch {
+		return "Invalid";
+	}
+}
+
+function credentialSource(profile: Record<string, unknown>) {
+	if (profile.type === "git") return "Git credential helper or SSH configuration";
+	const credentials = ownRecord(profile.credentials);
+	if (profile.type === "webdav") return credentials?.password ? "Settings file" : "Missing";
+	if (credentials?.accessKeyId && credentials.secretAccessKey) return "Settings file";
+	return "Missing";
+}

package/src/sync-backend.ts

@@ -0,0 +1,97 @@
+import type { Snapshot } from "./types.js";
+
+export type PublicationCapability =
+	| "read-check-write-verify"
+	| "conditional-required"
+	| "atomic-conditional"
+	| "lease-protected";
+
+export interface RemoteHead {
+	/** Backend-owned reference used only to retrieve this immutable snapshot. */
+	snapshotRef: string;
+	/** Backend-neutral snapshot content identity retained in local sync state. */
+	snapshotId: string;
+	revision: string;
+	createdAt: string;
+	machine: string;
+	syncSessions: boolean;
+}
+
+export interface RemoteHistoryEntry {
+	snapshotRef: string;
+	snapshotId: string;
+	createdAt: string;
+	machine: string;
+	syncSessions: boolean;
+}
+
+export type ExpectedRemoteHead = { kind: "missing" } | { kind: "revision"; revision: string };
+
+export interface PublishSnapshotOptions {
+	signal?: AbortSignal;
+	/** Called at the backend's active-head commit boundary. */
+	onCommit?: () => void;
+}
+
+export interface PublishSnapshotResult {
+	head: RemoteHead;
+	warnings: string[];
+}
+
+export interface BackendDiagnostic {
+	key: string;
+	level: "info" | "warning" | "error";
+	message: string;
+}
+
+export interface SyncBackend {
+	readonly identity: string;
+	readonly destination: string;
+	readonly capability: PublicationCapability;
+	/** Compare opaque revisions produced by this backend identity. */
+	sameRevision(left: string, right: string): boolean;
+	readHead(signal?: AbortSignal): Promise<RemoteHead | undefined>;
+	readSnapshot(reference: string, signal?: AbortSignal): Promise<Snapshot>;
+	publishSnapshot(
+		snapshot: Snapshot,
+		expected: ExpectedRemoteHead,
+		options?: PublishSnapshotOptions,
+	): Promise<PublishSnapshotResult>;
+	listHistory(signal?: AbortSignal): Promise<RemoteHistoryEntry[]>;
+	diagnose(signal?: AbortSignal): Promise<BackendDiagnostic[]>;
+}
+
+export class SyncBackendConflictError extends Error {
+	readonly code = "SYNC_BACKEND_CONFLICT";
+	readonly phase: "before-commit" | "after-commit";
+	readonly currentHead?: RemoteHead;
+	readonly candidateMayHaveBeenActive: boolean;
+
+	constructor(
+		message: string,
+		options: ErrorOptions & {
+			phase?: "before-commit" | "after-commit";
+			currentHead?: RemoteHead;
+			candidateMayHaveBeenActive?: boolean;
+		} = {},
+	) {
+		super(message, options);
+		this.name = "SyncBackendConflictError";
+		this.phase = options.phase ?? "before-commit";
+		this.currentHead = options.currentHead;
+		this.candidateMayHaveBeenActive = options.candidateMayHaveBeenActive ?? false;
+	}
+}
+
+export class SyncBackendPublicationOutcomeUnknownError extends Error {
+	readonly code = "SYNC_BACKEND_PUBLICATION_OUTCOME_UNKNOWN";
+
+	constructor(message: string, options?: ErrorOptions) {
+		super(message, options);
+		this.name = "SyncBackendPublicationOutcomeUnknownError";
+	}
+}
+
+export function expectedRemoteHead(head: RemoteHead | undefined): ExpectedRemoteHead {
+	return head ? { kind: "revision", revision: head.revision } : { kind: "missing" };
+}