@narumitw/pi-sync 0.25.0 → 0.29.0

package/src/s3-client.ts

@@ -10,7 +10,10 @@ function iso8601Basic(date: Date) {
 }
 
 function snapshotIncludesSessions(snapshot: Snapshot) {
-	return snapshot.syncSessions === true || snapshot.files.some((file) => file.path.startsWith("sessions/"));
+	return (
+		snapshot.syncSessions === true ||
+		snapshot.files.some((file) => file.path.startsWith("sessions/"))
+	);
 }
 
 function isCloudflareR2Endpoint(endpoint: string | undefined) {
@@ -18,7 +21,9 @@ function isCloudflareR2Endpoint(endpoint: string | undefined) {
 	if (!value) return false;
 	try {
 		const hostname = new URL(value).hostname.toLowerCase();
-		return hostname === "r2.cloudflarestorage.com" || hostname.endsWith(".r2.cloudflarestorage.com");
+		return (
+			hostname === "r2.cloudflarestorage.com" || hostname.endsWith(".r2.cloudflarestorage.com")
+		);
 	} catch {
 		return false;
 	}
@@ -27,11 +32,13 @@ function isCloudflareR2Endpoint(endpoint: string | undefined) {
 export class S3Client {
 	private config: SyncConfig;
 	private endpoint: URL;
+	private signal?: AbortSignal;
 	private omitSessionTokenAfterRejection = false;
 
-	constructor(config: SyncConfig) {
+	constructor(config: SyncConfig, signal?: AbortSignal) {
 		this.config = config;
 		this.endpoint = new URL(config.endpoint);
+		this.signal = signal;
 	}
 
 	async getJson<T>(key: string): Promise<RemoteObject<T>> {
@@ -48,11 +55,15 @@ export class S3Client {
 			// Retry so the transient blip is absorbed instead of surfacing as a
 			// "pi-sync auto sync skipped" warning on every session start.
 			if (body.length > 0) {
-				return { value: JSON.parse(body) as T, etag: normalizeEtag(object.headers.get("etag")), missing: false };
+				return {
+					value: JSON.parse(body) as T,
+					etag: normalizeEtag(object.headers.get("etag")),
+					missing: false,
+				};
 			}
 			lastError = new Error(`S3 GET returned an empty body for ${key}`);
 			if (attempt < maxAttempts) {
-				await sleep(250 * attempt);
+				await sleep(250 * attempt, undefined, { signal: this.signal });
 			}
 		}
 		throw lastError;
@@ -78,7 +89,7 @@ export class S3Client {
 			}
 			lastError = new Error(`S3 GET returned an empty body for ${key}`);
 			if (attempt < maxAttempts) {
-				await sleep(250 * attempt);
+				await sleep(250 * attempt, undefined, { signal: this.signal });
 			}
 		}
 		throw lastError;
@@ -92,7 +103,8 @@ export class S3Client {
 	async putBuffer(key: string, body: Buffer, contentType: string) {
 		const headers: Record<string, string> = { "content-type": contentType };
 		const response = await this.request("PUT", key, body, headers);
-		if (!response.ok) throw new Error(`S3 PUT failed (${response.status}): ${await response.text()}`);
+		if (!response.ok)
+			throw new Error(`S3 PUT failed (${response.status}): ${await response.text()}`);
 	}
 
 	private async request(
@@ -114,7 +126,12 @@ export class S3Client {
 				sessionToken,
 				region: this.config.region,
 			});
-			return fetch(url, { method, headers, body: body ? new Uint8Array(body) : undefined });
+			return fetch(url, {
+				method,
+				headers,
+				body: body ? new Uint8Array(body) : undefined,
+				signal: this.signal,
+			});
 		};
 		const sessionToken = this.omitSessionTokenAfterRejection ? undefined : this.config.sessionToken;
 		const response = await send(sessionToken);
@@ -125,7 +142,10 @@ export class S3Client {
 		return retry;
 	}
 
-	private async shouldRetryWithoutSessionToken(response: Response, sessionToken: string | undefined) {
+	private async shouldRetryWithoutSessionToken(
+		response: Response,
+		sessionToken: string | undefined,
+	) {
 		if (
 			!sessionToken ||
 			!isCloudflareR2Endpoint(this.config.endpoint) ||
@@ -160,7 +180,9 @@ async function signedHeaders(input: {
 	};
 	if (input.sessionToken) headers["x-amz-security-token"] = input.sessionToken;
 	const signedHeaderNames = Object.keys(headers).sort();
-	const canonicalHeaders = signedHeaderNames.map((name) => `${name}:${headers[name]?.trim()}\n`).join("");
+	const canonicalHeaders = signedHeaderNames
+		.map((name) => `${name}:${headers[name]?.trim()}\n`)
+		.join("");
 	const canonicalRequest = [
 		input.method,
 		input.url.pathname,
@@ -170,7 +192,12 @@ async function signedHeaders(input: {
 		payloadHash,
 	].join("\n");
 	const scope = `${dateStamp}/${input.region}/s3/aws4_request`;
-	const stringToSign = ["AWS4-HMAC-SHA256", amzDate, scope, sha256(Buffer.from(canonicalRequest))].join("\n");
+	const stringToSign = [
+		"AWS4-HMAC-SHA256",
+		amzDate,
+		scope,
+		sha256(Buffer.from(canonicalRequest)),
+	].join("\n");
 	const signingKey = hmac(
 		hmac(hmac(hmac(Buffer.from(`AWS4${input.secretAccessKey}`), dateStamp), input.region), "s3"),
 		"aws4_request",
@@ -206,7 +233,11 @@ export function profilePrefix(config: SyncConfig) {
 	return posixJoin(config.prefix, "profiles", config.profile);
 }
 
-export function pointerFor(config: SyncConfig, snapshot: Snapshot, checksum: string): LatestPointer {
+export function pointerFor(
+	config: SyncConfig,
+	snapshot: Snapshot,
+	checksum: string,
+): LatestPointer {
 	return {
 		version: VERSION,
 		profile: config.profile,
@@ -227,5 +258,8 @@ function normalizeEtag(value: string | null) {
 }
 
 function isSecurityTokenInvalidArgument(text: string) {
-	return text.includes("<Code>InvalidArgument</Code>") && text.includes("<Message>X-Amz-Security-Token</Message>");
+	return (
+		text.includes("<Code>InvalidArgument</Code>") &&
+		text.includes("<Message>X-Amz-Security-Token</Message>")
+	);
 }

package/src/settings-management.ts

@@ -0,0 +1,307 @@
+import { randomUUID } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import {
+	effectiveTargetRemoteIdentity,
+	localConfigPath,
+	readLocalConfigObject,
+	stateDir,
+	statePathForConfig,
+	writeLocalConfigObject,
+} from "./config.js";
+import { withLock } from "./lock.js";
+import { safeName } from "./paths.js";
+import type { StorageProfileSettings, SyncConfig, SyncTargetSettings } from "./types.js";
+
+const LEGACY_FIELDS = new Set([
+	"endpoint",
+	"bucket",
+	"region",
+	"accessKeyId",
+	"secretAccessKey",
+	"sessionToken",
+	"profile",
+	"prefix",
+	"autoSync",
+	"syncFiles",
+	"syncSessions",
+	"extraFiles",
+]);
+
+export interface MigrationResult {
+	settings: Record<string, unknown>;
+	backupPath: string;
+}
+
+export async function migrateLegacySettings(
+	targetName: string,
+	storageProfileName: string,
+): Promise<MigrationResult> {
+	validateName(targetName, "target");
+	validateName(storageProfileName, "storage profile");
+	return withLock("settings-migration", async () => {
+		const configPath = localConfigPath();
+		const originalBytes = await fs.readFile(configPath);
+		const current = await readLocalConfigObject();
+		if (!current) throw new Error("No legacy pi-sync settings are available to migrate.");
+		if (current.version === 2) {
+			throw new Error("pi-sync settings already use profiles and targets.");
+		}
+		const next = legacySettingsAsV2(current, targetName, storageProfileName);
+		const backupPath = await writeMigrationBackup(originalBytes);
+		await adoptLegacyState(current, next, targetName);
+		const latestBytes = await fs.readFile(configPath);
+		if (!latestBytes.equals(originalBytes)) {
+			throw new Error("pi-sync settings changed during migration; no settings were replaced.");
+		}
+		await writeLocalConfigObject(next);
+		return { settings: next, backupPath };
+	});
+}
+
+export function legacySettingsAsV2(
+	legacy: Record<string, unknown>,
+	targetName: string,
+	storageProfileName: string,
+) {
+	validateName(targetName, "target");
+	validateName(storageProfileName, "storage profile");
+	const unknown = Object.fromEntries(
+		Object.entries(legacy).filter(([key]) => !LEGACY_FIELDS.has(key) && key !== "version"),
+	);
+	const profile = compactObject({
+		kind: inferKind(legacy.endpoint),
+		endpoint: legacy.endpoint,
+		region: legacy.region,
+		accessKeyId: legacy.accessKeyId,
+		secretAccessKey: legacy.secretAccessKey,
+		sessionToken: legacy.sessionToken,
+	});
+	const target = compactObject({
+		profile: storageProfileName,
+		bucket: legacy.bucket,
+		prefix: legacy.prefix,
+		namespace: legacy.profile ?? targetName,
+		autoSync: legacy.autoSync,
+		syncFiles: legacy.syncFiles,
+		syncSessions: legacy.syncSessions,
+		extraFiles: legacy.extraFiles,
+		legacyStateProfile: legacy.profile ?? "default",
+	});
+	return {
+		...unknown,
+		version: 2,
+		activeTarget: targetName,
+		profiles: { [storageProfileName]: profile },
+		targets: { [targetName]: target },
+	};
+}
+
+export async function saveNewV2Settings(input: {
+	targetName: string;
+	storageProfileName: string;
+	profile: StorageProfileSettings;
+	target: SyncTargetSettings;
+}) {
+	validateName(input.targetName, "target");
+	validateName(input.storageProfileName, "storage profile");
+	const current = (await readLocalConfigObject()) ?? {};
+	if (Object.keys(current).length > 0)
+		throw new Error(`Settings already exist: ${localConfigPath()}`);
+	const settings = {
+		version: 2,
+		activeTarget: input.targetName,
+		profiles: { [input.storageProfileName]: { ...input.profile } },
+		targets: {
+			[input.targetName]: { ...input.target, profile: input.storageProfileName },
+		},
+	};
+	await writeLocalConfigObject(settings);
+	return settings;
+}
+
+export async function addStorageProfile(name: string, profile: StorageProfileSettings) {
+	validateName(name, "storage profile");
+	await updateV2Settings((settings) => {
+		const profiles = requireObject(settings.profiles, "profiles");
+		if (Object.hasOwn(profiles, name)) throw new Error(`Storage profile already exists: ${name}`);
+		return { ...settings, profiles: { ...profiles, [name]: { ...profile } } };
+	});
+}
+
+export async function updateStorageProfile(
+	name: string,
+	update: (profile: Record<string, unknown>) => Record<string, unknown>,
+) {
+	validateName(name, "storage profile");
+	await updateV2Settings((settings) => {
+		const profiles = requireObject(settings.profiles, "profiles");
+		const profile = requireObject(profiles[name], "storage profile");
+		return { ...settings, profiles: { ...profiles, [name]: update(profile) } };
+	});
+}
+
+export async function addSyncTarget(name: string, target: SyncTargetSettings) {
+	validateName(name, "target");
+	await updateV2Settings((settings) => {
+		const targets = requireObject(settings.targets, "targets");
+		const profiles = requireObject(settings.profiles, "profiles");
+		if (Object.hasOwn(targets, name)) throw new Error(`Sync target already exists: ${name}`);
+		if (!target.profile || !Object.hasOwn(profiles, target.profile)) {
+			throw new Error(`Storage profile not found: ${target.profile ?? "missing"}`);
+		}
+		assertUniqueRemoteIdentity(targets, name, target);
+		return { ...settings, targets: { ...targets, [name]: { ...target } } };
+	});
+}
+
+export async function updateSyncTarget(
+	name: string,
+	update: (target: Record<string, unknown>) => Record<string, unknown>,
+) {
+	validateName(name, "target");
+	await updateV2Settings((settings) => {
+		const targets = requireObject(settings.targets, "targets");
+		const target = requireObject(targets[name], "target");
+		const nextTarget = update(target);
+		assertUniqueRemoteIdentity(targets, name, nextTarget);
+		return { ...settings, targets: { ...targets, [name]: nextTarget } };
+	});
+}
+
+export async function removeSyncTarget(name: string) {
+	validateName(name, "target");
+	await updateV2Settings((settings) => {
+		const targets = requireObject(settings.targets, "targets");
+		if (!Object.hasOwn(targets, name)) throw new Error(`Sync target not found: ${name}`);
+		if (settings.activeTarget === name && Object.keys(targets).length > 1) {
+			throw new Error("Switch to another target before removing the current target.");
+		}
+		const nextTargets = { ...targets };
+		delete nextTargets[name];
+		return {
+			...settings,
+			targets: nextTargets,
+			activeTarget: Object.keys(nextTargets)[0],
+		};
+	});
+}
+
+export async function removeStorageProfile(name: string) {
+	validateName(name, "storage profile");
+	await updateV2Settings((settings) => {
+		const profiles = requireObject(settings.profiles, "profiles");
+		const targets = requireObject(settings.targets, "targets");
+		const referenced = Object.entries(targets).find(
+			([, target]) => requireObject(target, "target").profile === name,
+		)?.[0];
+		if (referenced) throw new Error(`Storage profile “${name}” is used by target “${referenced}”.`);
+		if (!Object.hasOwn(profiles, name)) throw new Error(`Storage profile not found: ${name}`);
+		const nextProfiles = { ...profiles };
+		delete nextProfiles[name];
+		return { ...settings, profiles: nextProfiles };
+	});
+}
+
+async function updateV2Settings(
+	update: (settings: Record<string, unknown>) => Record<string, unknown>,
+) {
+	return withLock("settings", async () => {
+		const current = await readLocalConfigObject();
+		if (current?.version !== 2) {
+			throw new Error("Profiles and targets require version 2 pi-sync settings.");
+		}
+		const next = update(current);
+		await writeLocalConfigObject(next);
+		return next;
+	});
+}
+
+async function writeMigrationBackup(bytes: Buffer) {
+	const directory = path.join(stateDir(), "backups", "config");
+	await fs.mkdir(directory, { recursive: true });
+	const backupPath = path.join(
+		directory,
+		`pi-sync.local.${new Date().toISOString().replace(/[:.]/gu, "-")}.${randomUUID().slice(0, 8)}.json`,
+	);
+	await fs.writeFile(backupPath, bytes, { flag: "wx", mode: 0o600 });
+	if (process.platform !== "win32") await fs.chmod(backupPath, 0o600);
+	return backupPath;
+}
+
+async function adoptLegacyState(
+	legacy: Record<string, unknown>,
+	next: Record<string, unknown>,
+	targetName: string,
+) {
+	const legacyProfile = typeof legacy.profile === "string" ? legacy.profile : "default";
+	const source = path.join(stateDir(), `${safeName(legacyProfile)}.state.json`);
+	let bytes: Buffer;
+	try {
+		bytes = await fs.readFile(source);
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
+		throw error;
+	}
+	const targets = requireObject(next.targets, "targets");
+	const target = requireObject(targets[targetName], "target");
+	const profile = typeof target.namespace === "string" ? target.namespace : targetName;
+	const destination = statePathForConfig({
+		settingsVersion: 2,
+		target: targetName,
+		profile,
+	} as SyncConfig);
+	await fs.mkdir(path.dirname(destination), { recursive: true });
+	try {
+		await fs.writeFile(destination, bytes, { flag: "wx", mode: 0o600 });
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+	}
+}
+
+function assertUniqueRemoteIdentity(
+	targets: Record<string, unknown>,
+	name: string,
+	target: SyncTargetSettings,
+) {
+	const identity = effectiveTargetRemoteIdentity(target as Record<string, unknown>, name);
+	for (const [otherName, value] of Object.entries(targets)) {
+		if (
+			otherName !== name &&
+			effectiveTargetRemoteIdentity(requireObject(value, "target"), otherName) === identity
+		) {
+			throw new Error(`Target “${name}” duplicates the remote destination of “${otherName}”.`);
+		}
+	}
+}
+
+function compactObject(value: Record<string, unknown>) {
+	return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
+}
+
+function inferKind(endpoint: unknown) {
+	return typeof endpoint === "string" && endpoint.includes(".r2.cloudflarestorage.com")
+		? "r2"
+		: "s3-compatible";
+}
+
+function requireObject(value: unknown, name: string): Record<string, unknown> {
+	if (!value || typeof value !== "object" || Array.isArray(value)) {
+		throw new Error(`Invalid pi-sync settings: ${name} must be an object.`);
+	}
+	return value as Record<string, unknown>;
+}
+
+function validateName(value: string, label: string) {
+	if (
+		!value.trim() ||
+		value.length > 100 ||
+		value === "__proto__" ||
+		value === "prototype" ||
+		value === "constructor" ||
+		// biome-ignore lint/suspicious/noControlCharactersInRegex: Stored identifiers cannot contain controls.
+		/[\u0000-\u001f\u007f-\u009f]/u.test(value)
+	) {
+		throw new Error(`Invalid ${label} name.`);
+	}
+}

package/src/snapshot-apply.ts

@@ -3,7 +3,14 @@ import type { Dirent } from "node:fs";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { agentDir } from "./config.js";
-import { assertWithinRoot, isDeniedPath, isPathInside, parentPaths, safeJoin, toPosix } from "./paths.js";
+import {
+	assertWithinRoot,
+	isDeniedPath,
+	isPathInside,
+	parentPaths,
+	safeJoin,
+	toPosix,
+} from "./paths.js";
 import {
 	createSnapshot,
 	isSessionFilePath,
@@ -12,6 +19,10 @@ import {
 	snapshotIncludesSessions,
 	snapshotTarget,
 } from "./snapshot.js";
+import {
+	applySnapshotTransaction,
+	recoverPendingSnapshotTransactions,
+} from "./snapshot-transaction.js";
 import type { Snapshot, SnapshotApplyPlan, SnapshotOptions } from "./types.js";
 
 function sha256(value: Buffer) {
@@ -25,11 +36,13 @@ function fileHashMap(snapshot: Snapshot) {
 export async function applySnapshot(
 	snapshot: Snapshot,
 	protectedRelativePaths = new Set<string>(),
-	options: Pick<SnapshotOptions, "sessionDir" | "extraFiles"> = {},
+	options: Pick<SnapshotOptions, "syncFiles" | "sessionDir" | "extraFiles"> = {},
 ) {
 	const root = agentDir();
 	const { sessionDir } = options;
+	await recoverPendingSnapshotTransactions();
 	const current = await createSnapshot(snapshot.profile, {
+		syncFiles: options.syncFiles,
 		syncSessions: snapshotIncludesSessions(snapshot),
 		sessionDir,
 		extraFiles: options.extraFiles,
@@ -45,12 +58,7 @@ export async function applySnapshot(
 		snapshot,
 	);
 	await preflightSnapshotMutations(root, plan, sessionDir);
-	for (const target of plan.deletes) {
-		await fs.rm(target, { force: true, recursive: true });
-	}
-	for (const item of plan.writes) {
-		await fs.writeFile(item.target, item.content);
-	}
+	await applySnapshotTransaction(plan, { sessionDir });
 	return appliedFileHashMap(snapshot, current, protectedRelativePaths);
 }
 
@@ -79,7 +87,8 @@ export function preflightSnapshotApply(
 
 		const target = snapshotTarget(root, normalized, options.sessionDir);
 		const content = decodeBase64Strict(file.contentBase64, normalized);
-		if (sha256(content) !== file.sha256) throw new Error(`Checksum mismatch in snapshot file: ${normalized}`);
+		if (sha256(content) !== file.sha256)
+			throw new Error(`Checksum mismatch in snapshot file: ${normalized}`);
 		writes.push({ target, content });
 	}
 
@@ -108,7 +117,9 @@ export function protectSnapshotApplyPlan(
 ): SnapshotApplyPlan {
 	if (protectedRelativePaths.size === 0) return plan;
 	const protectedTargets = new Set(
-		[...protectedRelativePaths].map((relativePath) => snapshotTarget(root, relativePath, sessionDir)),
+		[...protectedRelativePaths].map((relativePath) =>
+			snapshotTarget(root, relativePath, sessionDir),
+		),
 	);
 	return {
 		writes: plan.writes.filter((item) => !protectedTargets.has(item.target)),
@@ -141,7 +152,11 @@ export async function addTopLevelCaseVariantDeletes(
 	const deletes = new Set(plan.deletes);
 	for (const entry of entries) {
 		const canonicalPath = topLevelPaths.get(entry.name.toLowerCase());
-		if (canonicalPath && entry.name !== canonicalPath && (entry.isFile() || entry.isSymbolicLink())) {
+		if (
+			canonicalPath &&
+			entry.name !== canonicalPath &&
+			(entry.isFile() || entry.isSymbolicLink())
+		) {
 			deletes.add(safeJoin(root, entry.name));
 		}
 	}
@@ -198,7 +213,11 @@ async function preflightSnapshotMutations(
 		await assertNoSymlinkParents(rootForTarget(root, target, sessionDir), target);
 	}
 	for (const item of plan.writes) {
-		await prepareSnapshotWrite(rootForTarget(root, item.target, sessionDir), item.target, deletePaths);
+		await prepareSnapshotWrite(
+			rootForTarget(root, item.target, sessionDir),
+			item.target,
+			deletePaths,
+		);
 	}
 }
 
@@ -212,7 +231,8 @@ async function prepareSnapshotWrite(root: string, target: string, deletePaths: S
 	await ensureSafeDirectory(root, path.dirname(target));
 	try {
 		const stat = await fs.lstat(target);
-		if (stat.isSymbolicLink()) throw new Error(`Refusing to overwrite symlink during snapshot apply: ${target}`);
+		if (stat.isSymbolicLink())
+			throw new Error(`Refusing to overwrite symlink during snapshot apply: ${target}`);
 		if (stat.isDirectory() && !deletePaths.has(target)) {
 			throw new Error(`Refusing to overwrite directory during snapshot apply: ${target}`);
 		}
@@ -230,8 +250,10 @@ async function ensureSafeDirectory(root: string, directory: string) {
 		current = path.join(current, part);
 		try {
 			const stat = await fs.lstat(current);
-			if (stat.isSymbolicLink()) throw new Error(`Refusing to follow symlink during snapshot apply: ${current}`);
-			if (!stat.isDirectory()) throw new Error(`Snapshot path parent is not a directory: ${current}`);
+			if (stat.isSymbolicLink())
+				throw new Error(`Refusing to follow symlink during snapshot apply: ${current}`);
+			if (!stat.isDirectory())
+				throw new Error(`Snapshot path parent is not a directory: ${current}`);
 		} catch (error) {
 			if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
 			await fs.mkdir(current);
@@ -249,8 +271,10 @@ async function assertNoSymlinkParents(root: string, target: string) {
 		current = path.join(current, part);
 		try {
 			const stat = await fs.lstat(current);
-			if (stat.isSymbolicLink()) throw new Error(`Refusing to follow symlink during snapshot apply: ${current}`);
-			if (!stat.isDirectory()) throw new Error(`Snapshot path parent is not a directory: ${current}`);
+			if (stat.isSymbolicLink())
+				throw new Error(`Refusing to follow symlink during snapshot apply: ${current}`);
+			if (!stat.isDirectory())
+				throw new Error(`Snapshot path parent is not a directory: ${current}`);
 		} catch (error) {
 			if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
 			throw error;