@narumitw/pi-sync 0.13.0 → 0.13.1

package/src/s3-client.ts

@@ -0,0 +1,194 @@
+import { createHash, createHmac } from "node:crypto";
+import { encodeKey, posixJoin } from "./paths.js";
+import type { LatestPointer, RemoteObject, Snapshot, SyncConfig } from "./types.js";
+
+const VERSION = 1;
+
+function iso8601Basic(date: Date) {
+	return date.toISOString().replace(/[:-]|\.\d{3}/g, "");
+}
+
+function snapshotIncludesSessions(snapshot: Snapshot) {
+	return snapshot.syncSessions === true || snapshot.files.some((file) => file.path.startsWith("sessions/"));
+}
+
+function isCloudflareR2Endpoint(endpoint: string | undefined) {
+	const value = endpoint?.trim();
+	if (!value) return false;
+	try {
+		const hostname = new URL(value).hostname.toLowerCase();
+		return hostname === "r2.cloudflarestorage.com" || hostname.endsWith(".r2.cloudflarestorage.com");
+	} catch {
+		return false;
+	}
+}
+
+export class S3Client {
+	private config: SyncConfig;
+	private endpoint: URL;
+	private omitSessionTokenAfterRejection = false;
+
+	constructor(config: SyncConfig) {
+		this.config = config;
+		this.endpoint = new URL(config.endpoint);
+	}
+
+	async getJson<T>(key: string): Promise<RemoteObject<T>> {
+		const object = await this.request("GET", key);
+		if (object.status === 404) return { missing: true };
+		if (!object.ok) throw new Error(`S3 GET failed (${object.status}): ${await object.text()}`);
+		return { value: (await object.json()) as T, etag: normalizeEtag(object.headers.get("etag")), missing: false };
+	}
+
+	async getBuffer(key: string): Promise<RemoteObject<Buffer>> {
+		const object = await this.request("GET", key);
+		if (object.status === 404) return { missing: true };
+		if (!object.ok) throw new Error(`S3 GET failed (${object.status}): ${await object.text()}`);
+		return { value: Buffer.from(await object.arrayBuffer()), etag: normalizeEtag(object.headers.get("etag")), missing: false };
+	}
+
+	async putJson(key: string, value: unknown) {
+		const body = Buffer.from(JSON.stringify(value, null, "\t"), "utf8");
+		await this.putBuffer(key, body, "application/json");
+	}
+
+	async putBuffer(key: string, body: Buffer, contentType: string) {
+		const headers: Record<string, string> = { "content-type": contentType };
+		const response = await this.request("PUT", key, body, headers);
+		if (!response.ok) throw new Error(`S3 PUT failed (${response.status}): ${await response.text()}`);
+	}
+
+	private async request(
+		method: "GET" | "PUT",
+		key: string,
+		body?: Buffer,
+		extraHeaders: Record<string, string> = {},
+	) {
+		const url = new URL(this.endpoint.toString());
+		url.pathname = posixJoin(url.pathname, this.config.bucket, encodeKey(key));
+		const send = async (sessionToken: string | undefined) => {
+			const headers = await signedHeaders({
+				method,
+				url,
+				body,
+				extraHeaders,
+				accessKeyId: this.config.accessKeyId,
+				secretAccessKey: this.config.secretAccessKey,
+				sessionToken,
+				region: this.config.region,
+			});
+			return fetch(url, { method, headers, body: body ? new Uint8Array(body) : undefined });
+		};
+		const sessionToken = this.omitSessionTokenAfterRejection ? undefined : this.config.sessionToken;
+		const response = await send(sessionToken);
+		if (!(await this.shouldRetryWithoutSessionToken(response, sessionToken))) return response;
+
+		const retry = await send(undefined);
+		if (retry.ok || retry.status === 404) this.omitSessionTokenAfterRejection = true;
+		return retry;
+	}
+
+	private async shouldRetryWithoutSessionToken(response: Response, sessionToken: string | undefined) {
+		if (
+			!sessionToken ||
+			!isCloudflareR2Endpoint(this.config.endpoint) ||
+			response.ok ||
+			response.status !== 400
+		) {
+			return false;
+		}
+		return isSecurityTokenInvalidArgument(await response.clone().text());
+	}
+}
+
+async function signedHeaders(input: {
+	method: string;
+	url: URL;
+	body?: Buffer;
+	extraHeaders: Record<string, string>;
+	accessKeyId: string;
+	secretAccessKey: string;
+	sessionToken?: string;
+	region: string;
+}) {
+	const now = new Date();
+	const amzDate = iso8601Basic(now);
+	const dateStamp = amzDate.slice(0, 8);
+	const payloadHash = sha256(input.body ?? Buffer.alloc(0));
+	const headers: Record<string, string> = {
+		...lowercaseKeys(input.extraHeaders),
+		host: input.url.host,
+		"x-amz-content-sha256": payloadHash,
+		"x-amz-date": amzDate,
+	};
+	if (input.sessionToken) headers["x-amz-security-token"] = input.sessionToken;
+	const signedHeaderNames = Object.keys(headers).sort();
+	const canonicalHeaders = signedHeaderNames.map((name) => `${name}:${headers[name]?.trim()}\n`).join("");
+	const canonicalRequest = [
+		input.method,
+		input.url.pathname,
+		input.url.searchParams.toString(),
+		canonicalHeaders,
+		signedHeaderNames.join(";"),
+		payloadHash,
+	].join("\n");
+	const scope = `${dateStamp}/${input.region}/s3/aws4_request`;
+	const stringToSign = ["AWS4-HMAC-SHA256", amzDate, scope, sha256(Buffer.from(canonicalRequest))].join("\n");
+	const signingKey = hmac(
+		hmac(hmac(hmac(Buffer.from(`AWS4${input.secretAccessKey}`), dateStamp), input.region), "s3"),
+		"aws4_request",
+	);
+	const signature = createHmac("sha256", signingKey).update(stringToSign).digest("hex");
+	return {
+		...headers,
+		authorization: `AWS4-HMAC-SHA256 Credential=${input.accessKeyId}/${scope}, SignedHeaders=${signedHeaderNames.join(";")}, Signature=${signature}`,
+	};
+}
+
+function hmac(key: Buffer, value: string) {
+	return createHmac("sha256", key).update(value).digest();
+}
+
+function sha256(value: Buffer) {
+	return createHash("sha256").update(value).digest("hex");
+}
+
+export function latestKey(config: SyncConfig) {
+	return posixJoin(profilePrefix(config), "latest.json");
+}
+
+export function historyKey(config: SyncConfig) {
+	return posixJoin(profilePrefix(config), "history.json");
+}
+
+export function snapshotKey(config: SyncConfig, id: string) {
+	return posixJoin(profilePrefix(config), "snapshots", `${id}.json.gz`);
+}
+
+export function profilePrefix(config: SyncConfig) {
+	return posixJoin(config.prefix, "profiles", config.profile);
+}
+
+export function pointerFor(config: SyncConfig, snapshot: Snapshot, checksum: string): LatestPointer {
+	return {
+		version: VERSION,
+		profile: config.profile,
+		snapshot: snapshot.id,
+		sha256: checksum,
+		createdAt: snapshot.createdAt,
+		machine: snapshot.machine,
+		syncSessions: snapshotIncludesSessions(snapshot),
+	};
+}
+
+function lowercaseKeys(value: Record<string, string>) {
+	return Object.fromEntries(Object.entries(value).map(([key, item]) => [key.toLowerCase(), item]));
+}
+
+function normalizeEtag(value: string | null) {
+	return value ?? undefined;
+}
+
+function isSecurityTokenInvalidArgument(text: string) {
+	return text.includes("<Code>InvalidArgument</Code>") && text.includes("<Message>X-Amz-Security-Token</Message>");
+}

package/src/snapshot-apply.ts

@@ -0,0 +1,259 @@
+import { createHash } from "node:crypto";
+import type { Dirent } from "node:fs";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { agentDir } from "./config.js";
+import { assertWithinRoot, isDeniedPath, isPathInside, parentPaths, safeJoin, toPosix } from "./paths.js";
+import {
+	createSnapshot,
+	isSessionFilePath,
+	isSessionPath,
+	sessionStorageRoot,
+	snapshotIncludesSessions,
+	snapshotTarget,
+} from "./snapshot.js";
+import type { Snapshot, SnapshotApplyPlan, SnapshotOptions } from "./types.js";
+
+function sha256(value: Buffer) {
+	return createHash("sha256").update(value).digest("hex");
+}
+
+function fileHashMap(snapshot: Snapshot) {
+	return Object.fromEntries(snapshot.files.map((file) => [file.path, file.sha256]));
+}
+
+export async function applySnapshot(
+	snapshot: Snapshot,
+	protectedRelativePaths = new Set<string>(),
+	options: Pick<SnapshotOptions, "sessionDir" | "extraFiles"> = {},
+) {
+	const root = agentDir();
+	const { sessionDir } = options;
+	const current = await createSnapshot(snapshot.profile, {
+		syncSessions: snapshotIncludesSessions(snapshot),
+		sessionDir,
+		extraFiles: options.extraFiles,
+	});
+	const plan = await addTopLevelCaseVariantDeletes(
+		root,
+		protectSnapshotApplyPlan(
+			root,
+			preflightSnapshotApply(root, snapshot, current, { sessionDir }),
+			protectedRelativePaths,
+			sessionDir,
+		),
+		snapshot,
+	);
+	await preflightSnapshotMutations(root, plan, sessionDir);
+	for (const target of plan.deletes) {
+		await fs.rm(target, { force: true, recursive: true });
+	}
+	for (const item of plan.writes) {
+		await fs.writeFile(item.target, item.content);
+	}
+	return appliedFileHashMap(snapshot, current, protectedRelativePaths);
+}
+
+export function preflightSnapshotApply(
+	root: string,
+	snapshot: Snapshot,
+	current: Snapshot,
+	options: { sessionDir?: string } = {},
+): SnapshotApplyPlan {
+	const seenPaths = new Set<string>();
+	const remotePaths = new Set<string>();
+	const writes: Array<{ target: string; content: Buffer }> = [];
+	const deletes: string[] = [];
+
+	for (const file of snapshot.files) {
+		const normalized = toPosix(file.path);
+		if (!isSafeSnapshotPath(file.path)) {
+			throw new Error(`Unsafe path in snapshot: ${file.path}`);
+		}
+		if (isSessionPath(normalized) && !isSessionFilePath(normalized)) {
+			throw new Error(`Unsafe session path in snapshot: ${file.path}`);
+		}
+		if (seenPaths.has(normalized)) throw new Error(`Duplicate path in snapshot: ${normalized}`);
+		seenPaths.add(normalized);
+		remotePaths.add(normalized);
+
+		const target = snapshotTarget(root, normalized, options.sessionDir);
+		const content = decodeBase64Strict(file.contentBase64, normalized);
+		if (sha256(content) !== file.sha256) throw new Error(`Checksum mismatch in snapshot file: ${normalized}`);
+		writes.push({ target, content });
+	}
+
+	const deletePaths = new Set<string>();
+	for (const file of current.files) {
+		const normalized = toPosix(file.path);
+		if (!remotePaths.has(normalized)) {
+			deletePaths.add(snapshotTarget(root, normalized, options.sessionDir));
+		}
+		for (const remotePath of parentPaths(normalized)) {
+			if (remotePaths.has(remotePath)) {
+				deletePaths.add(snapshotTarget(root, remotePath, options.sessionDir));
+			}
+		}
+	}
+	deletes.push(...deletePaths);
+
+	return { writes, deletes };
+}
+
+export function protectSnapshotApplyPlan(
+	root: string,
+	plan: SnapshotApplyPlan,
+	protectedRelativePaths: Set<string>,
+	sessionDir?: string,
+): SnapshotApplyPlan {
+	if (protectedRelativePaths.size === 0) return plan;
+	const protectedTargets = new Set(
+		[...protectedRelativePaths].map((relativePath) => snapshotTarget(root, relativePath, sessionDir)),
+	);
+	return {
+		writes: plan.writes.filter((item) => !protectedTargets.has(item.target)),
+		deletes: plan.deletes.filter((target) => !protectedTargets.has(target)),
+	};
+}
+
+export async function addTopLevelCaseVariantDeletes(
+	root: string,
+	plan: SnapshotApplyPlan,
+	snapshot: Pick<Snapshot, "files">,
+): Promise<SnapshotApplyPlan> {
+	const topLevelPaths = new Map<string, string>();
+	for (const file of snapshot.files) {
+		const normalized = toPosix(file.path);
+		if (!normalized.includes("/") && isSafeSnapshotPath(file.path)) {
+			topLevelPaths.set(normalized.toLowerCase(), normalized);
+		}
+	}
+	if (topLevelPaths.size === 0) return plan;
+
+	let entries: Dirent[];
+	try {
+		entries = await fs.readdir(root, { withFileTypes: true });
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === "ENOENT") return plan;
+		throw error;
+	}
+
+	const deletes = new Set(plan.deletes);
+	for (const entry of entries) {
+		const canonicalPath = topLevelPaths.get(entry.name.toLowerCase());
+		if (canonicalPath && entry.name !== canonicalPath && (entry.isFile() || entry.isSymbolicLink())) {
+			deletes.add(safeJoin(root, entry.name));
+		}
+	}
+	return { ...plan, deletes: [...deletes] };
+}
+
+export function appliedFileHashMap(
+	snapshot: Snapshot,
+	current: Snapshot,
+	protectedRelativePaths: Set<string>,
+) {
+	const hashes = fileHashMap(snapshot);
+	if (protectedRelativePaths.size === 0) return hashes;
+	const currentHashes = fileHashMap(current);
+	for (const relativePath of protectedRelativePaths) {
+		const normalized = toPosix(relativePath);
+		if (currentHashes[normalized]) {
+			hashes[normalized] = currentHashes[normalized];
+		} else {
+			delete hashes[normalized];
+		}
+	}
+	return hashes;
+}
+
+function isSafeSnapshotPath(relativePath: string) {
+	if (relativePath.includes("\\")) return false;
+	const normalized = toPosix(relativePath);
+	return (
+		Boolean(normalized) &&
+		normalized !== "." &&
+		normalized !== ".." &&
+		!normalized.startsWith("../") &&
+		!path.posix.isAbsolute(normalized) &&
+		path.posix.normalize(normalized) === normalized &&
+		!isDeniedPath(normalized)
+	);
+}
+
+function decodeBase64Strict(value: string, filePath: string) {
+	if (!/^[A-Za-z0-9+/]*={0,2}$/.test(value) || value.length % 4 !== 0) {
+		throw new Error(`Invalid base64 content in snapshot file: ${filePath}`);
+	}
+	return Buffer.from(value, "base64");
+}
+
+async function preflightSnapshotMutations(
+	root: string,
+	plan: { deletes: string[]; writes: Array<{ target: string; content: Buffer }> },
+	sessionDir?: string,
+) {
+	const deletePaths = new Set(plan.deletes);
+	for (const target of plan.deletes) {
+		await assertNoSymlinkParents(rootForTarget(root, target, sessionDir), target);
+	}
+	for (const item of plan.writes) {
+		await prepareSnapshotWrite(rootForTarget(root, item.target, sessionDir), item.target, deletePaths);
+	}
+}
+
+function rootForTarget(root: string, target: string, sessionDir?: string) {
+	const sessionRoot = sessionDir ? sessionStorageRoot(root, sessionDir) : undefined;
+	if (sessionRoot && isPathInside(sessionRoot, target)) return sessionRoot;
+	return root;
+}
+
+async function prepareSnapshotWrite(root: string, target: string, deletePaths: Set<string>) {
+	await ensureSafeDirectory(root, path.dirname(target));
+	try {
+		const stat = await fs.lstat(target);
+		if (stat.isSymbolicLink()) throw new Error(`Refusing to overwrite symlink during snapshot apply: ${target}`);
+		if (stat.isDirectory() && !deletePaths.has(target)) {
+			throw new Error(`Refusing to overwrite directory during snapshot apply: ${target}`);
+		}
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+	}
+}
+
+async function ensureSafeDirectory(root: string, directory: string) {
+	assertWithinRoot(root, directory);
+	const rootPath = path.resolve(root);
+	const relative = path.relative(rootPath, path.resolve(directory));
+	let current = rootPath;
+	for (const part of relative.split(path.sep).filter(Boolean)) {
+		current = path.join(current, part);
+		try {
+			const stat = await fs.lstat(current);
+			if (stat.isSymbolicLink()) throw new Error(`Refusing to follow symlink during snapshot apply: ${current}`);
+			if (!stat.isDirectory()) throw new Error(`Snapshot path parent is not a directory: ${current}`);
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+			await fs.mkdir(current);
+		}
+	}
+}
+
+async function assertNoSymlinkParents(root: string, target: string) {
+	assertWithinRoot(root, target);
+	const rootPath = path.resolve(root);
+	const relative = path.relative(rootPath, path.resolve(target));
+	let current = rootPath;
+	const parts = relative.split(path.sep).filter(Boolean);
+	for (const part of parts.slice(0, -1)) {
+		current = path.join(current, part);
+		try {
+			const stat = await fs.lstat(current);
+			if (stat.isSymbolicLink()) throw new Error(`Refusing to follow symlink during snapshot apply: ${current}`);
+			if (!stat.isDirectory()) throw new Error(`Snapshot path parent is not a directory: ${current}`);
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
+			throw error;
+		}
+	}
+}