@narumitw/pi-sync 0.31.0 → 0.34.0

package/src/s3-client.ts

@@ -1,21 +1,17 @@
 import { createHash, createHmac } from "node:crypto";
 import { setTimeout as sleep } from "node:timers/promises";
 import { encodeKey, posixJoin } from "./paths.js";
-import type { LatestPointer, RemoteObject, Snapshot, SyncConfig } from "./types.js";
+import type { RemoteObject, ResolvedS3Backend } from "./types.js";
 
-const VERSION = 1;
+const MAX_JSON_RESPONSE_BYTES = 1024 * 1024;
+const MAX_SNAPSHOT_RESPONSE_BYTES = 256 * 1024 * 1024;
+const MAX_ERROR_RESPONSE_BYTES = 64 * 1024;
+const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
 
 function iso8601Basic(date: Date) {
 	return date.toISOString().replace(/[:-]|\.\d{3}/g, "");
 }
 
-function snapshotIncludesSessions(snapshot: Snapshot) {
-	return (
-		snapshot.syncSessions === true ||
-		snapshot.files.some((file) => file.path.startsWith("sessions/"))
-	);
-}
-
 function isCloudflareR2Endpoint(endpoint: string | undefined) {
 	const value = endpoint?.trim();
 	if (!value) return false;
@@ -29,16 +25,29 @@ function isCloudflareR2Endpoint(endpoint: string | undefined) {
 	}
 }
 
+export class S3ObjectAlreadyExistsError extends Error {
+	constructor(readonly key: string) {
+		super(`S3 object already exists: ${key}`);
+		this.name = "S3ObjectAlreadyExistsError";
+	}
+}
+
 export class S3Client {
-	private config: SyncConfig;
+	private config: ResolvedS3Backend;
 	private endpoint: URL;
 	private signal?: AbortSignal;
+	private requestTimeoutMs: number;
 	private omitSessionTokenAfterRejection = false;
 
-	constructor(config: SyncConfig, signal?: AbortSignal) {
+	constructor(
+		config: ResolvedS3Backend,
+		signal?: AbortSignal,
+		requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS,
+	) {
 		this.config = config;
-		this.endpoint = new URL(config.endpoint);
+		this.endpoint = new URL(config.profile.endpoint);
 		this.signal = signal;
+		this.requestTimeoutMs = requestTimeoutMs;
 	}
 
 	async getJson<T>(key: string): Promise<RemoteObject<T>> {
@@ -47,8 +56,10 @@ export class S3Client {
 		for (let attempt = 1; attempt <= maxAttempts; attempt++) {
 			const object = await this.request("GET", key);
 			if (object.status === 404) return { missing: true };
-			if (!object.ok) throw new Error(`S3 GET failed (${object.status}): ${await object.text()}`);
-			const body = await object.text();
+			if (!object.ok) {
+				throw new Error(`S3 GET failed (${object.status}): ${await this.readErrorText(object)}`);
+			}
+			const body = await readBoundedText(object, MAX_JSON_RESPONSE_BYTES, "S3 JSON response");
 			// R2 can intermittently return an empty 200 body (read-after-write
 			// inconsistency on a long-lived keep-alive connection), which makes
 			// response.json() throw "JSON Parse error: Unexpected EOF" under Bun.
@@ -75,8 +86,14 @@ export class S3Client {
 		for (let attempt = 1; attempt <= maxAttempts; attempt++) {
 			const object = await this.request("GET", key);
 			if (object.status === 404) return { missing: true };
-			if (!object.ok) throw new Error(`S3 GET failed (${object.status}): ${await object.text()}`);
-			const buffer = Buffer.from(await object.arrayBuffer());
+			if (!object.ok) {
+				throw new Error(`S3 GET failed (${object.status}): ${await this.readErrorText(object)}`);
+			}
+			const buffer = await readBoundedBuffer(
+				object,
+				MAX_SNAPSHOT_RESPONSE_BYTES,
+				"S3 snapshot response",
+			);
 			// R2 can intermittently return an empty 200 body on a long-lived
 			// keep-alive connection (same root cause as the getJson retry
 			// above). getBuffer is only used for snapshot .json.gz payloads,
@@ -100,11 +117,21 @@ export class S3Client {
 		await this.putBuffer(key, body, "application/json");
 	}
 
-	async putBuffer(key: string, body: Buffer, contentType: string) {
+	async putBuffer(
+		key: string,
+		body: Buffer,
+		contentType: string,
+		options: { ifAbsent?: boolean } = {},
+	) {
 		const headers: Record<string, string> = { "content-type": contentType };
+		if (options.ifAbsent) headers["if-none-match"] = "*";
 		const response = await this.request("PUT", key, body, headers);
-		if (!response.ok)
-			throw new Error(`S3 PUT failed (${response.status}): ${await response.text()}`);
+		if (options.ifAbsent && response.status === 412) {
+			throw new S3ObjectAlreadyExistsError(key);
+		}
+		if (!response.ok) {
+			throw new Error(`S3 PUT failed (${response.status}): ${await this.readErrorText(response)}`);
+		}
 	}
 
 	private async request(
@@ -114,26 +141,37 @@ export class S3Client {
 		extraHeaders: Record<string, string> = {},
 	) {
 		const url = new URL(this.endpoint.toString());
-		url.pathname = posixJoin(url.pathname, this.config.bucket, encodeKey(key));
+		url.pathname = posixJoin(url.pathname, this.config.destination.bucket, encodeKey(key));
 		const send = async (sessionToken: string | undefined) => {
+			const transportSignal = this.transportSignal();
 			const headers = await signedHeaders({
 				method,
 				url,
 				body,
 				extraHeaders,
-				accessKeyId: this.config.accessKeyId,
-				secretAccessKey: this.config.secretAccessKey,
+				accessKeyId: this.config.profile.accessKeyId,
+				secretAccessKey: this.config.profile.secretAccessKey,
 				sessionToken,
-				region: this.config.region,
-			});
-			return fetch(url, {
-				method,
-				headers,
-				body: body ? new Uint8Array(body) : undefined,
-				signal: this.signal,
+				region: this.config.profile.region,
 			});
+			try {
+				return await fetch(url, {
+					method,
+					headers,
+					body: body ? new Uint8Array(body) : undefined,
+					signal: transportSignal,
+				});
+			} catch (error) {
+				if (error instanceof Error && error.name === "AbortError") throw error;
+				if (error instanceof Error && error.name === "TimeoutError") {
+					throw new Error("S3 request timed out.", { cause: error });
+				}
+				throw new Error(`S3 request failed: ${this.redact(errorMessage(error))}`, { cause: error });
+			}
 		};
-		const sessionToken = this.omitSessionTokenAfterRejection ? undefined : this.config.sessionToken;
+		const sessionToken = this.omitSessionTokenAfterRejection
+			? undefined
+			: this.config.profile.sessionToken;
 		const response = await send(sessionToken);
 		if (!(await this.shouldRetryWithoutSessionToken(response, sessionToken))) return response;
 
@@ -142,20 +180,97 @@ export class S3Client {
 		return retry;
 	}
 
+	private transportSignal() {
+		const timeout = AbortSignal.timeout(this.requestTimeoutMs);
+		return this.signal ? AbortSignal.any([this.signal, timeout]) : timeout;
+	}
+
 	private async shouldRetryWithoutSessionToken(
 		response: Response,
 		sessionToken: string | undefined,
 	) {
 		if (
 			!sessionToken ||
-			!isCloudflareR2Endpoint(this.config.endpoint) ||
+			!isCloudflareR2Endpoint(this.config.profile.endpoint) ||
 			response.ok ||
 			response.status !== 400
 		) {
 			return false;
 		}
-		return isSecurityTokenInvalidArgument(await response.clone().text());
+		return isSecurityTokenInvalidArgument(
+			await readBoundedText(response.clone(), MAX_ERROR_RESPONSE_BYTES, "S3 error response"),
+		);
+	}
+
+	private async readErrorText(response: Response) {
+		try {
+			return this.redact(
+				await readBoundedText(response, MAX_ERROR_RESPONSE_BYTES, "S3 error response"),
+			);
+		} catch (error) {
+			return this.redact(errorMessage(error));
+		}
+	}
+
+	private redact(value: string) {
+		let redacted = value;
+		let endpointUsername: string | undefined;
+		let endpointPassword: string | undefined;
+		let endpointQueryValues: string[] = [];
+		try {
+			const endpoint = new URL(this.config.profile.endpoint);
+			endpointUsername = endpoint.username;
+			endpointPassword = endpoint.password;
+			endpointQueryValues = [...endpoint.searchParams.values()];
+		} catch {
+			// The constructor already validates normal runtime endpoints.
+		}
+		for (const secret of [
+			this.config.profile.accessKeyId,
+			this.config.profile.secretAccessKey,
+			this.config.profile.sessionToken,
+			endpointUsername,
+			endpointPassword,
+			...endpointQueryValues,
+		]) {
+			if (secret) redacted = redacted.replaceAll(secret, "[REDACTED]");
+		}
+		return redacted;
+	}
+}
+
+async function readBoundedText(response: Response, limit: number, label: string) {
+	return (await readBoundedBuffer(response, limit, label)).toString("utf8");
+}
+
+async function readBoundedBuffer(response: Response, limit: number, label: string) {
+	const contentLength = Number(response.headers.get("content-length"));
+	if (Number.isFinite(contentLength) && contentLength > limit) {
+		throw new Error(`${label} exceeds the ${limit}-byte limit.`);
 	}
+	if (!response.body) return Buffer.alloc(0);
+	const reader = response.body.getReader();
+	const chunks: Buffer[] = [];
+	let total = 0;
+	try {
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			total += value.byteLength;
+			if (total > limit) {
+				await reader.cancel().catch(() => undefined);
+				throw new Error(`${label} exceeds the ${limit}-byte limit.`);
+			}
+			chunks.push(Buffer.from(value));
+		}
+	} finally {
+		reader.releaseLock();
+	}
+	return Buffer.concat(chunks, total);
+}
+
+function errorMessage(error: unknown) {
+	return error instanceof Error ? error.message : String(error);
 }
 
 async function signedHeaders(input: {
@@ -217,38 +332,6 @@ function sha256(value: Buffer) {
 	return createHash("sha256").update(value).digest("hex");
 }
 
-export function latestKey(config: SyncConfig) {
-	return posixJoin(profilePrefix(config), "latest.json");
-}
-
-export function historyKey(config: SyncConfig) {
-	return posixJoin(profilePrefix(config), "history.json");
-}
-
-export function snapshotKey(config: SyncConfig, id: string) {
-	return posixJoin(profilePrefix(config), "snapshots", `${id}.json.gz`);
-}
-
-export function profilePrefix(config: SyncConfig) {
-	return posixJoin(config.prefix, "profiles", config.profile);
-}
-
-export function pointerFor(
-	config: SyncConfig,
-	snapshot: Snapshot,
-	checksum: string,
-): LatestPointer {
-	return {
-		version: VERSION,
-		profile: config.profile,
-		snapshot: snapshot.id,
-		sha256: checksum,
-		createdAt: snapshot.createdAt,
-		machine: snapshot.machine,
-		syncSessions: snapshotIncludesSessions(snapshot),
-	};
-}
-
 function lowercaseKeys(value: Record<string, string>) {
 	return Object.fromEntries(Object.entries(value).map(([key, item]) => [key.toLowerCase(), item]));
 }

package/src/s3-credentials-ui.ts

@@ -0,0 +1,92 @@
+import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
+import { promptSecret } from "./secret-input.js";
+
+export interface ChosenS3Credentials {
+	profileFields: { accessKeyId?: string; secretAccessKey?: string };
+	summary: string;
+	ready: boolean;
+	replace?: boolean;
+}
+
+export async function chooseS3CredentialUpdate(
+	ctx: ExtensionCommandContext,
+	profile: Record<string, unknown>,
+	signal?: AbortSignal,
+) {
+	const hasStored =
+		typeof profile.accessKeyId === "string" && typeof profile.secretAccessKey === "string";
+	if (hasStored) {
+		const action = await ctx.ui.select(
+			"Credentials",
+			["Keep current credentials", "Change credential source", "Cancel"],
+			{ signal },
+		);
+		throwIfAborted(signal);
+		if (!action || action === "Cancel") return undefined;
+		if (action === "Keep current credentials") {
+			return { profileFields: {}, summary: "Unchanged (values hidden)", ready: true };
+		}
+	}
+	const selected = await chooseS3Credentials(ctx, signal);
+	return selected ? { ...selected, replace: true } : undefined;
+}
+
+export function applyS3CredentialUpdate(
+	profile: Record<string, unknown>,
+	credentials: ChosenS3Credentials,
+) {
+	const next = { ...profile };
+	if (credentials.replace) {
+		delete next.accessKeyId;
+		delete next.secretAccessKey;
+		delete next.sessionToken;
+	}
+	return { ...next, ...credentials.profileFields };
+}
+
+export async function chooseS3Credentials(
+	ctx: ExtensionCommandContext,
+	signal?: AbortSignal,
+): Promise<ChosenS3Credentials | undefined> {
+	const choice = await ctx.ui.select(
+		"Credentials\n\nCredentials are stored in the private pi-sync settings file. Secret values are masked during input and never shown afterward.",
+		["Store credentials privately", "Cancel"],
+		{ signal },
+	);
+	throwIfAborted(signal);
+	if (choice !== "Store credentials privately") return undefined;
+	const accessKeyId = await requiredCredentialInput(ctx, "Access key ID", "access-key-id", signal);
+	if (!accessKeyId) return undefined;
+	const secretAccessKey = await promptSecret(ctx, "Secret access key", { signal });
+	throwIfAborted(signal);
+	if (secretAccessKey === undefined) return undefined;
+	return {
+		profileFields: { accessKeyId, secretAccessKey },
+		summary: "Stored privately (values hidden)",
+		ready: true,
+	};
+}
+
+async function requiredCredentialInput(
+	ctx: ExtensionCommandContext,
+	title: string,
+	placeholder: string,
+	signal?: AbortSignal,
+) {
+	const value = await ctx.ui.input(title, placeholder, { signal });
+	throwIfAborted(signal);
+	if (value === undefined) return undefined;
+	const normalized = value.trim();
+	if (!normalized) {
+		ctx.ui.notify(`${title} is required.`, "warning");
+		return undefined;
+	}
+	return normalized.includes("<") || normalized.includes(">") ? undefined : normalized;
+}
+
+function throwIfAborted(signal?: AbortSignal) {
+	if (!signal?.aborted) return;
+	throw signal.reason instanceof Error
+		? signal.reason
+		: new DOMException("The operation was aborted", "AbortError");
+}

package/src/secret-input.ts

@@ -0,0 +1,199 @@
+import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
+import {
+	CURSOR_MARKER,
+	decodeKittyPrintable,
+	type Focusable,
+	type KeybindingsManager,
+	Text,
+	truncateToWidth,
+} from "@earendil-works/pi-tui";
+
+const MASK = "•";
+
+export async function promptSecret(
+	ctx: ExtensionCommandContext,
+	title: string,
+	options: { required?: boolean; signal?: AbortSignal } = { required: true },
+) {
+	if (ctx.mode !== "tui" || options.signal?.aborted) return undefined;
+	const value = await ctx.ui.custom<string | undefined>((tui, theme, keybindings, done) => {
+		const heading = new Text("", 0, 0);
+		const hint = new Text("", 0, 0);
+		const applyTheme = () => {
+			heading.setText(theme.fg("accent", theme.bold(title)));
+			hint.setText(theme.fg("dim", "Enter save • Escape cancel • Input is hidden"));
+		};
+		applyTheme();
+		const input = new MaskedInput(keybindings);
+		let settled = false;
+		const finish = (result: string | undefined) => {
+			if (settled) return;
+			settled = true;
+			done(result);
+		};
+		const onAbort = () => finish(undefined);
+		options.signal?.addEventListener("abort", onAbort, { once: true });
+		const component: Focusable & {
+			render(width: number): string[];
+			invalidate(): void;
+			handleInput(data: string): void;
+			dispose(): void;
+		} = {
+			get focused() {
+				return input.focused;
+			},
+			set focused(focused: boolean) {
+				input.focused = focused;
+			},
+			render(width: number) {
+				const safeWidth = Math.max(1, width);
+				return [
+					...heading.render(safeWidth),
+					...input.render(safeWidth),
+					...hint.render(safeWidth),
+				].map((line) => truncateToWidth(line, safeWidth));
+			},
+			invalidate() {
+				applyTheme();
+				heading.invalidate();
+				input.invalidate();
+				hint.invalidate();
+			},
+			handleInput(data: string) {
+				if (keybindings.matches(data, "tui.select.cancel")) finish(undefined);
+				else if (keybindings.matches(data, "tui.input.submit")) finish(input.getValue());
+				else input.handleInput(data);
+				tui.requestRender();
+			},
+			dispose() {
+				options.signal?.removeEventListener("abort", onAbort);
+				input.clear();
+			},
+		};
+		return component;
+	});
+	if (value === undefined) return undefined;
+	if (options.required !== false && value.length === 0) {
+		ctx.ui.notify(`${title} is required.`, "warning");
+		return undefined;
+	}
+	return value;
+}
+
+class MaskedInput implements Focusable {
+	focused = false;
+	private value: string[] = [];
+	private cursor = 0;
+	private paste = "";
+	private pasting = false;
+
+	constructor(private readonly keybindings: KeybindingsManager) {}
+
+	getValue() {
+		return this.value.join("");
+	}
+
+	handleInput(data: string) {
+		if (data.includes("\u001b[200~")) {
+			this.pasting = true;
+			this.paste = "";
+			data = data.replace("\u001b[200~", "");
+		}
+		if (this.pasting) {
+			this.paste += data;
+			const end = this.paste.indexOf("\u001b[201~");
+			if (end < 0) return;
+			const pasted = this.paste
+				.slice(0, end)
+				.replace(/[\r\n]/gu, "")
+				.replace(/\t/gu, "    ");
+			this.insert(pasted);
+			const remaining = this.paste.slice(end + 6);
+			this.paste = "";
+			this.pasting = false;
+			if (remaining) this.handleInput(remaining);
+			return;
+		}
+		if (this.keybindings.matches(data, "tui.editor.deleteCharBackward")) {
+			if (this.cursor > 0) this.value.splice(--this.cursor, 1);
+			return;
+		}
+		if (this.keybindings.matches(data, "tui.editor.deleteCharForward")) {
+			if (this.cursor < this.value.length) this.value.splice(this.cursor, 1);
+			return;
+		}
+		if (this.keybindings.matches(data, "tui.editor.cursorLeft")) {
+			this.cursor = Math.max(0, this.cursor - 1);
+			return;
+		}
+		if (this.keybindings.matches(data, "tui.editor.cursorRight")) {
+			this.cursor = Math.min(this.value.length, this.cursor + 1);
+			return;
+		}
+		if (this.keybindings.matches(data, "tui.editor.cursorLineStart")) {
+			this.cursor = 0;
+			return;
+		}
+		if (this.keybindings.matches(data, "tui.editor.cursorLineEnd")) {
+			this.cursor = this.value.length;
+			return;
+		}
+		if (this.keybindings.matches(data, "tui.editor.deleteToLineStart")) {
+			this.value.splice(0, this.cursor);
+			this.cursor = 0;
+			return;
+		}
+		if (this.keybindings.matches(data, "tui.editor.deleteToLineEnd")) {
+			this.value.splice(this.cursor);
+			return;
+		}
+		const printable = decodeKittyPrintable(data) ?? data;
+		if (!hasControlCharacter(printable)) this.insert(printable);
+	}
+
+	render(width: number) {
+		const prompt = "> ";
+		const available = width - prompt.length;
+		if (available <= 0) return [truncateToWidth(prompt, Math.max(1, width))];
+		const contentWidth = Math.max(0, available - 1);
+		let start = 0;
+		if (this.value.length > contentWidth) {
+			start = Math.max(
+				0,
+				Math.min(this.cursor - Math.floor(contentWidth / 2), this.value.length - contentWidth),
+			);
+		}
+		const end = Math.min(this.value.length, start + contentWidth);
+		const visibleCursor = Math.max(0, Math.min(this.cursor - start, end - start));
+		const masks = Array.from({ length: end - start }, () => MASK);
+		const before = masks.slice(0, visibleCursor).join("");
+		const atCursor = visibleCursor < masks.length ? MASK : " ";
+		const after = masks.slice(visibleCursor + (visibleCursor < masks.length ? 1 : 0)).join("");
+		const marker = this.focused ? CURSOR_MARKER : "";
+		const line = `${prompt}${before}${marker}\u001b[7m${atCursor}\u001b[27m${after}`;
+		return [truncateToWidth(line, width, "")];
+	}
+
+	invalidate() {}
+
+	clear() {
+		this.value.fill("");
+		this.value = [];
+		this.paste = "";
+		this.cursor = 0;
+		this.pasting = false;
+	}
+
+	private insert(value: string) {
+		const characters = Array.from(value);
+		this.value.splice(this.cursor, 0, ...characters);
+		this.cursor += characters.length;
+	}
+}
+
+function hasControlCharacter(value: string) {
+	return [...value].some((character) => {
+		const code = character.codePointAt(0) ?? 0;
+		return code < 0x20 || (code >= 0x7f && code <= 0x9f);
+	});
+}