@davideasden/pi-undo 0.1.1 → 0.2.0

package/src/snapshot-store.ts

@@ -1,3 +1,4 @@
+import type { Stats } from "node:fs";
 import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, realpath, rm, stat } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -30,6 +31,11 @@ const GC_METADATA_FILE = "gc.json";
 const GC_RETENTION_MS = 7 * 24 * 60 * 60 * 1_000;
 const IGNORE_POLICY = "git-check-ignore-v1";
 const NULL_DEVICE = process.platform === "win32" ? "NUL" : "/dev/null";
+const TREE_CACHE_LIMIT = 256;
+const HASH_BATCH_MAX_PATHS = 128;
+const HASH_BATCH_MAX_ARGUMENT_BYTES = 24 * 1024;
+const INDEX_BATCH_MAX_ENTRIES = 4_096;
+const INDEX_BATCH_MAX_BYTES = 8 * 1024 * 1024;
 
 interface PinRecord {
 	readonly schemaVersion: 1;
@@ -60,6 +66,13 @@ interface CapturedRootResult {
 	readonly objectClosure: string;
 }
 
+interface VisibleLeaf {
+	readonly relativePath: string;
+	readonly kind: "file" | "symlink";
+	readonly mode: number;
+	readonly fingerprint: string;
+}
+
 export interface SnapshotStoreOptions {
 	readonly storeRoot?: string;
 	readonly git?: GitRunner;
@@ -93,6 +106,7 @@ export class SnapshotStoreError extends Error {
 
 export interface SnapshotStore {
 	capture(topology: RootTopology, scope?: readonly string[], options?: CaptureOptions): Promise<SnapshotManifest>;
+	listVisibleLeafPaths(topology: RootTopology, options?: CaptureOptions): Promise<readonly string[]>;
 	loadManifest(id: ManifestId): Promise<SnapshotManifest>;
 	assertComplete(id: ManifestId): Promise<void>;
 	listTree(id: ManifestId, root: string): Promise<readonly RestorePath[]>;
@@ -110,6 +124,8 @@ export class SnapshotStore {
 	private readonly lock: WorkspaceLock;
 	private readonly clock: () => number;
 	private readonly manifestLocations = new Map<string, string>();
+	// Tree ID 是内容寻址且不可变；这里只缓存解析元数据，blob bytes 与完整性仍逐次读取/校验。
+	private readonly treeEntriesCache = new Map<string, Promise<CapturedTreeEntry[]>>();
 
 	constructor(options: SnapshotStoreOptions = {}) {
 		this.storeRoot = resolve(options.storeRoot ?? join(tmpdir(), "pi-undo-snapshot-store"));
@@ -204,6 +220,72 @@ export class SnapshotStore {
 		}
 	}
 
+	async listVisibleLeafPaths(
+		topology: RootTopology,
+		options: CaptureOptions = {},
+	): Promise<readonly string[]> {
+		await this.assertPrivateStore(topology.workspaceIdentity);
+		const lockIdentity = `snapshot-store:${await prospectiveCanonicalPath(this.storesRoot)}`;
+		return this.lock.withLock(lockIdentity, () => this.listVisibleLeafPathsLocked(topology, options));
+	}
+
+	private async listVisibleLeafPathsLocked(
+		topology: RootTopology,
+		options: CaptureOptions,
+	): Promise<readonly string[]> {
+		let transactionDirectory: string | undefined;
+		try {
+			if (topology.fingerprint !== topologyFingerprint(topology.workspaceIdentity, topology.roots)) {
+				throw new SnapshotStoreError("capture_failed", "topology fingerprint 与 roots 不匹配");
+			}
+			const artifactExclusions = captureExclusions(topology.workspaceIdentity, options.excludePaths);
+			await this.assertTopology(topology, "可见路径枚举前 topology 已变化");
+			if (topology.roots.some((root) => root.state === "broken")) {
+				throw new SnapshotStoreError("capture_failed", "broken root 不能静默进入可见路径枚举");
+			}
+
+			const storeDirectory = this.storeDirectory(topology);
+			const transactionsRoot = join(storeDirectory, "transactions");
+			await mkdir(transactionsRoot, { recursive: true });
+			transactionDirectory = await mkdtemp(join(transactionsRoot, "visible-"));
+			const result = new Set<string>();
+			for (const root of topology.roots) {
+				if (root.state !== "active") continue;
+				const gitDirectory = this.rootGitDirectory(storeDirectory, root);
+				await this.ensurePrivateRepository(gitDirectory);
+				await this.assertNoAlternates(gitDirectory);
+				const absoluteRoot = workspaceRootPath(topology.workspaceIdentity, root.relativeRoot);
+				const indexPath = join(transactionDirectory, `${rootStoreId(root)}.index`);
+				const environment = privateGitEnvironment(gitDirectory, absoluteRoot, indexPath);
+				await this.runGit(["read-tree", "--empty"], { cwd: absoluteRoot, env: environment });
+				await this.validateIgnoreQuery(absoluteRoot, environment, root.gitBacked);
+				const exclusions = topology.roots
+					.filter((candidate) => isStrictRootAncestor(root.relativeRoot, candidate.relativeRoot))
+					.map((candidate) => rootRelativePath(root.relativeRoot, candidate.relativeRoot));
+				const exactExclusions = ownedArtifactExclusions(topology.roots, root.relativeRoot, artifactExclusions);
+				for (const leaf of await this.collectVisibleLeaves(
+					absoluteRoot,
+					environment,
+					root.gitBacked,
+					[],
+					exclusions,
+					exactExclusions,
+				)) {
+					result.add(workspaceRelativePath(root.relativeRoot, leaf.relativePath));
+				}
+			}
+			await this.assertTopology(topology, "可见路径枚举期间 topology 已变化");
+			return [...result].sort(comparePaths);
+		} catch (error) {
+			if (error instanceof SnapshotStoreError) throw error;
+			throw new SnapshotStoreError("capture_failed", errorMessage(error), { cause: error });
+		} finally {
+			if (transactionDirectory !== undefined) {
+				await rm(transactionDirectory, { recursive: true, force: true }).catch(() => {});
+			}
+		}
+	}
+
 	async loadManifest(id: ManifestId): Promise<SnapshotManifest> {
 		const manifestPath = await this.findManifestPath(id);
 		try {
@@ -242,7 +324,6 @@ export class SnapshotStore {
 				}
 				const gitDirectory = this.rootGitDirectory(storeDirectory, root);
 				await this.assertNoAlternates(gitDirectory);
-				await this.runPrivateGit(gitDirectory, ["cat-file", "-e", `${root.treeId}^{tree}`]);
 				const entries = await this.readTreeEntries(gitDirectory, root.treeId);
 				if (root.ignoredPresentPaths.some((ignoredPath) => entries.some(
 					(entry) => isPathAtOrBelow(ignoredPath, entry.relativePath) ||
@@ -250,9 +331,7 @@ export class SnapshotStore {
 				))) {
 					throw new SnapshotStoreError("object_missing", "ignored-present proof 与 root tree 冲突");
 				}
-				for (const entry of entries) {
-					await this.runPrivateGit(gitDirectory, ["cat-file", "-e", `${entry.objectId}^{blob}`]);
-				}
+				await this.assertObjectsComplete(gitDirectory, root.treeId, entries);
 				if (root.objectClosure !== treeObjectClosure(root.treeId, entries)) {
 					throw new SnapshotStoreError("object_missing", "root tree 对象闭包校验失败");
 				}
@@ -493,10 +572,7 @@ export class SnapshotStore {
 			throw new SnapshotStoreError("capture_failed", "git write-tree 未返回有效对象 ID");
 		}
 		const entries = await this.readTreeEntries(gitDirectory, treeId);
-		await this.runPrivateGit(gitDirectory, ["cat-file", "-e", `${treeId}^{tree}`]);
-		for (const entry of entries) {
-			await this.runPrivateGit(gitDirectory, ["cat-file", "-e", `${entry.objectId}^{blob}`]);
-		}
+		await this.assertObjectsComplete(gitDirectory, treeId, entries);
 		return {
 			treeId,
 			coverage,
@@ -565,13 +641,77 @@ export class SnapshotStore {
 		exclusions: readonly string[],
 		exactExclusions: readonly string[],
 	): Promise<void> {
-		if (inclusions === null) {
-			return;
+		const leaves = await this.collectVisibleLeaves(
+			cwd,
+			environment,
+			gitBacked,
+			inclusions,
+			exclusions,
+			exactExclusions,
+		);
+		const objectIds = new Map<string, string>();
+		for (const batch of hashPathBatches(leaves.filter((leaf) => leaf.kind === "file"))) {
+			for (const leaf of batch) await this.assertVisibleLeafUnchanged(cwd, leaf);
+			const output = await this.runGit([
+				"hash-object",
+				"-w",
+				"--no-filters",
+				"--",
+				...batch.map((leaf) => leaf.relativePath),
+			], { cwd, env: environment });
+			const hashes = parseObjectIdLines(output, batch.length);
+			for (const leaf of batch) await this.assertVisibleLeafUnchanged(cwd, leaf);
+			for (let index = 0; index < batch.length; index += 1) {
+				objectIds.set(batch[index]!.relativePath, hashes[index]!);
+			}
 		}
-		const pathspecs = inclusions.length === 0 ? ["."] : inclusions.map(literalPathspec);
-		for (const excluded of exclusions) {
-			pathspecs.push(excludeLiteralPathspec(excluded));
+		for (const leaf of leaves) {
+			if (leaf.kind !== "symlink") continue;
+			await this.assertVisibleLeafUnchanged(cwd, leaf);
+			const linkText = await readlink(join(cwd, ...leaf.relativePath.split("/")), { encoding: "buffer" });
+			decodeUtf8(linkText, "symlink target 不是可无损表示的 UTF-8");
+			const objectId = (await this.runGit(["hash-object", "-w", "--stdin"], {
+				cwd,
+				env: environment,
+				stdin: linkText,
+			})).trim();
+			if (!isObjectId(objectId)) {
+				throw new SnapshotStoreError("capture_failed", `文件对象 materialize 失败：${leaf.relativePath}`);
+			}
+			await this.assertVisibleLeafUnchanged(cwd, leaf);
+			objectIds.set(leaf.relativePath, objectId);
+		}
+		for (const indexInput of indexInfoBatches(leaves, objectIds)) {
+			await this.runGit(["update-index", "-z", "--index-info"], {
+				cwd,
+				env: environment,
+				stdin: indexInput,
+			});
+		}
+	}
+
+	private async assertVisibleLeafUnchanged(cwd: string, leaf: VisibleLeaf): Promise<void> {
+		await assertNoSymlinkEscape(cwd, leaf.relativePath);
+		const metadata = await lstat(join(cwd, ...leaf.relativePath.split("/"))).catch((error) => {
+			if (hasErrorCode(error, "ENOENT")) return null;
+			throw error;
+		});
+		if (metadata === null || visibleLeafFingerprint(metadata) !== leaf.fingerprint) {
+			throw new SnapshotStoreError("capture_failed", `捕获期间工作区叶子已变化：${leaf.relativePath}`);
 		}
+	}
+
+	private async collectVisibleLeaves(
+		cwd: string,
+		environment: Readonly<Record<string, string | undefined>>,
+		gitBacked: boolean,
+		inclusions: readonly string[] | null,
+		exclusions: readonly string[],
+		exactExclusions: readonly string[],
+	): Promise<VisibleLeaf[]> {
+		if (inclusions === null) return [];
+		const pathspecs = inclusions.length === 0 ? ["."] : inclusions.map(literalPathspec);
+		for (const excluded of exclusions) pathspecs.push(excludeLiteralPathspec(excluded));
 		const queryEnvironment = gitBacked ? sourceGitEnvironment() : environment;
 		const output = await this.runGitBytes([
 			...(gitBacked ? ["-c", "core.fsmonitor=false"] : []),
@@ -583,53 +723,38 @@ export class SnapshotStore {
 			"--",
 			...pathspecs,
 		], { cwd, env: queryEnvironment });
+		const result: VisibleLeaf[] = [];
 		for (const relativePath of parseNulPaths(output)) {
 			if (
 				exclusions.some((excluded) => isPathAtOrBelow(excluded, relativePath)) ||
 				exactExclusions.includes(relativePath)
-			) {
-				continue;
-			}
+			) continue;
 			relativeSafePath(cwd, relativePath);
 			await assertNoSymlinkEscape(cwd, relativePath);
-			const absolutePath = join(cwd, ...relativePath.split("/"));
-			const metadata = await lstat(absolutePath).catch((error) => {
-				if (hasErrorCode(error, "ENOENT")) {
-					return null;
-				}
+			const metadata = await lstat(join(cwd, ...relativePath.split("/"))).catch((error) => {
+				if (hasErrorCode(error, "ENOENT")) return null;
 				throw error;
 			});
-			if (metadata === null) {
-				continue;
-			}
-			let mode: string;
-			let objectId: string;
+			if (metadata === null) continue;
 			if (metadata.isSymbolicLink()) {
-				mode = "120000";
-				const linkText = await readlink(absolutePath, { encoding: "buffer" });
-				decodeUtf8(linkText, "symlink target 不是可无损表示的 UTF-8");
-				objectId = (await this.runGit(["hash-object", "-w", "--stdin"], {
-					cwd,
-					env: environment,
-					stdin: linkText,
-				})).trim();
+				result.push({
+					relativePath,
+					kind: "symlink",
+					mode: 0o120000,
+					fingerprint: visibleLeafFingerprint(metadata),
+				});
 			} else if (metadata.isFile()) {
-				mode = (metadata.mode & 0o111) === 0 ? "100644" : "100755";
-				objectId = (await this.runGit(["hash-object", "-w", "--no-filters", "--", relativePath], {
-					cwd,
-					env: environment,
-				})).trim();
+				result.push({
+					relativePath,
+					kind: "file",
+					mode: (metadata.mode & 0o111) === 0 ? 0o100644 : 0o100755,
+					fingerprint: visibleLeafFingerprint(metadata),
+				});
 			} else {
 				throw new SnapshotStoreError("capture_failed", `不支持的工作区文件类型：${relativePath}`);
 			}
-			if (!isObjectId(objectId)) {
-				throw new SnapshotStoreError("capture_failed", `文件对象 materialize 失败：${relativePath}`);
-			}
-			await this.runGit(["update-index", "--add", "--cacheinfo", mode, objectId, relativePath], {
-				cwd,
-				env: environment,
-			});
 		}
+		return result;
 	}
 
 	private async validateIgnoreQuery(
@@ -688,8 +813,52 @@ export class SnapshotStore {
 	}
 
 	private async readTreeEntries(gitDirectory: string, treeId: string): Promise<CapturedTreeEntry[]> {
-		const output = await this.runPrivateGitBytes(gitDirectory, ["ls-tree", "-r", "-l", "-z", treeId]);
-		return parseTreeEntries(output);
+		const key = `${gitDirectory}\0${treeId}`;
+		const cached = this.treeEntriesCache.get(key);
+		if (cached !== undefined) return cached;
+		const pending = this.runPrivateGitBytes(gitDirectory, ["ls-tree", "-r", "-l", "-z", treeId])
+			.then(parseTreeEntries);
+		this.treeEntriesCache.set(key, pending);
+		while (this.treeEntriesCache.size > TREE_CACHE_LIMIT) {
+			const oldest = this.treeEntriesCache.keys().next().value as string | undefined;
+			if (oldest === undefined || oldest === key) break;
+			this.treeEntriesCache.delete(oldest);
+		}
+		try {
+			return await pending;
+		} catch (error) {
+			if (this.treeEntriesCache.get(key) === pending) this.treeEntriesCache.delete(key);
+			throw error;
+		}
+	}
+
+	private async assertObjectsComplete(
+		gitDirectory: string,
+		treeId: string,
+		entries: readonly CapturedTreeEntry[],
+	): Promise<void> {
+		const expected = [
+			{ objectId: treeId, type: "tree" },
+			...[...new Set(entries.map((entry) => entry.objectId))].map((objectId) => ({ objectId, type: "blob" })),
+		];
+		const output = await this.runGit(["cat-file", "--batch-check"], {
+			env: privateObjectEnvironment(gitDirectory),
+			stdin: `${expected.map((object) => object.objectId).join("\n")}\n`,
+		});
+		const lines = output.endsWith("\n") ? output.slice(0, -1).split("\n") : output.split("\n");
+		if (lines.length !== expected.length) throw new Error("Git object batch-check 输出数量不匹配");
+		for (let index = 0; index < expected.length; index += 1) {
+			const object = expected[index]!;
+			const match = lines[index]!.match(/^([0-9a-f]{40,64}) (blob|tree) ([0-9]+)$/);
+			if (
+				match === null ||
+				match[1] !== object.objectId ||
+				match[2] !== object.type ||
+				!Number.isSafeInteger(Number(match[3]))
+			) {
+				throw new Error(`Git object batch-check 校验失败：${object.objectId}`);
+			}
+		}
 	}
 
 	private async readBlobText(gitDirectory: string, objectId: string): Promise<string> {
@@ -980,6 +1149,10 @@ function workspaceRootPath(workspaceIdentity: string, rootPath: string): string
 	return safe === "." ? workspaceIdentity : join(workspaceIdentity, ...safe.split("/"));
 }
 
+function workspaceRelativePath(rootPath: string, relativePath: string): string {
+	return rootPath === "." ? relativePath : `${rootPath}/${relativePath}`;
+}
+
 function rootStoreId(root: RootTopologyIdentity): string {
 	return checksum(canonicalJson({
 		relativeRoot: root.relativeRoot,
@@ -1061,6 +1234,76 @@ function isolatedGitConfiguration(): Readonly<Record<string, string | undefined>
 	};
 }
 
+function indexInfoBatches(
+	leaves: readonly VisibleLeaf[],
+	objectIds: ReadonlyMap<string, string>,
+): Buffer[] {
+	const result: Buffer[] = [];
+	let records: string[] = [];
+	let bytes = 0;
+	for (const leaf of leaves) {
+		const objectId = objectIds.get(leaf.relativePath);
+		if (objectId === undefined) {
+			throw new SnapshotStoreError("capture_failed", `文件对象 materialize 结果缺失：${leaf.relativePath}`);
+		}
+		const record = `${leaf.mode.toString(8)} ${objectId}\t${leaf.relativePath}\0`;
+		const recordBytes = Buffer.byteLength(record, "utf8");
+		if (
+			records.length > 0 &&
+			(records.length >= INDEX_BATCH_MAX_ENTRIES || bytes + recordBytes > INDEX_BATCH_MAX_BYTES)
+		) {
+			result.push(Buffer.from(records.join(""), "utf8"));
+			records = [];
+			bytes = 0;
+		}
+		records.push(record);
+		bytes += recordBytes;
+	}
+	if (records.length > 0) result.push(Buffer.from(records.join(""), "utf8"));
+	return result;
+}
+
+function visibleLeafFingerprint(metadata: Stats): string {
+	return checksum(canonicalJson({
+		kind: metadata.isSymbolicLink() ? "symlink" : metadata.isFile() ? "file" : "other",
+		dev: metadata.dev,
+		ino: metadata.ino,
+		mode: metadata.mode,
+		size: metadata.size,
+		mtimeMs: metadata.mtimeMs,
+		ctimeMs: metadata.ctimeMs,
+	}));
+}
+
+function hashPathBatches(leaves: readonly VisibleLeaf[]): VisibleLeaf[][] {
+	const result: VisibleLeaf[][] = [];
+	let current: VisibleLeaf[] = [];
+	let argumentBytes = 0;
+	for (const leaf of leaves) {
+		const leafBytes = Buffer.byteLength(leaf.relativePath, "utf8") + 1;
+		if (
+			current.length > 0 &&
+			(current.length >= HASH_BATCH_MAX_PATHS || argumentBytes + leafBytes > HASH_BATCH_MAX_ARGUMENT_BYTES)
+		) {
+			result.push(current);
+			current = [];
+			argumentBytes = 0;
+		}
+		current.push(leaf);
+		argumentBytes += leafBytes;
+	}
+	if (current.length > 0) result.push(current);
+	return result;
+}
+
+function parseObjectIdLines(output: string, expectedCount: number): string[] {
+	const lines = output.endsWith("\n") ? output.slice(0, -1).split("\n") : output.split("\n");
+	if (lines.length !== expectedCount || lines.some((line) => !isObjectId(line))) {
+		throw new SnapshotStoreError("capture_failed", "git hash-object 批量输出无效");
+	}
+	return lines;
+}
+
 function parseTreeEntries(output: Uint8Array): CapturedTreeEntry[] {
 	const entries: CapturedTreeEntry[] = [];
 	for (const record of splitNulRecords(output)) {