@davideasden/pi-undo 0.2.9 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/native/bin/pi-undo-fs-darwin-arm64 +0 -0
- package/package.json +1 -1
- package/src/native-metadata.ts +231 -0
- package/src/native-restore.ts +1 -1
- package/src/path-safety.ts +24 -0
- package/src/snapshot-store.ts +204 -39
|
Binary file
|
package/package.json
CHANGED
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { constants } from "node:fs";
|
|
4
|
+
import { access, rm, writeFile } from "node:fs/promises";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
|
|
7
|
+
import { nativeExecutable } from "./native-restore.ts";
|
|
8
|
+
|
|
9
|
+
const NATIVE_INSPECT_TIMEOUT_MS = 30_000;
|
|
10
|
+
const NATIVE_INSPECT_OUTPUT_LIMIT = 32 * 1024 * 1024;
|
|
11
|
+
|
|
12
|
+
export interface NativeMetadataEntry {
|
|
13
|
+
readonly path: string;
|
|
14
|
+
readonly kind: "absent" | "file" | "symlink" | "other";
|
|
15
|
+
readonly dev?: bigint;
|
|
16
|
+
readonly ino?: bigint;
|
|
17
|
+
readonly mode?: bigint;
|
|
18
|
+
readonly size?: bigint;
|
|
19
|
+
readonly mtimeNs?: bigint;
|
|
20
|
+
readonly ctimeNs?: bigint;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface NativeMetadataPort {
|
|
24
|
+
inspect(
|
|
25
|
+
workspaceRoot: string,
|
|
26
|
+
paths: readonly string[],
|
|
27
|
+
requestDirectory: string,
|
|
28
|
+
): Promise<readonly NativeMetadataEntry[] | undefined>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** 能力探测不支持时回退 TypeScript;已确认支持后的 inspect 错误保持 fail-closed。 */
|
|
32
|
+
export class NativeMetadataInspector implements NativeMetadataPort {
|
|
33
|
+
private readonly executable: string | undefined;
|
|
34
|
+
private capability: Promise<boolean> | undefined;
|
|
35
|
+
|
|
36
|
+
constructor(executable = nativeExecutable()) {
|
|
37
|
+
this.executable = process.env.PI_UNDO_DISABLE_NATIVE === "1" ? undefined : executable;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async inspect(
|
|
41
|
+
workspaceRoot: string,
|
|
42
|
+
paths: readonly string[],
|
|
43
|
+
requestDirectory: string,
|
|
44
|
+
): Promise<readonly NativeMetadataEntry[] | undefined> {
|
|
45
|
+
if (paths.length === 0) return [];
|
|
46
|
+
if (!await this.supportsInspect(requestDirectory)) return undefined;
|
|
47
|
+
const executable = this.executable!;
|
|
48
|
+
const requestPath = join(requestDirectory, `native-inspect-${process.pid}-${randomUUID()}.json`);
|
|
49
|
+
try {
|
|
50
|
+
await writeFile(requestPath, JSON.stringify({
|
|
51
|
+
schemaVersion: 1,
|
|
52
|
+
workspaceRoot,
|
|
53
|
+
paths,
|
|
54
|
+
}), { mode: 0o600, flag: "wx" });
|
|
55
|
+
return await runNativeInspect(executable, requestPath, paths);
|
|
56
|
+
} finally {
|
|
57
|
+
await rm(requestPath, { force: true }).catch(() => {});
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
private supportsInspect(requestDirectory: string): Promise<boolean> {
|
|
62
|
+
if (this.capability !== undefined) return this.capability;
|
|
63
|
+
this.capability = (async () => {
|
|
64
|
+
if (this.executable === undefined) return false;
|
|
65
|
+
try {
|
|
66
|
+
await access(this.executable, constants.X_OK);
|
|
67
|
+
return await probeNativeInspect(this.executable, requestDirectory);
|
|
68
|
+
} catch {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
})();
|
|
72
|
+
return this.capability;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function probeNativeInspect(executable: string, isolatedDirectory: string): Promise<boolean> {
|
|
77
|
+
return new Promise((resolve) => {
|
|
78
|
+
const child = spawn(executable, ["--capabilities"], {
|
|
79
|
+
cwd: isolatedDirectory,
|
|
80
|
+
shell: false,
|
|
81
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
82
|
+
windowsHide: true,
|
|
83
|
+
});
|
|
84
|
+
const stdout: Buffer[] = [];
|
|
85
|
+
let bytes = 0;
|
|
86
|
+
let settled = false;
|
|
87
|
+
const timeout = setTimeout(() => child.kill("SIGKILL"), 5_000);
|
|
88
|
+
child.stdout?.on("data", (chunk: Buffer | string) => {
|
|
89
|
+
const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
90
|
+
bytes += value.length;
|
|
91
|
+
if (bytes <= 64 * 1024) stdout.push(value);
|
|
92
|
+
else child.kill("SIGKILL");
|
|
93
|
+
});
|
|
94
|
+
child.once("error", () => {
|
|
95
|
+
if (settled) return;
|
|
96
|
+
settled = true;
|
|
97
|
+
clearTimeout(timeout);
|
|
98
|
+
resolve(false);
|
|
99
|
+
});
|
|
100
|
+
child.once("close", (code) => {
|
|
101
|
+
if (settled) return;
|
|
102
|
+
settled = true;
|
|
103
|
+
clearTimeout(timeout);
|
|
104
|
+
if (code !== 0 || bytes > 64 * 1024) return resolve(false);
|
|
105
|
+
try {
|
|
106
|
+
const value: unknown = JSON.parse(Buffer.concat(stdout).toString("utf8"));
|
|
107
|
+
resolve(isRecord(value) && value.ok === true && Array.isArray(value.capabilities) &&
|
|
108
|
+
value.capabilities.includes("inspect-v1"));
|
|
109
|
+
} catch {
|
|
110
|
+
resolve(false);
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function runNativeInspect(
|
|
117
|
+
executable: string,
|
|
118
|
+
requestPath: string,
|
|
119
|
+
expectedPaths: readonly string[],
|
|
120
|
+
): Promise<readonly NativeMetadataEntry[]> {
|
|
121
|
+
return new Promise((resolve, reject) => {
|
|
122
|
+
const child = spawn(executable, ["--inspect", requestPath], {
|
|
123
|
+
shell: false,
|
|
124
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
125
|
+
windowsHide: true,
|
|
126
|
+
});
|
|
127
|
+
const stdout: Buffer[] = [];
|
|
128
|
+
const stderr: Buffer[] = [];
|
|
129
|
+
let outputBytes = 0;
|
|
130
|
+
let settled = false;
|
|
131
|
+
let overflow = false;
|
|
132
|
+
const timeout = setTimeout(() => child.kill("SIGKILL"), NATIVE_INSPECT_TIMEOUT_MS);
|
|
133
|
+
const capture = (target: Buffer[]) => (chunk: Buffer | string): void => {
|
|
134
|
+
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
135
|
+
if (outputBytes + bytes.length > NATIVE_INSPECT_OUTPUT_LIMIT) {
|
|
136
|
+
overflow = true;
|
|
137
|
+
child.kill("SIGKILL");
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
target.push(bytes);
|
|
141
|
+
outputBytes += bytes.length;
|
|
142
|
+
};
|
|
143
|
+
child.stdout?.on("data", capture(stdout));
|
|
144
|
+
child.stderr?.on("data", capture(stderr));
|
|
145
|
+
child.once("error", (error) => {
|
|
146
|
+
if (settled) return;
|
|
147
|
+
settled = true;
|
|
148
|
+
clearTimeout(timeout);
|
|
149
|
+
reject(error);
|
|
150
|
+
});
|
|
151
|
+
child.once("close", (code) => {
|
|
152
|
+
if (settled) return;
|
|
153
|
+
settled = true;
|
|
154
|
+
clearTimeout(timeout);
|
|
155
|
+
if (overflow) {
|
|
156
|
+
reject(new Error("native metadata inspect 输出超过限制"));
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
if (code !== 0) {
|
|
160
|
+
reject(new Error(`native metadata inspect 失败:${Buffer.concat(stderr).toString("utf8").trim()}`));
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
try {
|
|
164
|
+
resolve(parseInspectResponse(Buffer.concat(stdout).toString("utf8"), expectedPaths));
|
|
165
|
+
} catch (error) {
|
|
166
|
+
reject(error);
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function parseInspectResponse(text: string, expectedPaths: readonly string[]): readonly NativeMetadataEntry[] {
|
|
173
|
+
const value: unknown = JSON.parse(text);
|
|
174
|
+
if (!isRecord(value) || value.ok !== true || value.processed !== expectedPaths.length || !Array.isArray(value.entries)) {
|
|
175
|
+
throw new Error("native metadata inspect 响应无效");
|
|
176
|
+
}
|
|
177
|
+
if (value.entries.length !== expectedPaths.length) throw new Error("native metadata inspect 条目数量不匹配");
|
|
178
|
+
return value.entries.map((candidate, index) => {
|
|
179
|
+
if (!isRecord(candidate) || candidate.path !== expectedPaths[index] ||
|
|
180
|
+
!isMetadataKind(candidate.kind)) {
|
|
181
|
+
throw new Error("native metadata inspect 条目无效");
|
|
182
|
+
}
|
|
183
|
+
if (candidate.kind === "absent") {
|
|
184
|
+
if ([candidate.dev, candidate.ino, candidate.mode, candidate.size, candidate.mtimeNs, candidate.ctimeNs]
|
|
185
|
+
.some((field) => field !== null && field !== undefined)) {
|
|
186
|
+
throw new Error("native metadata absent 条目包含 metadata");
|
|
187
|
+
}
|
|
188
|
+
return { path: candidate.path as string, kind: "absent" as const };
|
|
189
|
+
}
|
|
190
|
+
return {
|
|
191
|
+
path: candidate.path as string,
|
|
192
|
+
kind: candidate.kind,
|
|
193
|
+
dev: parseUnsigned(candidate.dev, 64),
|
|
194
|
+
ino: parseUnsigned(candidate.ino, 64),
|
|
195
|
+
mode: parseUnsigned(candidate.mode, 32),
|
|
196
|
+
size: parseUnsigned(candidate.size, 64),
|
|
197
|
+
mtimeNs: parseTimestamp(candidate.mtimeNs),
|
|
198
|
+
ctimeNs: parseTimestamp(candidate.ctimeNs),
|
|
199
|
+
};
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function parseUnsigned(value: unknown, bits: 32 | 64): bigint {
|
|
204
|
+
const maxDigits = bits === 32 ? 10 : 20;
|
|
205
|
+
if (typeof value !== "string" || value.length > maxDigits || !/^(?:0|[1-9][0-9]*)$/.test(value)) {
|
|
206
|
+
throw new Error("native metadata unsigned 字段无效");
|
|
207
|
+
}
|
|
208
|
+
const parsed = BigInt(value);
|
|
209
|
+
if (parsed > (1n << BigInt(bits)) - 1n) throw new Error("native metadata unsigned 字段越界");
|
|
210
|
+
return parsed;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function parseTimestamp(value: unknown): bigint {
|
|
214
|
+
if (typeof value !== "string" || value.length > 30 || !/^-?(?:0|[1-9][0-9]*)$/.test(value)) {
|
|
215
|
+
throw new Error("native metadata timestamp 字段无效");
|
|
216
|
+
}
|
|
217
|
+
const parsed = BigInt(value);
|
|
218
|
+
const billion = 1_000_000_000n;
|
|
219
|
+
const minimum = -(1n << 63n) * billion;
|
|
220
|
+
const maximum = ((1n << 63n) - 1n) * billion + (billion - 1n);
|
|
221
|
+
if (parsed < minimum || parsed > maximum) throw new Error("native metadata timestamp 字段越界");
|
|
222
|
+
return parsed;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function isMetadataKind(value: unknown): value is NativeMetadataEntry["kind"] {
|
|
226
|
+
return value === "absent" || value === "file" || value === "symlink" || value === "other";
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
230
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
231
|
+
}
|
package/src/native-restore.ts
CHANGED
|
@@ -79,7 +79,7 @@ export async function createNativeFileBatch(options: {
|
|
|
79
79
|
};
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
-
function nativeExecutable(): string | undefined {
|
|
82
|
+
export function nativeExecutable(): string | undefined {
|
|
83
83
|
const platform = process.platform === "darwin"
|
|
84
84
|
? "darwin"
|
|
85
85
|
: process.platform === "linux" ? "linux"
|
package/src/path-safety.ts
CHANGED
|
@@ -64,6 +64,30 @@ export async function assertNoSymlinkEscape(root: string, relativePath: string):
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
/** 批量核验叶子路径的全部父目录;共享目录只执行一次 lstat。 */
|
|
68
|
+
export async function assertNoSymlinkParents(root: string, relativePaths: readonly string[]): Promise<void> {
|
|
69
|
+
const directories = new Set<string>();
|
|
70
|
+
for (const relativePath of relativePaths) {
|
|
71
|
+
const safePath = relativeSafePath(root, relativePath);
|
|
72
|
+
if (safePath === ".") continue;
|
|
73
|
+
const parts = safePath.split("/");
|
|
74
|
+
for (let index = 1; index < parts.length; index += 1) {
|
|
75
|
+
directories.add(parts.slice(0, index).join("/"));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const ordered = [...directories].sort((left, right) => pathDepth(left) - pathDepth(right) || left.localeCompare(right));
|
|
79
|
+
for (const directory of ordered) {
|
|
80
|
+
try {
|
|
81
|
+
const metadata = await lstat(join(resolve(root), ...directory.split("/")));
|
|
82
|
+
if (metadata.isSymbolicLink()) fail("symlink_escape", "中间路径组件不能是 symlink");
|
|
83
|
+
if (!metadata.isDirectory()) fail("unsafe_path", "中间路径组件不是目录");
|
|
84
|
+
} catch (error) {
|
|
85
|
+
if (hasErrorCode(error, "ENOENT")) continue;
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
67
91
|
export function pathSetsOverlap(leftPaths: readonly string[], rightPaths: readonly string[]): boolean {
|
|
68
92
|
for (const path of leftPaths) assertRelativeCandidate(path);
|
|
69
93
|
for (const path of rightPaths) assertRelativeCandidate(path);
|
package/src/snapshot-store.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { BigIntStats } from "node:fs";
|
|
2
2
|
import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, realpath, rm, stat } from "node:fs/promises";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
@@ -20,7 +20,9 @@ import type {
|
|
|
20
20
|
SnapshotManifest,
|
|
21
21
|
SnapshotRoot,
|
|
22
22
|
} from "./model.ts";
|
|
23
|
-
import {
|
|
23
|
+
import type { NativeMetadataEntry, NativeMetadataPort } from "./native-metadata.ts";
|
|
24
|
+
import { NativeMetadataInspector } from "./native-metadata.ts";
|
|
25
|
+
import { assertNoSymlinkEscape, assertNoSymlinkParents, pathSetsOverlap, relativeSafePath } from "./path-safety.ts";
|
|
24
26
|
import { RootDiscovery, type RootTopology } from "./root-discovery.ts";
|
|
25
27
|
import { WorkspaceLock } from "./workspace-lock.ts";
|
|
26
28
|
|
|
@@ -42,6 +44,7 @@ const INDEX_BATCH_MAX_BYTES = 8 * 1024 * 1024;
|
|
|
42
44
|
const BLOB_CACHE_MAX_BYTES = 128 * 1024 * 1024;
|
|
43
45
|
const BLOB_BATCH_MAX_BYTES = 16 * 1024 * 1024;
|
|
44
46
|
const BLOB_BATCH_MAX_ENTRIES = process.platform === "win32" ? 256 : 2_048;
|
|
47
|
+
const RACY_CLEAN_WINDOW_NS = 2_000_000_000n;
|
|
45
48
|
|
|
46
49
|
interface PinRecord {
|
|
47
50
|
readonly schemaVersion: 1;
|
|
@@ -70,6 +73,17 @@ interface CapturedRootResult {
|
|
|
70
73
|
readonly ignoredPresentPaths: readonly string[];
|
|
71
74
|
readonly ignoreClosure: string;
|
|
72
75
|
readonly objectClosure: string;
|
|
76
|
+
readonly cacheUpdate: VisibleLeafCacheUpdate;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
interface VisibleLeafMetadata {
|
|
80
|
+
readonly kind: "file" | "symlink" | "other";
|
|
81
|
+
readonly dev: bigint;
|
|
82
|
+
readonly ino: bigint;
|
|
83
|
+
readonly mode: bigint;
|
|
84
|
+
readonly size: bigint;
|
|
85
|
+
readonly mtimeNs: bigint;
|
|
86
|
+
readonly ctimeNs: bigint;
|
|
73
87
|
}
|
|
74
88
|
|
|
75
89
|
interface VisibleLeaf {
|
|
@@ -77,6 +91,25 @@ interface VisibleLeaf {
|
|
|
77
91
|
readonly kind: "file" | "symlink";
|
|
78
92
|
readonly mode: number;
|
|
79
93
|
readonly fingerprint: string;
|
|
94
|
+
readonly changedAtNs: bigint;
|
|
95
|
+
readonly cacheable: boolean;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
interface CachedVisibleLeaf extends VisibleLeaf {
|
|
99
|
+
readonly objectId: string;
|
|
100
|
+
readonly verifiedAtNs: bigint;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
interface StagedWorktree {
|
|
104
|
+
readonly leaves: readonly VisibleLeaf[];
|
|
105
|
+
readonly objectIds: ReadonlyMap<string, string>;
|
|
106
|
+
readonly verifiedAtNs: bigint;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
interface VisibleLeafCacheUpdate {
|
|
110
|
+
readonly gitDirectory: string;
|
|
111
|
+
readonly staged: StagedWorktree;
|
|
112
|
+
readonly inclusions: readonly string[] | null;
|
|
80
113
|
}
|
|
81
114
|
|
|
82
115
|
interface CachedBlob {
|
|
@@ -90,6 +123,7 @@ export interface SnapshotStoreOptions {
|
|
|
90
123
|
readonly discovery?: RootDiscovery;
|
|
91
124
|
readonly lock?: WorkspaceLock;
|
|
92
125
|
readonly clock?: () => number;
|
|
126
|
+
readonly nativeMetadata?: NativeMetadataPort;
|
|
93
127
|
}
|
|
94
128
|
|
|
95
129
|
export interface CaptureOptions {
|
|
@@ -141,11 +175,13 @@ export class SnapshotStore {
|
|
|
141
175
|
private readonly discovery: RootDiscovery;
|
|
142
176
|
private readonly lock: WorkspaceLock;
|
|
143
177
|
private readonly clock: () => number;
|
|
178
|
+
private readonly nativeMetadata: NativeMetadataPort;
|
|
144
179
|
private readonly manifestLocations = new Map<string, string>();
|
|
145
180
|
// Tree 与 blob 都由 object ID 内容寻址;缓存只复用已从私有 ODB 读取的不可变内容。
|
|
146
181
|
private readonly treeEntriesCache = new Map<string, Promise<CapturedTreeEntry[]>>();
|
|
147
182
|
private readonly treeBlobMembership = new Map<string, string>();
|
|
148
183
|
private readonly blobCache = new Map<string, CachedBlob>();
|
|
184
|
+
private readonly visibleLeafCache = new Map<string, Map<string, CachedVisibleLeaf>>();
|
|
149
185
|
private blobCacheBytes = 0;
|
|
150
186
|
|
|
151
187
|
constructor(options: SnapshotStoreOptions = {}) {
|
|
@@ -155,6 +191,7 @@ export class SnapshotStore {
|
|
|
155
191
|
this.discovery = options.discovery ?? new RootDiscovery(this.git);
|
|
156
192
|
this.lock = options.lock ?? new WorkspaceLock();
|
|
157
193
|
this.clock = options.clock ?? Date.now;
|
|
194
|
+
this.nativeMetadata = options.nativeMetadata ?? new NativeMetadataInspector();
|
|
158
195
|
}
|
|
159
196
|
|
|
160
197
|
static supportsValidatedBlobBatch(store: SnapshotStore): boolean {
|
|
@@ -223,6 +260,7 @@ export class SnapshotStore {
|
|
|
223
260
|
transactionDirectory = await mkdtemp(join(transactionsRoot, "capture-"));
|
|
224
261
|
|
|
225
262
|
const roots: SnapshotRoot[] = [];
|
|
263
|
+
const cacheUpdates: VisibleLeafCacheUpdate[] = [];
|
|
226
264
|
for (const root of topology.roots) {
|
|
227
265
|
if (root.state !== "active") {
|
|
228
266
|
const coverage = rootCaptureCoverage(root.relativeRoot, scope);
|
|
@@ -242,6 +280,7 @@ export class SnapshotStore {
|
|
|
242
280
|
artifactExclusions,
|
|
243
281
|
);
|
|
244
282
|
roots.push(snapshotRoot(root, captured));
|
|
283
|
+
cacheUpdates.push(captured.cacheUpdate);
|
|
245
284
|
}
|
|
246
285
|
|
|
247
286
|
await this.assertTopology(topology, "捕获期间 topology 已变化");
|
|
@@ -261,6 +300,7 @@ export class SnapshotStore {
|
|
|
261
300
|
await this.touchStore(storeDirectory);
|
|
262
301
|
await writeContentAddressed(manifestPath, Buffer.from(canonicalJson(manifest), "utf8"));
|
|
263
302
|
this.manifestLocations.set(manifestId, manifestPath);
|
|
303
|
+
for (const update of cacheUpdates) this.rememberVisibleLeaves(update);
|
|
264
304
|
return manifest;
|
|
265
305
|
} catch (error) {
|
|
266
306
|
if (error instanceof SnapshotStoreError) {
|
|
@@ -624,6 +664,9 @@ export class SnapshotStore {
|
|
|
624
664
|
this.manifestLocations.delete(id);
|
|
625
665
|
}
|
|
626
666
|
}
|
|
667
|
+
for (const gitDirectory of this.visibleLeafCache.keys()) {
|
|
668
|
+
if (gitDirectory.startsWith(`${storeDirectory}${sep}`)) this.visibleLeafCache.delete(gitDirectory);
|
|
669
|
+
}
|
|
627
670
|
} catch (error) {
|
|
628
671
|
await writeJsonAtomic(join(storeDirectory, GC_METADATA_FILE), {
|
|
629
672
|
schemaVersion: SCHEMA_VERSION,
|
|
@@ -659,13 +702,15 @@ export class SnapshotStore {
|
|
|
659
702
|
.map((candidate) => rootRelativePath(root.relativeRoot, candidate.relativeRoot));
|
|
660
703
|
const exactExclusions = ownedArtifactExclusions(topology.roots, root.relativeRoot, artifactExclusions);
|
|
661
704
|
const inclusions = ownedRootInclusions(requestedInclusions, exclusions);
|
|
662
|
-
await this.stageWorktree(
|
|
705
|
+
const staged = await this.stageWorktree(
|
|
663
706
|
absoluteRoot,
|
|
664
707
|
environment,
|
|
665
708
|
root.gitBacked,
|
|
666
709
|
inclusions,
|
|
667
710
|
exclusions,
|
|
668
711
|
exactExclusions,
|
|
712
|
+
this.visibleLeafCache.get(gitDirectory),
|
|
713
|
+
transactionDirectory,
|
|
669
714
|
);
|
|
670
715
|
const coverage = rootCoverageFromInclusions(inclusions);
|
|
671
716
|
const ignoredPresentPaths = await this.captureIgnoredPresentPaths(
|
|
@@ -687,6 +732,7 @@ export class SnapshotStore {
|
|
|
687
732
|
coverage,
|
|
688
733
|
...ignoredPresentProof(coverage, ignoredPresentPaths),
|
|
689
734
|
objectClosure: treeObjectClosure(treeId, entries),
|
|
735
|
+
cacheUpdate: { gitDirectory, staged, inclusions },
|
|
690
736
|
};
|
|
691
737
|
}
|
|
692
738
|
|
|
@@ -749,7 +795,9 @@ export class SnapshotStore {
|
|
|
749
795
|
inclusions: readonly string[] | null,
|
|
750
796
|
exclusions: readonly string[],
|
|
751
797
|
exactExclusions: ReadonlySet<string>,
|
|
752
|
-
|
|
798
|
+
cache: ReadonlyMap<string, CachedVisibleLeaf> | undefined,
|
|
799
|
+
requestDirectory: string,
|
|
800
|
+
): Promise<StagedWorktree> {
|
|
753
801
|
const leaves = await this.collectVisibleLeaves(
|
|
754
802
|
cwd,
|
|
755
803
|
environment,
|
|
@@ -757,12 +805,25 @@ export class SnapshotStore {
|
|
|
757
805
|
inclusions,
|
|
758
806
|
exclusions,
|
|
759
807
|
exactExclusions,
|
|
808
|
+
requestDirectory,
|
|
760
809
|
);
|
|
761
810
|
const objectIds = new Map<string, string>();
|
|
762
|
-
const
|
|
811
|
+
const uncached: VisibleLeaf[] = [];
|
|
812
|
+
for (const leaf of leaves) {
|
|
813
|
+
const cached = cache?.get(leaf.relativePath);
|
|
814
|
+
if (
|
|
815
|
+
leaf.cacheable && cached?.cacheable === true && cached.kind === leaf.kind &&
|
|
816
|
+
cached.mode === leaf.mode && cached.fingerprint === leaf.fingerprint &&
|
|
817
|
+
cached.verifiedAtNs > cached.changedAtNs + RACY_CLEAN_WINDOW_NS
|
|
818
|
+
) {
|
|
819
|
+
objectIds.set(leaf.relativePath, cached.objectId);
|
|
820
|
+
} else {
|
|
821
|
+
uncached.push(leaf);
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
const hashBatches = hashPathBatches(uncached.filter((leaf) => leaf.kind === "file"));
|
|
763
825
|
const hashedBatches = await mapConcurrentOrdered(hashBatches, HASH_BATCH_CONCURRENCY, async (batch) => {
|
|
764
|
-
await
|
|
765
|
-
this.assertVisibleLeafUnchanged(cwd, leaf));
|
|
826
|
+
await this.assertVisibleLeavesUnchanged(cwd, batch);
|
|
766
827
|
const output = await this.runGit([
|
|
767
828
|
"hash-object",
|
|
768
829
|
"-w",
|
|
@@ -771,15 +832,15 @@ export class SnapshotStore {
|
|
|
771
832
|
...batch.map((leaf) => leaf.relativePath),
|
|
772
833
|
], { cwd, env: environment });
|
|
773
834
|
const hashes = parseObjectIdLines(output, batch.length);
|
|
774
|
-
await
|
|
775
|
-
this.assertVisibleLeafUnchanged(cwd, leaf));
|
|
835
|
+
await this.assertVisibleLeavesUnchanged(cwd, batch);
|
|
776
836
|
return batch.map((leaf, index) => [leaf.relativePath, hashes[index]!] as const);
|
|
777
837
|
});
|
|
778
838
|
for (const batch of hashedBatches) {
|
|
779
839
|
for (const [relativePath, objectId] of batch) objectIds.set(relativePath, objectId);
|
|
780
840
|
}
|
|
781
|
-
for (const leaf of
|
|
841
|
+
for (const leaf of uncached) {
|
|
782
842
|
if (leaf.kind !== "symlink") continue;
|
|
843
|
+
await assertNoSymlinkEscape(cwd, leaf.relativePath);
|
|
783
844
|
await this.assertVisibleLeafUnchanged(cwd, leaf);
|
|
784
845
|
const linkText = await readlink(join(cwd, ...leaf.relativePath.split("/")), { encoding: "buffer" });
|
|
785
846
|
decodeUtf8(linkText, "symlink target 不是可无损表示的 UTF-8");
|
|
@@ -791,6 +852,7 @@ export class SnapshotStore {
|
|
|
791
852
|
if (!isObjectId(objectId)) {
|
|
792
853
|
throw new SnapshotStoreError("capture_failed", `文件对象 materialize 失败:${leaf.relativePath}`);
|
|
793
854
|
}
|
|
855
|
+
await assertNoSymlinkEscape(cwd, leaf.relativePath);
|
|
794
856
|
await this.assertVisibleLeafUnchanged(cwd, leaf);
|
|
795
857
|
objectIds.set(leaf.relativePath, objectId);
|
|
796
858
|
}
|
|
@@ -801,15 +863,67 @@ export class SnapshotStore {
|
|
|
801
863
|
stdin: indexInput,
|
|
802
864
|
});
|
|
803
865
|
}
|
|
866
|
+
await this.assertVisibleLeavesUnchanged(cwd, leaves, requestDirectory);
|
|
867
|
+
return { leaves, objectIds, verifiedAtNs: BigInt(Date.now()) * 1_000_000n };
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
private rememberVisibleLeaves(update: VisibleLeafCacheUpdate): void {
|
|
871
|
+
const { gitDirectory, staged, inclusions } = update;
|
|
872
|
+
const cache = inclusions !== null && inclusions.length === 0
|
|
873
|
+
? new Map<string, CachedVisibleLeaf>()
|
|
874
|
+
: new Map(this.visibleLeafCache.get(gitDirectory));
|
|
875
|
+
if (inclusions !== null && inclusions.length > 0) {
|
|
876
|
+
for (const relativePath of cache.keys()) {
|
|
877
|
+
if (inclusions.some((inclusion) => isPathAtOrBelow(inclusion, relativePath))) {
|
|
878
|
+
cache.delete(relativePath);
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
for (const leaf of staged.leaves) {
|
|
883
|
+
const objectId = staged.objectIds.get(leaf.relativePath);
|
|
884
|
+
if (objectId === undefined) continue;
|
|
885
|
+
if (!leaf.cacheable) {
|
|
886
|
+
cache.delete(leaf.relativePath);
|
|
887
|
+
continue;
|
|
888
|
+
}
|
|
889
|
+
cache.set(leaf.relativePath, { ...leaf, objectId, verifiedAtNs: staged.verifiedAtNs });
|
|
890
|
+
}
|
|
891
|
+
this.visibleLeafCache.set(gitDirectory, cache);
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
private async assertVisibleLeavesUnchanged(
|
|
895
|
+
cwd: string,
|
|
896
|
+
leaves: readonly VisibleLeaf[],
|
|
897
|
+
requestDirectory?: string,
|
|
898
|
+
): Promise<void> {
|
|
899
|
+
if (requestDirectory !== undefined) {
|
|
900
|
+
const inspected = await this.nativeMetadata.inspect(
|
|
901
|
+
cwd,
|
|
902
|
+
leaves.map((leaf) => leaf.relativePath),
|
|
903
|
+
requestDirectory,
|
|
904
|
+
);
|
|
905
|
+
if (inspected !== undefined) {
|
|
906
|
+
for (let index = 0; index < leaves.length; index += 1) {
|
|
907
|
+
const leaf = leaves[index]!;
|
|
908
|
+
const metadata = nativeVisibleLeafMetadata(inspected[index]!);
|
|
909
|
+
if (metadata === null || visibleLeafFingerprint(metadata) !== leaf.fingerprint) {
|
|
910
|
+
throw new SnapshotStoreError("capture_failed", `捕获期间工作区叶子已变化:${leaf.relativePath}`);
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
await assertNoSymlinkParents(cwd, leaves.map((leaf) => leaf.relativePath));
|
|
917
|
+
await mapConcurrentOrdered(leaves, FILE_SYSTEM_INSPECTION_CONCURRENCY, (leaf) =>
|
|
918
|
+
this.assertVisibleLeafUnchanged(cwd, leaf));
|
|
804
919
|
}
|
|
805
920
|
|
|
806
921
|
private async assertVisibleLeafUnchanged(cwd: string, leaf: VisibleLeaf): Promise<void> {
|
|
807
|
-
await
|
|
808
|
-
const metadata = await lstat(join(cwd, ...leaf.relativePath.split("/"))).catch((error) => {
|
|
922
|
+
const metadata = await lstat(join(cwd, ...leaf.relativePath.split("/")), { bigint: true }).catch((error) => {
|
|
809
923
|
if (hasErrorCode(error, "ENOENT")) return null;
|
|
810
924
|
throw error;
|
|
811
925
|
});
|
|
812
|
-
if (metadata === null || visibleLeafFingerprint(metadata) !== leaf.fingerprint) {
|
|
926
|
+
if (metadata === null || visibleLeafFingerprint(visibleLeafMetadataFromStats(metadata)) !== leaf.fingerprint) {
|
|
813
927
|
throw new SnapshotStoreError("capture_failed", `捕获期间工作区叶子已变化:${leaf.relativePath}`);
|
|
814
928
|
}
|
|
815
929
|
}
|
|
@@ -864,6 +978,7 @@ export class SnapshotStore {
|
|
|
864
978
|
inclusions: readonly string[] | null,
|
|
865
979
|
exclusions: readonly string[],
|
|
866
980
|
exactExclusions: ReadonlySet<string>,
|
|
981
|
+
requestDirectory: string,
|
|
867
982
|
): Promise<VisibleLeaf[]> {
|
|
868
983
|
const paths = await this.queryVisibleLeafPaths(
|
|
869
984
|
cwd,
|
|
@@ -873,33 +988,46 @@ export class SnapshotStore {
|
|
|
873
988
|
exclusions,
|
|
874
989
|
exactExclusions,
|
|
875
990
|
);
|
|
876
|
-
const
|
|
991
|
+
const nativeEntries = await this.nativeMetadata.inspect(cwd, paths, requestDirectory);
|
|
992
|
+
const metadataEntries = nativeEntries === undefined
|
|
993
|
+
? await this.collectVisibleLeafMetadataFallback(cwd, paths)
|
|
994
|
+
: nativeEntries.map((entry) => nativeVisibleLeafMetadata(entry));
|
|
995
|
+
const leaves: VisibleLeaf[] = [];
|
|
996
|
+
for (let index = 0; index < paths.length; index += 1) {
|
|
997
|
+
const relativePath = paths[index]!;
|
|
998
|
+
const metadata = metadataEntries[index]!;
|
|
999
|
+
if (metadata === null) continue;
|
|
1000
|
+
if (metadata.kind === "other") {
|
|
1001
|
+
throw new SnapshotStoreError("capture_failed", `不支持的工作区文件类型:${relativePath}`);
|
|
1002
|
+
}
|
|
1003
|
+
const cacheable = visibleLeafMetadataCacheable(metadata);
|
|
1004
|
+
leaves.push({
|
|
1005
|
+
relativePath,
|
|
1006
|
+
kind: metadata.kind,
|
|
1007
|
+
mode: metadata.kind === "symlink"
|
|
1008
|
+
? 0o120000
|
|
1009
|
+
: (metadata.mode & 0o111n) === 0n ? 0o100644 : 0o100755,
|
|
1010
|
+
fingerprint: visibleLeafFingerprint(metadata),
|
|
1011
|
+
changedAtNs: metadata.mtimeNs > metadata.ctimeNs ? metadata.mtimeNs : metadata.ctimeNs,
|
|
1012
|
+
cacheable,
|
|
1013
|
+
});
|
|
1014
|
+
}
|
|
1015
|
+
return leaves;
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
private async collectVisibleLeafMetadataFallback(
|
|
1019
|
+
cwd: string,
|
|
1020
|
+
paths: readonly string[],
|
|
1021
|
+
): Promise<readonly (VisibleLeafMetadata | null)[]> {
|
|
1022
|
+
await assertNoSymlinkParents(cwd, paths);
|
|
1023
|
+
return mapConcurrentOrdered(paths, FILE_SYSTEM_INSPECTION_CONCURRENCY, async (relativePath) => {
|
|
877
1024
|
relativeSafePath(cwd, relativePath);
|
|
878
|
-
await
|
|
879
|
-
const metadata = await lstat(join(cwd, ...relativePath.split("/"))).catch((error) => {
|
|
1025
|
+
const metadata = await lstat(join(cwd, ...relativePath.split("/")), { bigint: true }).catch((error) => {
|
|
880
1026
|
if (hasErrorCode(error, "ENOENT")) return null;
|
|
881
1027
|
throw error;
|
|
882
1028
|
});
|
|
883
|
-
|
|
884
|
-
if (metadata.isSymbolicLink()) {
|
|
885
|
-
return {
|
|
886
|
-
relativePath,
|
|
887
|
-
kind: "symlink" as const,
|
|
888
|
-
mode: 0o120000,
|
|
889
|
-
fingerprint: visibleLeafFingerprint(metadata),
|
|
890
|
-
};
|
|
891
|
-
}
|
|
892
|
-
if (metadata.isFile()) {
|
|
893
|
-
return {
|
|
894
|
-
relativePath,
|
|
895
|
-
kind: "file" as const,
|
|
896
|
-
mode: (metadata.mode & 0o111) === 0 ? 0o100644 : 0o100755,
|
|
897
|
-
fingerprint: visibleLeafFingerprint(metadata),
|
|
898
|
-
};
|
|
899
|
-
}
|
|
900
|
-
throw new SnapshotStoreError("capture_failed", `不支持的工作区文件类型:${relativePath}`);
|
|
1029
|
+
return metadata === null ? null : visibleLeafMetadataFromStats(metadata);
|
|
901
1030
|
});
|
|
902
|
-
return leaves.filter((leaf): leaf is VisibleLeaf => leaf !== null);
|
|
903
1031
|
}
|
|
904
1032
|
|
|
905
1033
|
private async validateIgnoreQuery(
|
|
@@ -937,6 +1065,7 @@ export class SnapshotStore {
|
|
|
937
1065
|
}
|
|
938
1066
|
await mkdir(dirname(gitDirectory), { recursive: true });
|
|
939
1067
|
await this.runGit(["init", "--bare", "--quiet", gitDirectory], { env: cleanGitEnvironment() });
|
|
1068
|
+
this.visibleLeafCache.delete(gitDirectory);
|
|
940
1069
|
await this.configurePrivateRepository(gitDirectory);
|
|
941
1070
|
}
|
|
942
1071
|
|
|
@@ -1559,18 +1688,54 @@ function indexInfoBatches(
|
|
|
1559
1688
|
return result;
|
|
1560
1689
|
}
|
|
1561
1690
|
|
|
1562
|
-
function
|
|
1563
|
-
return
|
|
1691
|
+
function visibleLeafMetadataFromStats(metadata: BigIntStats): VisibleLeafMetadata {
|
|
1692
|
+
return {
|
|
1564
1693
|
kind: metadata.isSymbolicLink() ? "symlink" : metadata.isFile() ? "file" : "other",
|
|
1565
1694
|
dev: metadata.dev,
|
|
1566
1695
|
ino: metadata.ino,
|
|
1567
1696
|
mode: metadata.mode,
|
|
1568
1697
|
size: metadata.size,
|
|
1569
|
-
|
|
1570
|
-
|
|
1698
|
+
mtimeNs: metadata.mtimeNs,
|
|
1699
|
+
ctimeNs: metadata.ctimeNs,
|
|
1700
|
+
};
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
function nativeVisibleLeafMetadata(entry: NativeMetadataEntry): VisibleLeafMetadata | null {
|
|
1704
|
+
if (entry.kind === "absent") return null;
|
|
1705
|
+
if (
|
|
1706
|
+
entry.dev === undefined || entry.ino === undefined || entry.mode === undefined ||
|
|
1707
|
+
entry.size === undefined || entry.mtimeNs === undefined || entry.ctimeNs === undefined
|
|
1708
|
+
) {
|
|
1709
|
+
throw new SnapshotStoreError("capture_failed", `native metadata 缺少字段:${entry.path}`);
|
|
1710
|
+
}
|
|
1711
|
+
return {
|
|
1712
|
+
kind: entry.kind,
|
|
1713
|
+
dev: entry.dev,
|
|
1714
|
+
ino: entry.ino,
|
|
1715
|
+
mode: entry.mode,
|
|
1716
|
+
size: entry.size,
|
|
1717
|
+
mtimeNs: entry.mtimeNs,
|
|
1718
|
+
ctimeNs: entry.ctimeNs,
|
|
1719
|
+
};
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
function visibleLeafFingerprint(metadata: VisibleLeafMetadata): string {
|
|
1723
|
+
return checksum(canonicalJson({
|
|
1724
|
+
kind: metadata.kind,
|
|
1725
|
+
dev: metadata.dev.toString(),
|
|
1726
|
+
ino: metadata.ino.toString(),
|
|
1727
|
+
mode: metadata.mode.toString(),
|
|
1728
|
+
size: metadata.size.toString(),
|
|
1729
|
+
mtimeNs: metadata.mtimeNs.toString(),
|
|
1730
|
+
ctimeNs: metadata.ctimeNs.toString(),
|
|
1571
1731
|
}));
|
|
1572
1732
|
}
|
|
1573
1733
|
|
|
1734
|
+
function visibleLeafMetadataCacheable(metadata: VisibleLeafMetadata): boolean {
|
|
1735
|
+
// dev/ino/ctime 缺失时无法证明路径仍指向同一未修改对象,必须回退内容 hash。
|
|
1736
|
+
return metadata.dev !== 0n && metadata.ino !== 0n && metadata.ctimeNs > 0n;
|
|
1737
|
+
}
|
|
1738
|
+
|
|
1574
1739
|
async function mapConcurrentOrdered<T, R>(
|
|
1575
1740
|
values: readonly T[],
|
|
1576
1741
|
concurrency: number,
|