@davideasden/pi-undo 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/native-performance.md +18 -0
- package/native/bin/pi-undo-fs-darwin-arm64 +0 -0
- package/native/bin/pi-undo-fs-darwin-x64 +0 -0
- package/native/bin/pi-undo-fs-linux-arm64 +0 -0
- package/native/bin/pi-undo-fs-linux-x64 +0 -0
- package/native/bin/pi-undo-fs-win32-arm64.exe +0 -0
- package/native/bin/pi-undo-fs-win32-x64.exe +0 -0
- package/package.json +1 -1
- package/src/resident-scan-client.ts +21 -14
- package/src/restore-engine.ts +120 -15
- package/src/snapshot-store.ts +131 -12
|
@@ -259,8 +259,26 @@ npm run typecheck
|
|
|
259
259
|
PI_UNDO_PACK_MEMORY_BENCHMARK=1 npx vitest run test/durable-pack-memory-benchmark.test.ts
|
|
260
260
|
PI_UNDO_PACK_MEMORY_BENCHMARK=1 PI_UNDO_PACK_MEMORY_TRANSFER=1 npx vitest run test/durable-pack-memory-benchmark.test.ts
|
|
261
261
|
PI_UNDO_RUN_RESIDENT_LIVE=1 npx vitest run test/resident-live.test.ts
|
|
262
|
+
PI_UNDO_MEMORY_BENCHMARK=1 npx vitest run test/resident-memory-benchmark.test.ts --reporter=verbose
|
|
262
263
|
```
|
|
263
264
|
|
|
265
|
+
### Pi runtime 驻留内存基线
|
|
266
|
+
|
|
267
|
+
`test/resident-memory-benchmark.test.ts` 以独立 Vitest 进程分别运行 SDK-only 测试夹具与启用 pi-undo 的 10,000 个 ignored 依赖包工作区,采集 workspace 建立后、SDK 初始化、pi-undo warm-up、run、undo/redo 和操作后的 idle 内存。报告包含 Vitest worker RSS、runner 进程树 RSS、native helper RSS,以及 Node heap、external 和 ArrayBuffer;Node 在可用时显式 GC 后另记 retained 样本。worker RSS 包含 Vitest 与基准代码本身,不等同生产 Pi 进程的 RSS;runner 进程树 RSS 是各进程 RSS 相加,会重复计算共享内存页,应主要比较同口径结果,并查看 worker/helper 分项。
|
|
268
|
+
|
|
269
|
+
macOS arm64、Node 23.11.1 的一次采样:
|
|
270
|
+
|
|
271
|
+
| 阶段 | Pi SDK worker RSS | pi-undo worker RSS | native helper RSS |
|
|
272
|
+
| --- | ---: | ---: | ---: |
|
|
273
|
+
| SDK 初始化后 retained | 162 MiB | 161 MiB | 未启动 |
|
|
274
|
+
| pi-undo warm-up 后 retained | — | 174 MiB | 35 MiB |
|
|
275
|
+
| undo/redo 后 idle retained | 162 MiB | 183 MiB | 38 MiB |
|
|
276
|
+
| 操作期 worker 峰值 | — | 196 MiB | 峰值约 52 MiB |
|
|
277
|
+
|
|
278
|
+
操作后 Node worker 的 retained heap 约 49 MiB,external 约 5 MiB;操作后 runner-tree RSS 约 394 MiB,但该总和包含 Vitest runner 并重复计算共享页。此前在 Vitest worker 内只遍历它自己的子进程,漏掉了 sibling native helper;当前基准会在 macOS native 模式下断言 warm-up helper RSS 非零。
|
|
279
|
+
|
|
280
|
+
这组数据说明,SDK-only 测试夹具的 worker RSS 已约 162 MiB;该数字包含 Vitest 开销,不是生产 Pi 的独立测量,也说明几十 MiB 不能作为整个 Pi 进程的目标。当前扩展在该工作区上相对 SDK-only 夹具增加的 Node RSS 约 20–22 MiB,常驻原生 helper 另约 35–38 MiB。单次小文件操作没有覆盖大 payload 或大规模历史 pack;后续内存优化应继续测大型差异,并优先收紧/预算化缓存与 workspace 路径索引。
|
|
281
|
+
|
|
264
282
|
本轮未在本机验证 Linux、其他架构或 Windows 的构建与运行;发布仍依赖现有六平台构建流程。
|
|
265
283
|
|
|
266
284
|
## 上一轮原生优化的历史数据
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -25,6 +25,7 @@ export class ResidentScanClient {
|
|
|
25
25
|
private child: ChildProcessWithoutNullStreams | undefined;
|
|
26
26
|
private starting: Promise<ChildProcessWithoutNullStreams | undefined> | undefined;
|
|
27
27
|
private queue: Promise<unknown> = Promise.resolve();
|
|
28
|
+
private pendingScans = 0;
|
|
28
29
|
private buffer = "";
|
|
29
30
|
private waiter: LineWaiter | undefined;
|
|
30
31
|
private closed = false;
|
|
@@ -36,6 +37,7 @@ export class ResidentScanClient {
|
|
|
36
37
|
private readonly timeoutMs: number,
|
|
37
38
|
/** 与一次性 helper 共享的持久目录缓存;serve 启动时读取,初始扫描后写回。 */
|
|
38
39
|
private readonly cachePath?: string,
|
|
40
|
+
private readonly idleTimeoutMs = 11_000,
|
|
39
41
|
) {}
|
|
40
42
|
|
|
41
43
|
/** 建立进程并完成首次全量扫描;失败静默,正式扫描会回退。 */
|
|
@@ -44,11 +46,16 @@ export class ResidentScanClient {
|
|
|
44
46
|
}
|
|
45
47
|
|
|
46
48
|
scan(timeoutMs = this.timeoutMs): Promise<JsonObject | undefined> {
|
|
49
|
+
if (this.idleTimer !== undefined) clearTimeout(this.idleTimer);
|
|
50
|
+
this.pendingScans += 1;
|
|
47
51
|
const result = this.queue.then(
|
|
48
52
|
() => this.scanOne(timeoutMs),
|
|
49
53
|
() => this.scanOne(timeoutMs),
|
|
50
54
|
);
|
|
51
|
-
const settled = result.finally(() =>
|
|
55
|
+
const settled = result.finally(() => {
|
|
56
|
+
this.pendingScans -= 1;
|
|
57
|
+
if (this.pendingScans === 0) this.scheduleIdle();
|
|
58
|
+
});
|
|
52
59
|
this.queue = settled.then(() => undefined, () => undefined);
|
|
53
60
|
return settled;
|
|
54
61
|
}
|
|
@@ -68,7 +75,7 @@ export class ResidentScanClient {
|
|
|
68
75
|
}
|
|
69
76
|
|
|
70
77
|
private async ensureChild(timeoutMs: number): Promise<ChildProcessWithoutNullStreams | undefined> {
|
|
71
|
-
if (this.child !== undefined && this.child.exitCode === null) return this.child;
|
|
78
|
+
if (this.child !== undefined && this.child.exitCode === null && this.child.signalCode === null) return this.child;
|
|
72
79
|
this.starting ??= this.spawnChild(timeoutMs).finally(() => { this.starting = undefined; });
|
|
73
80
|
return await this.starting;
|
|
74
81
|
}
|
|
@@ -118,7 +125,7 @@ export class ResidentScanClient {
|
|
|
118
125
|
}
|
|
119
126
|
|
|
120
127
|
private requestLine(child: ChildProcessWithoutNullStreams, request: string, timeoutMs: number): Promise<string | undefined> {
|
|
121
|
-
if (this.child !== child || child.exitCode !== null) return Promise.resolve(undefined);
|
|
128
|
+
if (this.child !== child || child.exitCode !== null || child.signalCode !== null) return Promise.resolve(undefined);
|
|
122
129
|
try {
|
|
123
130
|
child.stdin.write(`${request}\n`);
|
|
124
131
|
} catch {
|
|
@@ -214,30 +221,30 @@ export class ResidentScanClient {
|
|
|
214
221
|
private scheduleIdle(): void {
|
|
215
222
|
if (this.idleTimer !== undefined) clearTimeout(this.idleTimer);
|
|
216
223
|
if (this.closed) return;
|
|
217
|
-
// 与 Rust 端
|
|
218
|
-
this.idleTimer = setTimeout(() =>
|
|
219
|
-
void this.close();
|
|
220
|
-
}, 610_000);
|
|
224
|
+
// 与 Rust 端 10s 看门狗对齐;持久目录缓存让之后的按需重启可复用扫描结果。
|
|
225
|
+
this.idleTimer = setTimeout(() => this.stopChild(), this.idleTimeoutMs);
|
|
221
226
|
this.idleTimer.unref?.();
|
|
222
227
|
}
|
|
223
228
|
|
|
229
|
+
private stopChild(): void {
|
|
230
|
+
const child = this.child;
|
|
231
|
+
if (child !== undefined) this.invalidate(child);
|
|
232
|
+
this.buffer = "";
|
|
233
|
+
}
|
|
234
|
+
|
|
224
235
|
/** 丢弃当前进程但保留客户端,使下一次请求可以重新握手。 */
|
|
225
236
|
reset(): void {
|
|
226
237
|
this.closed = false;
|
|
227
238
|
if (this.idleTimer !== undefined) clearTimeout(this.idleTimer);
|
|
228
|
-
|
|
229
|
-
if (child !== undefined) this.invalidate(child);
|
|
230
|
-
this.buffer = "";
|
|
239
|
+
this.stopChild();
|
|
231
240
|
}
|
|
232
241
|
|
|
233
242
|
async close(): Promise<void> {
|
|
234
243
|
this.closed = true;
|
|
235
244
|
if (this.idleTimer !== undefined) clearTimeout(this.idleTimer);
|
|
236
|
-
|
|
237
|
-
if (child !== undefined) this.invalidate(child);
|
|
245
|
+
this.stopChild();
|
|
238
246
|
await this.queue;
|
|
239
|
-
this.
|
|
240
|
-
this.child = undefined;
|
|
247
|
+
this.stopChild();
|
|
241
248
|
}
|
|
242
249
|
}
|
|
243
250
|
|
package/src/restore-engine.ts
CHANGED
|
@@ -27,6 +27,7 @@ import { recoverPackedMutations } from "./packed-recovery.ts";
|
|
|
27
27
|
import type { ManifestId, RestorePath, SnapshotManifest, SnapshotRoot } from "./model.ts";
|
|
28
28
|
import {
|
|
29
29
|
assertNoSymlinkEscape,
|
|
30
|
+
assertNoSymlinkParents,
|
|
30
31
|
MutationPathIndex,
|
|
31
32
|
relativeSafePath,
|
|
32
33
|
sortDeletePaths,
|
|
@@ -44,9 +45,11 @@ import {
|
|
|
44
45
|
import { SnapshotStore, SnapshotStoreError } from "./snapshot-store.ts";
|
|
45
46
|
import { WorkspaceLock } from "./workspace-lock.ts";
|
|
46
47
|
|
|
47
|
-
const PREPARED_PLAN_CACHE_LIMIT =
|
|
48
|
+
const PREPARED_PLAN_CACHE_LIMIT = 2;
|
|
49
|
+
const PREPARED_PLAN_PATH_LIMIT = 8_192;
|
|
50
|
+
const DURABLE_CACHE_INDEX_LIMIT = 16;
|
|
48
51
|
const DURABLE_CACHE_INDEX_FILE = "index.json";
|
|
49
|
-
const DURABLE_PACK_MEMORY_MAX_BYTES =
|
|
52
|
+
const DURABLE_PACK_MEMORY_MAX_BYTES = 8 * 1024 * 1024;
|
|
50
53
|
const RESTORE_FILE_BATCH_MAX_ENTRIES = 1_024;
|
|
51
54
|
const RESTORE_FILE_BATCH_MAX_BYTES = 64 * 1024 * 1024;
|
|
52
55
|
const RESTORE_FILE_PREPARE_CONCURRENCY = 32;
|
|
@@ -201,6 +204,7 @@ export class RestoreEngine {
|
|
|
201
204
|
private readonly discovery: RootDiscovery;
|
|
202
205
|
private readonly beforeMutation: RestoreEngineOptions["beforeMutation"];
|
|
203
206
|
private readonly preparedPlans = new Map<string, PreparedRestorePlan>();
|
|
207
|
+
private readonly preparedPathsByPlan = new WeakMap<RestorePlan, PreparedRestorePlan>();
|
|
204
208
|
private readonly durablePackCache = new Map<string, CachedDurablePack>();
|
|
205
209
|
private readonly durablePackByPair = new Map<string, CachedDurablePair>();
|
|
206
210
|
private readonly verifiedDurablePacks = new Set<string>();
|
|
@@ -399,7 +403,12 @@ export class RestoreEngine {
|
|
|
399
403
|
false,
|
|
400
404
|
);
|
|
401
405
|
}
|
|
402
|
-
this.
|
|
406
|
+
const resident = this.durablePackByPair.get(pairKey);
|
|
407
|
+
if (resident?.planDigest === cached.planDigest) {
|
|
408
|
+
this.verifiedDurablePacks.add(`${pairKey}:${cached.planDigest}`);
|
|
409
|
+
} else {
|
|
410
|
+
this.verifiedDurablePacks.delete(`${pairKey}:${cached.planDigest}`);
|
|
411
|
+
}
|
|
403
412
|
return true;
|
|
404
413
|
} catch (error) {
|
|
405
414
|
rethrowOperationFailure(error);
|
|
@@ -427,22 +436,46 @@ export class RestoreEngine {
|
|
|
427
436
|
const currentPaths = prepared === undefined
|
|
428
437
|
? await this.readOwnedPaths(current, scope)
|
|
429
438
|
: prepared.currentPaths;
|
|
439
|
+
const fileGroups = new Map<string, OwnedPath[]>();
|
|
440
|
+
const nonFilePaths = new Map<string, OwnedPath>();
|
|
441
|
+
const absentPaths = new Set<string>();
|
|
430
442
|
for (const path of scope) {
|
|
431
|
-
if (path === "."
|
|
432
|
-
// 目录 scope 不能由单个 lstat 证明全部 rollback 内容,继续走 scoped capture。
|
|
443
|
+
if (path === ".") {
|
|
433
444
|
return false;
|
|
434
445
|
}
|
|
435
|
-
}
|
|
436
|
-
const matches = await mapConcurrentOrdered(scope, RESTORE_FILE_VERIFY_CONCURRENCY, async (path) => {
|
|
437
446
|
const owned = currentPaths.get(path);
|
|
438
447
|
if (owned === undefined) {
|
|
439
|
-
|
|
440
|
-
|
|
448
|
+
absentPaths.add(path);
|
|
449
|
+
continue;
|
|
441
450
|
}
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
451
|
+
if (owned.entry.kind === "directory") {
|
|
452
|
+
// 目录 scope 不能由单个 lstat/hash 证明全部 rollback 内容。
|
|
453
|
+
return false;
|
|
454
|
+
}
|
|
455
|
+
if (owned.entry.kind === "file") {
|
|
456
|
+
const group = fileGroups.get(owned.root.relativeRoot) ?? [];
|
|
457
|
+
group.push(owned);
|
|
458
|
+
fileGroups.set(owned.root.relativeRoot, group);
|
|
459
|
+
} else {
|
|
460
|
+
nonFilePaths.set(path, owned);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
const fileResults = await allCompleted([...fileGroups.values()].map((files) =>
|
|
464
|
+
this.verifySourceFiles(current.manifestId, files),
|
|
465
|
+
));
|
|
466
|
+
if (fileResults.some((result) => !result)) return false;
|
|
467
|
+
for (const path of absentPaths) {
|
|
468
|
+
await assertNoSymlinkEscape(this.workspaceRoot, path);
|
|
469
|
+
if (!await this.pathIsAbsent(path)) return false;
|
|
470
|
+
}
|
|
471
|
+
for (const [path, owned] of nonFilePaths) {
|
|
472
|
+
try {
|
|
473
|
+
await this.verifyEntry(current.manifestId, owned);
|
|
474
|
+
} catch {
|
|
475
|
+
return false;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
return true;
|
|
446
479
|
} catch (error) {
|
|
447
480
|
rethrowOperationFailure(error);
|
|
448
481
|
return false;
|
|
@@ -588,6 +621,13 @@ export class RestoreEngine {
|
|
|
588
621
|
pack: DurablePack,
|
|
589
622
|
): Promise<void> {
|
|
590
623
|
const bytes = await this.durablePackFileBytes(pack.storagePath);
|
|
624
|
+
if (bytes > this.durablePackCacheMaxBytes) {
|
|
625
|
+
const residentKey = `${pairKey}:${planDigest}`;
|
|
626
|
+
this.verifiedDurablePacks.delete(residentKey);
|
|
627
|
+
const existing = this.durablePackByPair.get(pairKey);
|
|
628
|
+
if (existing?.planDigest === planDigest) this.dropResidentDurablePack(pairKey);
|
|
629
|
+
return;
|
|
630
|
+
}
|
|
591
631
|
const previous = this.durablePackByPair.get(pairKey);
|
|
592
632
|
if (previous !== undefined) {
|
|
593
633
|
this.durablePackMemoryBytes -= previous.bytes;
|
|
@@ -606,6 +646,7 @@ export class RestoreEngine {
|
|
|
606
646
|
const cached = this.durablePackByPair.get(pairKey);
|
|
607
647
|
if (cached === undefined) return;
|
|
608
648
|
this.durablePackByPair.delete(pairKey);
|
|
649
|
+
this.verifiedDurablePacks.delete(`${pairKey}:${cached.planDigest}`);
|
|
609
650
|
this.durablePackMemoryBytes = Math.max(0, this.durablePackMemoryBytes - cached.bytes);
|
|
610
651
|
}
|
|
611
652
|
|
|
@@ -613,6 +654,9 @@ export class RestoreEngine {
|
|
|
613
654
|
for (const [pairKey, cached] of this.durablePackByPair) {
|
|
614
655
|
if (cached.planDigest === planDigest) this.dropResidentDurablePack(pairKey);
|
|
615
656
|
}
|
|
657
|
+
for (const key of this.verifiedDurablePacks) {
|
|
658
|
+
if (key.endsWith(`:${planDigest}`)) this.verifiedDurablePacks.delete(key);
|
|
659
|
+
}
|
|
616
660
|
}
|
|
617
661
|
|
|
618
662
|
private async readIndexedDurablePack(
|
|
@@ -717,7 +761,7 @@ export class RestoreEngine {
|
|
|
717
761
|
}
|
|
718
762
|
|
|
719
763
|
private async evictPersistedDurablePacks(keepPlanDigest: string): Promise<void> {
|
|
720
|
-
while (this.durableIndex !== undefined && this.durableIndex.entries.length >
|
|
764
|
+
while (this.durableIndex !== undefined && this.durableIndex.entries.length > DURABLE_CACHE_INDEX_LIMIT) {
|
|
721
765
|
const oldest = this.durableIndex.entries.find((entry) => entry.planDigest !== keepPlanDigest);
|
|
722
766
|
if (oldest === undefined) break;
|
|
723
767
|
await this.dropPersistedDurablePack(oldest.planDigest);
|
|
@@ -781,7 +825,10 @@ export class RestoreEngine {
|
|
|
781
825
|
targetPaths: ReadonlyMap<string, OwnedPath>,
|
|
782
826
|
): void {
|
|
783
827
|
const cachedPlan = cloneRestorePlan(plan);
|
|
784
|
-
|
|
828
|
+
const prepared = { plan: cachedPlan, currentPaths, targetPaths };
|
|
829
|
+
this.preparedPathsByPlan.set(plan, prepared);
|
|
830
|
+
if (currentPaths.size + targetPaths.size > PREPARED_PLAN_PATH_LIMIT) return;
|
|
831
|
+
this.preparedPlans.set(preparedPlanKey(cachedPlan), prepared);
|
|
785
832
|
while (this.preparedPlans.size > PREPARED_PLAN_CACHE_LIMIT) {
|
|
786
833
|
const oldest = this.preparedPlans.keys().next().value as string | undefined;
|
|
787
834
|
if (oldest === undefined) break;
|
|
@@ -790,6 +837,12 @@ export class RestoreEngine {
|
|
|
790
837
|
}
|
|
791
838
|
|
|
792
839
|
private takePreparedPlan(plan: RestorePlan): PreparedRestorePlan | undefined {
|
|
840
|
+
const transient = this.preparedPathsByPlan.get(plan);
|
|
841
|
+
this.preparedPathsByPlan.delete(plan);
|
|
842
|
+
if (transient !== undefined) {
|
|
843
|
+
this.preparedPlans.delete(preparedPlanKey(transient.plan));
|
|
844
|
+
return transient;
|
|
845
|
+
}
|
|
793
846
|
const key = preparedPlanKey(plan);
|
|
794
847
|
const prepared = this.preparedPlans.get(key);
|
|
795
848
|
if (prepared !== undefined) this.preparedPlans.delete(key);
|
|
@@ -1359,6 +1412,53 @@ export class RestoreEngine {
|
|
|
1359
1412
|
};
|
|
1360
1413
|
}
|
|
1361
1414
|
|
|
1415
|
+
private async verifySourceFiles(manifestId: ManifestId, files: readonly OwnedPath[]): Promise<boolean> {
|
|
1416
|
+
if (files.length === 0) return true;
|
|
1417
|
+
const hashWorkspaceFiles = (this.store as SnapshotStore & {
|
|
1418
|
+
readonly hashWorkspaceFiles?: (workspaceRoot: string, relativePaths: readonly string[]) => Promise<readonly string[]>;
|
|
1419
|
+
}).hashWorkspaceFiles;
|
|
1420
|
+
if (hashWorkspaceFiles === undefined) {
|
|
1421
|
+
await allCompleted(files.map((owned) => this.verifyEntry(manifestId, owned)));
|
|
1422
|
+
return true;
|
|
1423
|
+
}
|
|
1424
|
+
const root = files[0]!.root;
|
|
1425
|
+
if (files.some((owned) => owned.root.relativeRoot !== root.relativeRoot)) return false;
|
|
1426
|
+
const workspacePaths = files.map((owned) => owned.absolutePath);
|
|
1427
|
+
const before = await mapConcurrentOrdered(files, RESTORE_FILE_VERIFY_CONCURRENCY, (owned) =>
|
|
1428
|
+
this.sourceFileMetadata(owned));
|
|
1429
|
+
if (before.some((metadata) => metadata === undefined)) return false;
|
|
1430
|
+
await assertNoSymlinkParents(this.workspaceRoot, workspacePaths);
|
|
1431
|
+
const rootPath = root.relativeRoot === "."
|
|
1432
|
+
? this.workspaceRoot
|
|
1433
|
+
: join(this.workspaceRoot, ...root.relativeRoot.split("/"));
|
|
1434
|
+
const objectIds = await hashWorkspaceFiles.call(this.store,
|
|
1435
|
+
rootPath,
|
|
1436
|
+
files.map((owned) => owned.entry.relativePath),
|
|
1437
|
+
);
|
|
1438
|
+
if (objectIds.length !== files.length) return false;
|
|
1439
|
+
const after = await mapConcurrentOrdered(files, RESTORE_FILE_VERIFY_CONCURRENCY, (owned) =>
|
|
1440
|
+
this.sourceFileMetadata(owned));
|
|
1441
|
+
await assertNoSymlinkParents(this.workspaceRoot, workspacePaths);
|
|
1442
|
+
return files.every((owned, index) =>
|
|
1443
|
+
objectIds[index] === owned.entry.blobId &&
|
|
1444
|
+
before[index] !== undefined && after[index] !== undefined &&
|
|
1445
|
+
sameSourceMetadata(before[index]!, after[index]!));
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
private async sourceFileMetadata(owned: OwnedPath): Promise<BigIntStats | undefined> {
|
|
1449
|
+
try {
|
|
1450
|
+
const metadata = await lstat(this.absolutePath(owned.absolutePath), { bigint: true });
|
|
1451
|
+
if (!metadata.isFile() || metadata.isSymbolicLink() || owned.entry.blobId === null ||
|
|
1452
|
+
metadata.size !== BigInt(owned.entry.size) ||
|
|
1453
|
+
(metadata.mode & 0o111n) !== BigInt(owned.entry.mode & 0o111)) {
|
|
1454
|
+
return undefined;
|
|
1455
|
+
}
|
|
1456
|
+
return metadata;
|
|
1457
|
+
} catch (error) {
|
|
1458
|
+
if (hasErrorCode(error, "ENOENT") || hasErrorCode(error, "ENOTDIR")) return undefined;
|
|
1459
|
+
throw error;
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1362
1462
|
private async readOwnedPaths(
|
|
1363
1463
|
manifest: SnapshotManifest,
|
|
1364
1464
|
scopePaths?: readonly string[],
|
|
@@ -2639,6 +2739,11 @@ async function mapConcurrentOrdered<T, R>(
|
|
|
2639
2739
|
return results;
|
|
2640
2740
|
}
|
|
2641
2741
|
|
|
2742
|
+
function sameSourceMetadata(left: BigIntStats, right: BigIntStats): boolean {
|
|
2743
|
+
return left.dev === right.dev && left.ino === right.ino && left.mode === right.mode &&
|
|
2744
|
+
left.size === right.size && left.mtimeNs === right.mtimeNs && left.ctimeNs === right.ctimeNs;
|
|
2745
|
+
}
|
|
2746
|
+
|
|
2642
2747
|
function durablePairKey(
|
|
2643
2748
|
currentManifestId: ManifestId,
|
|
2644
2749
|
targetManifestId: ManifestId,
|
package/src/snapshot-store.ts
CHANGED
|
@@ -34,10 +34,11 @@ const GC_METADATA_FILE = "gc.json";
|
|
|
34
34
|
const GC_RETENTION_MS = 7 * 24 * 60 * 60 * 1_000;
|
|
35
35
|
const IGNORE_POLICY = "git-check-ignore-v1";
|
|
36
36
|
const NULL_DEVICE = process.platform === "win32" ? "NUL" : "/dev/null";
|
|
37
|
-
const TREE_CACHE_LIMIT =
|
|
37
|
+
const TREE_CACHE_LIMIT = 16;
|
|
38
|
+
const TREE_CACHE_ENTRY_LIMIT = 32_768;
|
|
38
39
|
const TREE_SCOPE_COMPACT_MIN_PATHS = process.platform === "win32" ? 512 : 4_096;
|
|
39
40
|
const TREE_SCOPE_PROBE_MAX_BYTES = 2 * 1024 * 1024;
|
|
40
|
-
const TREE_BLOB_MEMBERSHIP_LIMIT =
|
|
41
|
+
const TREE_BLOB_MEMBERSHIP_LIMIT = 8_192;
|
|
41
42
|
const HASH_BATCH_MAX_PATHS = process.platform === "win32" ? 128 : 2_048;
|
|
42
43
|
const HASH_BATCH_MAX_ARGUMENT_BYTES = process.platform === "win32" ? 24 * 1024 : 128 * 1024;
|
|
43
44
|
const HASH_BATCH_CONCURRENCY = 4;
|
|
@@ -51,11 +52,14 @@ const METADATA_BATCHES_MAX_PATHS = 16_384;
|
|
|
51
52
|
const METADATA_BATCHES_MAX_REQUEST_BYTES = 4 * 1024 * 1024;
|
|
52
53
|
const INDEX_BATCH_MAX_ENTRIES = 4_096;
|
|
53
54
|
const INDEX_BATCH_MAX_BYTES = 8 * 1024 * 1024;
|
|
54
|
-
const
|
|
55
|
+
const DEFAULT_BLOB_CACHE_MAX_BYTES = 16 * 1024 * 1024;
|
|
56
|
+
const BLOB_CACHE_MAX_ENTRIES = 1_024;
|
|
55
57
|
const BLOB_BATCH_MAX_BYTES = 16 * 1024 * 1024;
|
|
56
58
|
const BLOB_BATCH_MAX_ENTRIES = process.platform === "win32" ? 256 : 2_048;
|
|
57
59
|
const RACY_CLEAN_WINDOW_NS = 2_000_000_000n;
|
|
58
60
|
const LEAF_CACHE_FILE = "leaf-cache.json";
|
|
61
|
+
const VISIBLE_LEAF_CACHE_MAX_ENTRIES = 8_192;
|
|
62
|
+
const VISIBLE_LEAF_CACHE_MAX_FILE_BYTES = 8 * 1024 * 1024;
|
|
59
63
|
// 私有 ODB 的 loose object 达到阈值后合并为 pack;阈值取得足够大,避免把高频 capture
|
|
60
64
|
// 变成高频 repack,同时让大改动后的 cat-file 查询尽快走 pack 索引。
|
|
61
65
|
const COMPACT_MIN_LOOSE_OBJECTS = 4_096;
|
|
@@ -241,6 +245,8 @@ export interface SnapshotStoreOptions {
|
|
|
241
245
|
readonly lock?: WorkspaceLock;
|
|
242
246
|
readonly clock?: () => number;
|
|
243
247
|
readonly nativeMetadata?: NativeMetadataPort;
|
|
248
|
+
/** 仅供测试或受控调优:0 禁用 blob 内存缓存。 */
|
|
249
|
+
readonly blobCacheMaxBytes?: number;
|
|
244
250
|
/** 仅供测试或受控调优:0 禁用 loose object 合并。 */
|
|
245
251
|
readonly compactMinLooseObjects?: number;
|
|
246
252
|
/** 阈值持续达到多久后才真正合并;默认 5 分钟。 */
|
|
@@ -329,10 +335,13 @@ export class SnapshotStore {
|
|
|
329
335
|
private readonly nativeMetadata: NativeMetadataPort;
|
|
330
336
|
private readonly compactMinLooseObjects: number;
|
|
331
337
|
private readonly compactMinAgeMs: number;
|
|
338
|
+
private readonly blobCacheMaxBytes: number;
|
|
332
339
|
private lastCompactCheckAt: number | undefined;
|
|
333
340
|
private readonly manifestLocations = new Map<string, string>();
|
|
334
341
|
// Tree 与 blob 都由 object ID 内容寻址;缓存只复用已从私有 ODB 读取的不可变内容。
|
|
335
342
|
private readonly treeEntriesCache = new Map<string, Promise<CapturedTreeEntry[]>>();
|
|
343
|
+
private readonly treeEntryCounts = new Map<string, number>();
|
|
344
|
+
private treeEntriesCachedCount = 0;
|
|
336
345
|
private readonly treeBlobMembership = new Map<string, string>();
|
|
337
346
|
private readonly blobCache = new Map<string, CachedBlob>();
|
|
338
347
|
private readonly visibleLeafCache = new Map<string, Map<string, CachedVisibleLeaf>>();
|
|
@@ -353,6 +362,10 @@ export class SnapshotStore {
|
|
|
353
362
|
this.nativeMetadata = options.nativeMetadata ?? new NativeMetadataInspector();
|
|
354
363
|
this.compactMinLooseObjects = options.compactMinLooseObjects ?? COMPACT_MIN_LOOSE_OBJECTS;
|
|
355
364
|
this.compactMinAgeMs = options.compactMinAgeMs ?? COMPACT_MIN_AGE_MS;
|
|
365
|
+
this.blobCacheMaxBytes = options.blobCacheMaxBytes ?? DEFAULT_BLOB_CACHE_MAX_BYTES;
|
|
366
|
+
if (!Number.isSafeInteger(this.blobCacheMaxBytes) || this.blobCacheMaxBytes < 0) {
|
|
367
|
+
throw new Error("blob 内存缓存预算必须是非负整数");
|
|
368
|
+
}
|
|
356
369
|
}
|
|
357
370
|
|
|
358
371
|
static supportsValidatedBlobBatch(store: SnapshotStore): boolean {
|
|
@@ -382,6 +395,26 @@ export class SnapshotStore {
|
|
|
382
395
|
return result;
|
|
383
396
|
}
|
|
384
397
|
|
|
398
|
+
async hashWorkspaceFiles(workspaceRoot: string, relativePaths: readonly string[]): Promise<readonly string[]> {
|
|
399
|
+
if (relativePaths.length === 0) return [];
|
|
400
|
+
const paths = relativePaths.map((path) => {
|
|
401
|
+
relativeSafePath(workspaceRoot, path);
|
|
402
|
+
if (path.split("/").some((part) => part.toLowerCase() === ".git")) {
|
|
403
|
+
throw new SnapshotStoreError("object_missing", "不能 hash Git metadata");
|
|
404
|
+
}
|
|
405
|
+
return path;
|
|
406
|
+
});
|
|
407
|
+
const batches = workspaceHashPathBatches(paths);
|
|
408
|
+
const hashed = await mapConcurrentOrdered(batches, HASH_BATCH_CONCURRENCY, async (batch) => {
|
|
409
|
+
const output = await this.runGit(["hash-object", "--no-filters", "--", ...batch], {
|
|
410
|
+
cwd: workspaceRoot,
|
|
411
|
+
env: sourceGitEnvironment(),
|
|
412
|
+
});
|
|
413
|
+
return parseObjectIdLines(output, batch.length);
|
|
414
|
+
});
|
|
415
|
+
return hashed.flat();
|
|
416
|
+
}
|
|
417
|
+
|
|
385
418
|
async durableCacheDirectory(): Promise<string> {
|
|
386
419
|
const directory = join(this.storeRoot, "durable-cache");
|
|
387
420
|
await mkdir(directory, { recursive: true });
|
|
@@ -1614,10 +1647,16 @@ export class SnapshotStore {
|
|
|
1614
1647
|
const cache = inclusions !== null && inclusions.length === 0
|
|
1615
1648
|
? new Map<string, CachedVisibleLeaf>()
|
|
1616
1649
|
: new Map(previous);
|
|
1650
|
+
let available = VISIBLE_LEAF_CACHE_MAX_ENTRIES - (
|
|
1651
|
+
[...this.visibleLeafCache.entries()]
|
|
1652
|
+
.filter(([directory]) => directory !== gitDirectory)
|
|
1653
|
+
.reduce((total, [, entries]) => total + entries.size, 0) + cache.size
|
|
1654
|
+
);
|
|
1617
1655
|
if (inclusions !== null && inclusions.length > 0) {
|
|
1618
1656
|
for (const relativePath of cache.keys()) {
|
|
1619
1657
|
if (inclusions.some((inclusion) => isPathAtOrBelow(inclusion, relativePath))) {
|
|
1620
1658
|
cache.delete(relativePath);
|
|
1659
|
+
available += 1;
|
|
1621
1660
|
}
|
|
1622
1661
|
}
|
|
1623
1662
|
}
|
|
@@ -1625,15 +1664,39 @@ export class SnapshotStore {
|
|
|
1625
1664
|
const objectId = staged.objectIds.get(leaf.relativePath);
|
|
1626
1665
|
if (objectId === undefined) continue;
|
|
1627
1666
|
if (!leaf.cacheable) {
|
|
1628
|
-
cache.delete(leaf.relativePath);
|
|
1667
|
+
if (cache.delete(leaf.relativePath)) available += 1;
|
|
1629
1668
|
continue;
|
|
1630
1669
|
}
|
|
1670
|
+
if (!cache.has(leaf.relativePath) && available <= 0) continue;
|
|
1671
|
+
if (!cache.has(leaf.relativePath)) available -= 1;
|
|
1631
1672
|
cache.set(leaf.relativePath, { ...leaf, objectId, verifiedAtNs: staged.verifiedAtNs });
|
|
1632
1673
|
}
|
|
1674
|
+
this.visibleLeafCache.set(gitDirectory, cache);
|
|
1675
|
+
for (const storeDirectory of this.trimVisibleLeafCache()) {
|
|
1676
|
+
this.leafCacheDirtyDirectories.add(storeDirectory);
|
|
1677
|
+
}
|
|
1633
1678
|
if (!samePersistedLeafCache(previous, cache)) {
|
|
1634
1679
|
this.leafCacheDirtyDirectories.add(storeDirectoryForGitDirectory(gitDirectory));
|
|
1635
1680
|
}
|
|
1636
|
-
|
|
1681
|
+
}
|
|
1682
|
+
|
|
1683
|
+
private trimVisibleLeafCache(): Set<string> {
|
|
1684
|
+
let count = [...this.visibleLeafCache.values()].reduce((total, cache) => total + cache.size, 0);
|
|
1685
|
+
const changedDirectories = new Set<string>();
|
|
1686
|
+
while (count > VISIBLE_LEAF_CACHE_MAX_ENTRIES) {
|
|
1687
|
+
let removed = false;
|
|
1688
|
+
for (const [gitDirectory, cache] of this.visibleLeafCache) {
|
|
1689
|
+
const oldest = cache.keys().next().value as string | undefined;
|
|
1690
|
+
if (oldest === undefined) continue;
|
|
1691
|
+
cache.delete(oldest);
|
|
1692
|
+
count -= 1;
|
|
1693
|
+
changedDirectories.add(storeDirectoryForGitDirectory(gitDirectory));
|
|
1694
|
+
removed = true;
|
|
1695
|
+
break;
|
|
1696
|
+
}
|
|
1697
|
+
if (!removed) break;
|
|
1698
|
+
}
|
|
1699
|
+
return changedDirectories;
|
|
1637
1700
|
}
|
|
1638
1701
|
|
|
1639
1702
|
/** 从 storeDirectory 读取持久化叶子缓存并合并进内存;进程内已有条目优先。 */
|
|
@@ -1642,16 +1705,22 @@ export class SnapshotStore {
|
|
|
1642
1705
|
this.leafCacheDirectoriesLoaded.add(storeDirectory);
|
|
1643
1706
|
let file: unknown;
|
|
1644
1707
|
try {
|
|
1708
|
+
const metadata = await lstat(join(storeDirectory, LEAF_CACHE_FILE));
|
|
1709
|
+
if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > VISIBLE_LEAF_CACHE_MAX_FILE_BYTES) return;
|
|
1645
1710
|
file = JSON.parse(await readFile(join(storeDirectory, LEAF_CACHE_FILE), "utf8"));
|
|
1646
1711
|
} catch {
|
|
1647
1712
|
return; // 缺失或损坏:忽略,本次 capture 走冷路径并重建缓存。
|
|
1648
1713
|
}
|
|
1649
1714
|
if (!isPersistedLeafCacheFile(file)) return;
|
|
1650
1715
|
const prefix = `${storeDirectory}${sep}`;
|
|
1716
|
+
let remaining = VISIBLE_LEAF_CACHE_MAX_ENTRIES -
|
|
1717
|
+
[...this.visibleLeafCache.values()].reduce((total, cache) => total + cache.size, 0);
|
|
1651
1718
|
for (const [gitDirectory, entries] of Object.entries(file.entries)) {
|
|
1719
|
+
if (remaining <= 0) break;
|
|
1652
1720
|
if (!gitDirectory.startsWith(prefix) || this.visibleLeafCache.has(gitDirectory)) continue;
|
|
1653
1721
|
const cache = new Map<string, CachedVisibleLeaf>();
|
|
1654
1722
|
for (const [relativePath, entry] of Object.entries(entries)) {
|
|
1723
|
+
if (remaining <= 0) break;
|
|
1655
1724
|
cache.set(relativePath, {
|
|
1656
1725
|
relativePath,
|
|
1657
1726
|
kind: entry.kind,
|
|
@@ -1663,6 +1732,7 @@ export class SnapshotStore {
|
|
|
1663
1732
|
objectId: entry.objectId,
|
|
1664
1733
|
verifiedAtNs: BigInt(entry.verifiedAtNs),
|
|
1665
1734
|
});
|
|
1735
|
+
remaining -= 1;
|
|
1666
1736
|
}
|
|
1667
1737
|
this.visibleLeafCache.set(gitDirectory, cache);
|
|
1668
1738
|
}
|
|
@@ -2002,7 +2072,11 @@ export class SnapshotStore {
|
|
|
2002
2072
|
: [...new Set(rootScopePaths.map((path) => relativeSafePath("/", path)))].sort(comparePaths);
|
|
2003
2073
|
const key = `${gitDirectory}\0${treeId}\0${scope === undefined ? "*" : checksum(canonicalJson(scope))}`;
|
|
2004
2074
|
const cached = this.treeEntriesCache.get(key);
|
|
2005
|
-
if (cached !== undefined)
|
|
2075
|
+
if (cached !== undefined) {
|
|
2076
|
+
this.treeEntriesCache.delete(key);
|
|
2077
|
+
this.treeEntriesCache.set(key, cached);
|
|
2078
|
+
return cached;
|
|
2079
|
+
}
|
|
2006
2080
|
const pending = (async () => {
|
|
2007
2081
|
const queryScope = scope === undefined || scope.length < TREE_SCOPE_COMPACT_MIN_PATHS
|
|
2008
2082
|
? scope
|
|
@@ -2031,16 +2105,40 @@ export class SnapshotStore {
|
|
|
2031
2105
|
while (this.treeEntriesCache.size > TREE_CACHE_LIMIT) {
|
|
2032
2106
|
const oldest = this.treeEntriesCache.keys().next().value as string | undefined;
|
|
2033
2107
|
if (oldest === undefined || oldest === key) break;
|
|
2034
|
-
this.
|
|
2108
|
+
this.dropTreeEntriesCache(oldest);
|
|
2035
2109
|
}
|
|
2036
2110
|
try {
|
|
2037
|
-
|
|
2111
|
+
const entries = await pending;
|
|
2112
|
+
this.cacheTreeEntries(key, pending, entries.length);
|
|
2113
|
+
return entries;
|
|
2038
2114
|
} catch (error) {
|
|
2039
|
-
if (this.treeEntriesCache.get(key) === pending) this.
|
|
2115
|
+
if (this.treeEntriesCache.get(key) === pending) this.dropTreeEntriesCache(key);
|
|
2040
2116
|
throw error;
|
|
2041
2117
|
}
|
|
2042
2118
|
}
|
|
2043
2119
|
|
|
2120
|
+
private cacheTreeEntries(key: string, pending: Promise<CapturedTreeEntry[]>, count: number): void {
|
|
2121
|
+
if (this.treeEntriesCache.get(key) !== pending) return;
|
|
2122
|
+
if (count > TREE_CACHE_ENTRY_LIMIT) {
|
|
2123
|
+
this.dropTreeEntriesCache(key);
|
|
2124
|
+
return;
|
|
2125
|
+
}
|
|
2126
|
+
const previous = this.treeEntryCounts.get(key) ?? 0;
|
|
2127
|
+
this.treeEntryCounts.set(key, count);
|
|
2128
|
+
this.treeEntriesCachedCount += count - previous;
|
|
2129
|
+
while (this.treeEntriesCache.size > TREE_CACHE_LIMIT || this.treeEntriesCachedCount > TREE_CACHE_ENTRY_LIMIT) {
|
|
2130
|
+
const oldestKey = this.treeEntriesCache.keys().next().value as string | undefined;
|
|
2131
|
+
if (oldestKey === undefined || oldestKey === key) break;
|
|
2132
|
+
this.dropTreeEntriesCache(oldestKey);
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
|
|
2136
|
+
private dropTreeEntriesCache(key: string): void {
|
|
2137
|
+
this.treeEntriesCache.delete(key);
|
|
2138
|
+
this.treeEntriesCachedCount -= this.treeEntryCounts.get(key) ?? 0;
|
|
2139
|
+
this.treeEntryCounts.delete(key);
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2044
2142
|
private async compactTreeQueryScope(
|
|
2045
2143
|
gitDirectory: string,
|
|
2046
2144
|
treeId: string,
|
|
@@ -2209,7 +2307,7 @@ export class SnapshotStore {
|
|
|
2209
2307
|
|
|
2210
2308
|
private rememberBlobBytes(gitDirectory: string, objectId: string, bytes: Uint8Array): void {
|
|
2211
2309
|
const key = blobCacheKey(gitDirectory, objectId);
|
|
2212
|
-
if (this.blobCache.has(key) || bytes.byteLength >
|
|
2310
|
+
if (this.blobCacheMaxBytes === 0 || this.blobCache.has(key) || bytes.byteLength > this.blobCacheMaxBytes) return;
|
|
2213
2311
|
const entry: CachedBlob = { promise: Promise.resolve(bytes), size: 0 };
|
|
2214
2312
|
this.blobCache.set(key, entry);
|
|
2215
2313
|
this.finishBlobCacheEntry(key, entry, bytes.byteLength);
|
|
@@ -2217,13 +2315,13 @@ export class SnapshotStore {
|
|
|
2217
2315
|
|
|
2218
2316
|
private finishBlobCacheEntry(key: string, entry: CachedBlob, size: number): void {
|
|
2219
2317
|
if (this.blobCache.get(key) !== entry) return;
|
|
2220
|
-
if (size >
|
|
2318
|
+
if (this.blobCacheMaxBytes === 0 || size > this.blobCacheMaxBytes) {
|
|
2221
2319
|
this.blobCache.delete(key);
|
|
2222
2320
|
return;
|
|
2223
2321
|
}
|
|
2224
2322
|
entry.size = size;
|
|
2225
2323
|
this.blobCacheBytes += size;
|
|
2226
|
-
while (this.blobCacheBytes >
|
|
2324
|
+
while (this.blobCacheBytes > this.blobCacheMaxBytes || this.blobCache.size > BLOB_CACHE_MAX_ENTRIES) {
|
|
2227
2325
|
const oldestKey = this.blobCache.keys().next().value as string | undefined;
|
|
2228
2326
|
if (oldestKey === undefined) break;
|
|
2229
2327
|
const oldest = this.blobCache.get(oldestKey)!;
|
|
@@ -2970,6 +3068,27 @@ function hashPathBatches(leaves: readonly VisibleLeaf[]): VisibleLeaf[][] {
|
|
|
2970
3068
|
return result;
|
|
2971
3069
|
}
|
|
2972
3070
|
|
|
3071
|
+
function workspaceHashPathBatches(paths: readonly string[]): string[][] {
|
|
3072
|
+
const result: string[][] = [];
|
|
3073
|
+
let current: string[] = [];
|
|
3074
|
+
let argumentBytes = 0;
|
|
3075
|
+
for (const path of paths) {
|
|
3076
|
+
const pathBytes = Buffer.byteLength(path, "utf8") + 1;
|
|
3077
|
+
if (
|
|
3078
|
+
current.length > 0 &&
|
|
3079
|
+
(current.length >= HASH_BATCH_MAX_PATHS || argumentBytes + pathBytes > HASH_BATCH_MAX_ARGUMENT_BYTES)
|
|
3080
|
+
) {
|
|
3081
|
+
result.push(current);
|
|
3082
|
+
current = [];
|
|
3083
|
+
argumentBytes = 0;
|
|
3084
|
+
}
|
|
3085
|
+
current.push(path);
|
|
3086
|
+
argumentBytes += pathBytes;
|
|
3087
|
+
}
|
|
3088
|
+
if (current.length > 0) result.push(current);
|
|
3089
|
+
return result;
|
|
3090
|
+
}
|
|
3091
|
+
|
|
2973
3092
|
function parseObjectIdLines(output: string, expectedCount: number): string[] {
|
|
2974
3093
|
const lines = output.endsWith("\n") ? output.slice(0, -1).split("\n") : output.split("\n");
|
|
2975
3094
|
if (lines.length !== expectedCount || lines.some((line) => !isObjectId(line))) {
|