@davideasden/pi-undo 0.3.6 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/docs/native-performance.md +140 -4
- package/native/bin/pi-undo-fs-darwin-arm64 +0 -0
- package/native/bin/pi-undo-fs-darwin-x64 +0 -0
- package/native/bin/pi-undo-fs-linux-arm64 +0 -0
- package/native/bin/pi-undo-fs-linux-x64 +0 -0
- package/native/bin/pi-undo-fs-win32-arm64.exe +0 -0
- package/native/bin/pi-undo-fs-win32-x64.exe +0 -0
- package/package.json +1 -1
- package/src/controller.ts +50 -20
- package/src/durable-pack.ts +26 -11
- package/src/native-directory-scan.ts +19 -8
- package/src/path-safety.ts +42 -0
- package/src/pi-runtime.ts +8 -0
- package/src/resident-scan-client.ts +5 -1
- package/src/restore-engine.ts +155 -47
- package/src/snapshot-store.ts +180 -9
package/src/snapshot-store.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { BigIntStats } from "node:fs";
|
|
2
|
-
import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, realpath, rm, stat } from "node:fs/promises";
|
|
2
|
+
import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, realpath, rm, stat, writeFile } from "node:fs/promises";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
5
5
|
|
|
@@ -56,6 +56,17 @@ const BLOB_BATCH_MAX_BYTES = 16 * 1024 * 1024;
|
|
|
56
56
|
const BLOB_BATCH_MAX_ENTRIES = process.platform === "win32" ? 256 : 2_048;
|
|
57
57
|
const RACY_CLEAN_WINDOW_NS = 2_000_000_000n;
|
|
58
58
|
const LEAF_CACHE_FILE = "leaf-cache.json";
|
|
59
|
+
// 私有 ODB 的 loose object 达到阈值后合并为 pack;阈值取得足够大,避免把高频 capture
|
|
60
|
+
// 变成高频 repack,同时让大改动后的 cat-file 查询尽快走 pack 索引。
|
|
61
|
+
const COMPACT_MIN_LOOSE_OBJECTS = 4_096;
|
|
62
|
+
const COMPACT_MIN_LOOSE_KIB = 256 * 1024;
|
|
63
|
+
// 阈值达到后不立即打包:刚写入的大批对象正是下一次 undo 要校验的对象,
|
|
64
|
+
// 立刻 repack 只会把成本提前,单次操作反而更慢。连续 5 分钟维持高压后才合并,
|
|
65
|
+
// 让打包摊销到后续多次 plan/verify 上。
|
|
66
|
+
const COMPACT_MIN_AGE_MS = 5 * 60 * 1_000;
|
|
67
|
+
const COMPACT_PENDING_FILE = "pi-undo-compact.json";
|
|
68
|
+
// `git count-objects -v` 在 2 万 loose object 的库上约 80-100 ms;不需要每次 settle 都付。
|
|
69
|
+
const COMPACT_CHECK_INTERVAL_MS = 60_000;
|
|
59
70
|
|
|
60
71
|
interface PinRecord {
|
|
61
72
|
readonly schemaVersion: 1;
|
|
@@ -64,6 +75,11 @@ interface PinRecord {
|
|
|
64
75
|
readonly updatedAt: string;
|
|
65
76
|
}
|
|
66
77
|
|
|
78
|
+
interface CompactPendingRecord {
|
|
79
|
+
readonly schemaVersion: 1;
|
|
80
|
+
readonly observedAt: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
67
83
|
interface StoreGcRecord {
|
|
68
84
|
readonly schemaVersion: 1;
|
|
69
85
|
readonly lastUsedAt: number;
|
|
@@ -225,6 +241,10 @@ export interface SnapshotStoreOptions {
|
|
|
225
241
|
readonly lock?: WorkspaceLock;
|
|
226
242
|
readonly clock?: () => number;
|
|
227
243
|
readonly nativeMetadata?: NativeMetadataPort;
|
|
244
|
+
/** 仅供测试或受控调优:0 禁用 loose object 合并。 */
|
|
245
|
+
readonly compactMinLooseObjects?: number;
|
|
246
|
+
/** 阈值持续达到多久后才真正合并;默认 5 分钟。 */
|
|
247
|
+
readonly compactMinAgeMs?: number;
|
|
228
248
|
}
|
|
229
249
|
|
|
230
250
|
export interface CaptureOptions {
|
|
@@ -234,6 +254,11 @@ export interface CaptureOptions {
|
|
|
234
254
|
* 因此校验窗口内出现的漂移依然会被拒绝。仅限调用方与本次枚举之间没有文件变更时使用。
|
|
235
255
|
*/
|
|
236
256
|
readonly topologyAlreadyValidated?: boolean;
|
|
257
|
+
/**
|
|
258
|
+
* 调用方已证明 baseline 一定不是当前状态时跳过 baseline 探测;完整 capture 仍会
|
|
259
|
+
* 重新建立对象、ignored proof 与 leaves 缓存。
|
|
260
|
+
*/
|
|
261
|
+
readonly skipBaselineProbe?: boolean;
|
|
237
262
|
/**
|
|
238
263
|
* 跳过捕获或枚举结束后的 topology 二次校验(safety-snapshot post 扫描)。
|
|
239
264
|
* 仅在调用方已持有工作区独占锁、且刚刚完成 discover 并确认 topology 未变化时传 true。
|
|
@@ -302,6 +327,9 @@ export class SnapshotStore {
|
|
|
302
327
|
private readonly lock: WorkspaceLock;
|
|
303
328
|
private readonly clock: () => number;
|
|
304
329
|
private readonly nativeMetadata: NativeMetadataPort;
|
|
330
|
+
private readonly compactMinLooseObjects: number;
|
|
331
|
+
private readonly compactMinAgeMs: number;
|
|
332
|
+
private lastCompactCheckAt: number | undefined;
|
|
305
333
|
private readonly manifestLocations = new Map<string, string>();
|
|
306
334
|
// Tree 与 blob 都由 object ID 内容寻址;缓存只复用已从私有 ODB 读取的不可变内容。
|
|
307
335
|
private readonly treeEntriesCache = new Map<string, Promise<CapturedTreeEntry[]>>();
|
|
@@ -323,6 +351,8 @@ export class SnapshotStore {
|
|
|
323
351
|
this.lock = options.lock ?? new WorkspaceLock();
|
|
324
352
|
this.clock = options.clock ?? Date.now;
|
|
325
353
|
this.nativeMetadata = options.nativeMetadata ?? new NativeMetadataInspector();
|
|
354
|
+
this.compactMinLooseObjects = options.compactMinLooseObjects ?? COMPACT_MIN_LOOSE_OBJECTS;
|
|
355
|
+
this.compactMinAgeMs = options.compactMinAgeMs ?? COMPACT_MIN_AGE_MS;
|
|
326
356
|
}
|
|
327
357
|
|
|
328
358
|
static supportsValidatedBlobBatch(store: SnapshotStore): boolean {
|
|
@@ -521,9 +551,13 @@ export class SnapshotStore {
|
|
|
521
551
|
throw new SnapshotStoreError("capture_failed", `broken root 不能静默进入 baseline 校验: ${brokenRoots.join(", ")}`);
|
|
522
552
|
}
|
|
523
553
|
const baselineValidationStartedAt = performance.now();
|
|
524
|
-
let baselineFresh
|
|
554
|
+
let baselineFresh = false;
|
|
525
555
|
try {
|
|
526
|
-
|
|
556
|
+
// skipBaselineProbe 只由调用方对“已确定失效”的 baseline 传入;此时探测必然失败,
|
|
557
|
+
// 跳过它可以把一次完整 collectVisibleLeaves + ignored proof 从结算路径上省掉。
|
|
558
|
+
if (options.skipBaselineProbe !== true) {
|
|
559
|
+
baselineFresh = await this.isBaselineFresh(topology, baseline, scope, options, diagnostic);
|
|
560
|
+
}
|
|
527
561
|
} finally {
|
|
528
562
|
publishCaptureTiming(options.onTiming, "settled.baselineValidation", baselineValidationStartedAt);
|
|
529
563
|
}
|
|
@@ -538,11 +572,23 @@ export class SnapshotStore {
|
|
|
538
572
|
// 已完成一次捕获前 topology 校验;完整回退仍保留捕获后的校验。
|
|
539
573
|
if (diagnostic !== undefined) diagnostic.strategy = "baseline-fallback";
|
|
540
574
|
const fullCaptureStartedAt = performance.now();
|
|
575
|
+
let captured: SnapshotManifest;
|
|
541
576
|
try {
|
|
542
|
-
|
|
577
|
+
captured = await this.captureLocked(
|
|
578
|
+
topology,
|
|
579
|
+
scope,
|
|
580
|
+
{ ...options, topologyAlreadyValidated: true, skipPostTopologyCheck: options.skipPostTopologyCheck },
|
|
581
|
+
diagnostic,
|
|
582
|
+
);
|
|
543
583
|
} finally {
|
|
544
584
|
publishCaptureTiming(options.onTiming, "settled.fullCapture", fullCaptureStartedAt);
|
|
545
585
|
}
|
|
586
|
+
// 完整捕获刚写入一批 loose object;结算路径已持有 store 与 workspace 锁,
|
|
587
|
+
// 是唯一可以在不打断恢复读取的情况下做维护的窗口。维护时长单独报告。
|
|
588
|
+
const compactStartedAt = performance.now();
|
|
589
|
+
await this.compactLooseObjectsLocked(topology);
|
|
590
|
+
publishCaptureTiming(options.onTiming, "settled.compact", compactStartedAt);
|
|
591
|
+
return captured;
|
|
546
592
|
} catch (error) {
|
|
547
593
|
if (error instanceof SnapshotStoreError) {
|
|
548
594
|
throw error;
|
|
@@ -551,6 +597,106 @@ export class SnapshotStore {
|
|
|
551
597
|
}
|
|
552
598
|
}
|
|
553
599
|
|
|
600
|
+
/**
|
|
601
|
+
* 把私有 ODB 的 loose object 合并为 pack。
|
|
602
|
+
*
|
|
603
|
+
* 私有仓库没有 refs,`git repack -a` 不会把 manifest 引用的对象视为可达,
|
|
604
|
+
* 因此这里直接按磁盘上的 loose object 列表调用 pack-objects,再 prune-packed。
|
|
605
|
+
* 维护失败不影响已经完成的 capture;取消/超时仍按操作契约向上传递。
|
|
606
|
+
*/
|
|
607
|
+
private async compactLooseObjectsLocked(topology: RootTopology): Promise<void> {
|
|
608
|
+
if (this.compactMinLooseObjects <= 0) return;
|
|
609
|
+
const now = this.clock();
|
|
610
|
+
if (this.lastCompactCheckAt !== undefined && now - this.lastCompactCheckAt < COMPACT_CHECK_INTERVAL_MS) return;
|
|
611
|
+
this.lastCompactCheckAt = now;
|
|
612
|
+
const storeDirectory = this.storeDirectory(topology);
|
|
613
|
+
let roots: string[];
|
|
614
|
+
try {
|
|
615
|
+
roots = await readdir(join(storeDirectory, "roots"));
|
|
616
|
+
} catch {
|
|
617
|
+
return;
|
|
618
|
+
}
|
|
619
|
+
for (const root of roots) {
|
|
620
|
+
try {
|
|
621
|
+
await this.compactGitDirectory(join(storeDirectory, "roots", root, "git"));
|
|
622
|
+
} catch {
|
|
623
|
+
// 维护是 best-effort:capture 已经完成,repack 失败/取消不影响本次结果,
|
|
624
|
+
// 未合并的 loose object 会在下一次阈值检查时重试。
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
private async compactGitDirectory(gitDirectory: string): Promise<void> {
|
|
630
|
+
const environment = cleanGitEnvironment();
|
|
631
|
+
const output = await this.runGit(
|
|
632
|
+
["--git-dir", gitDirectory, "count-objects", "-v"],
|
|
633
|
+
{ env: environment },
|
|
634
|
+
);
|
|
635
|
+
const looseCount = parseCountObjectsField(output, "count");
|
|
636
|
+
const looseKiB = parseCountObjectsField(output, "size");
|
|
637
|
+
const pendingPath = join(gitDirectory, COMPACT_PENDING_FILE);
|
|
638
|
+
if (looseCount < this.compactMinLooseObjects && looseKiB < COMPACT_MIN_LOOSE_KIB) {
|
|
639
|
+
// 已经低于阈值(例如 store 被清理):撤销待合并状态。
|
|
640
|
+
await rm(pendingPath, { force: true }).catch(() => {});
|
|
641
|
+
return;
|
|
642
|
+
}
|
|
643
|
+
const pending = await readCompactPending(pendingPath);
|
|
644
|
+
const now = this.clock();
|
|
645
|
+
if (pending === undefined) {
|
|
646
|
+
if (this.compactMinAgeMs <= 0) {
|
|
647
|
+
// 测试或显式配置:立即合并。
|
|
648
|
+
} else {
|
|
649
|
+
// 只是延迟标记;不 fsync,丢失最多让合并再等一个窗口。
|
|
650
|
+
await writeFile(
|
|
651
|
+
pendingPath,
|
|
652
|
+
JSON.stringify({ schemaVersion: 1, observedAt: now } satisfies CompactPendingRecord),
|
|
653
|
+
{ mode: 0o600 },
|
|
654
|
+
).catch(() => {});
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
} else if (now - pending.observedAt < this.compactMinAgeMs) {
|
|
658
|
+
return;
|
|
659
|
+
}
|
|
660
|
+
const objects = await this.listLooseObjectIds(gitDirectory);
|
|
661
|
+
if (objects.length === 0) {
|
|
662
|
+
await rm(pendingPath, { force: true }).catch(() => {});
|
|
663
|
+
return;
|
|
664
|
+
}
|
|
665
|
+
await this.runGit(
|
|
666
|
+
["--git-dir", gitDirectory, "pack-objects", "-q", join(gitDirectory, "objects", "pack", "pack")],
|
|
667
|
+
{ env: environment, stdin: `${objects.join("\n")}\n` },
|
|
668
|
+
);
|
|
669
|
+
await this.runGit(["--git-dir", gitDirectory, "prune-packed"], { env: environment });
|
|
670
|
+
await rm(pendingPath, { force: true }).catch(() => {});
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
private async listLooseObjectIds(gitDirectory: string): Promise<string[]> {
|
|
674
|
+
const objectsDirectory = join(gitDirectory, "objects");
|
|
675
|
+
const result: string[] = [];
|
|
676
|
+
let directories: string[];
|
|
677
|
+
try {
|
|
678
|
+
directories = await readdir(objectsDirectory);
|
|
679
|
+
} catch {
|
|
680
|
+
return result;
|
|
681
|
+
}
|
|
682
|
+
for (const directory of directories) {
|
|
683
|
+
if (!/^[0-9a-f]{2}$/.test(directory)) continue;
|
|
684
|
+
let files: string[];
|
|
685
|
+
try {
|
|
686
|
+
files = await readdir(join(objectsDirectory, directory));
|
|
687
|
+
} catch {
|
|
688
|
+
continue;
|
|
689
|
+
}
|
|
690
|
+
for (const file of files) {
|
|
691
|
+
// 兼容 SHA-1(40 字符)与 SHA-256(64 字符)对象目录。
|
|
692
|
+
if (/^[0-9a-f]{38}$/.test(file) || /^[0-9a-f]{62}$/.test(file)) {
|
|
693
|
+
result.push(`${directory}${file}`);
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
return result.sort();
|
|
698
|
+
}
|
|
699
|
+
|
|
554
700
|
private async isBaselineFresh(
|
|
555
701
|
topology: RootTopology,
|
|
556
702
|
baseline: SnapshotManifest,
|
|
@@ -1024,11 +1170,15 @@ export class SnapshotStore {
|
|
|
1024
1170
|
pending.push({ gitDirectory, objectId: request.blobId });
|
|
1025
1171
|
}
|
|
1026
1172
|
const bytesById = new Map<string, Uint8Array>();
|
|
1027
|
-
|
|
1173
|
+
// 同一 ODB 的对象 ID 去重必须用 Set:10k 个唯一 blob 时 Array.includes 是 O(n²)。
|
|
1174
|
+
const byGitDirectory = new Map<string, Set<string>>();
|
|
1028
1175
|
for (const { gitDirectory, objectId } of pending) {
|
|
1029
|
-
|
|
1030
|
-
if (
|
|
1031
|
-
|
|
1176
|
+
let ids = byGitDirectory.get(gitDirectory);
|
|
1177
|
+
if (ids === undefined) {
|
|
1178
|
+
ids = new Set<string>();
|
|
1179
|
+
byGitDirectory.set(gitDirectory, ids);
|
|
1180
|
+
}
|
|
1181
|
+
ids.add(objectId);
|
|
1032
1182
|
}
|
|
1033
1183
|
for (const [gitDirectory, objectIds] of byGitDirectory) {
|
|
1034
1184
|
for (const objectId of objectIds) {
|
|
@@ -1036,7 +1186,7 @@ export class SnapshotStore {
|
|
|
1036
1186
|
if (cached === undefined) continue;
|
|
1037
1187
|
bytesById.set(objectId, await cached.promise);
|
|
1038
1188
|
}
|
|
1039
|
-
const missing = objectIds.filter((objectId) => !bytesById.has(objectId));
|
|
1189
|
+
const missing = [...objectIds].filter((objectId) => !bytesById.has(objectId));
|
|
1040
1190
|
if (missing.length === 0) continue;
|
|
1041
1191
|
// batch 头部自带长度,无需预先知道 size;按响应自身长度解析后逐条校验。
|
|
1042
1192
|
const loaded = parseBatchBlobOutputByOrder(
|
|
@@ -2828,6 +2978,27 @@ function parseObjectIdLines(output: string, expectedCount: number): string[] {
|
|
|
2828
2978
|
return lines;
|
|
2829
2979
|
}
|
|
2830
2980
|
|
|
2981
|
+
async function readCompactPending(path: string): Promise<CompactPendingRecord | undefined> {
|
|
2982
|
+
try {
|
|
2983
|
+
const value: unknown = JSON.parse(await readFile(path, "utf8"));
|
|
2984
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
|
|
2985
|
+
const record = value as Record<string, unknown>;
|
|
2986
|
+
if (record.schemaVersion !== 1 || typeof record.observedAt !== "number" ||
|
|
2987
|
+
!Number.isSafeInteger(record.observedAt)) return undefined;
|
|
2988
|
+
return { schemaVersion: 1, observedAt: record.observedAt };
|
|
2989
|
+
} catch {
|
|
2990
|
+
return undefined;
|
|
2991
|
+
}
|
|
2992
|
+
}
|
|
2993
|
+
|
|
2994
|
+
/** 解析 `git count-objects -v` 的整数字段;缺失或非法值按 0 处理(维护是 best-effort)。 */
|
|
2995
|
+
function parseCountObjectsField(output: string, field: string): number {
|
|
2996
|
+
const match = new RegExp(`^${field}: (\\d+)$`, "m").exec(output);
|
|
2997
|
+
if (match === null) return 0;
|
|
2998
|
+
const value = Number(match[1]);
|
|
2999
|
+
return Number.isSafeInteger(value) ? value : 0;
|
|
3000
|
+
}
|
|
3001
|
+
|
|
2831
3002
|
function treeDirectoryPaths(entries: readonly CapturedTreeEntry[]): string[] {
|
|
2832
3003
|
const directories: string[] = [];
|
|
2833
3004
|
let previousParts: readonly string[] = [];
|