@davideasden/pi-undo 0.3.7 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/docs/native-performance.md +132 -0
- package/native/bin/pi-undo-fs-darwin-arm64 +0 -0
- package/native/bin/pi-undo-fs-darwin-x64 +0 -0
- package/native/bin/pi-undo-fs-linux-arm64 +0 -0
- package/native/bin/pi-undo-fs-linux-x64 +0 -0
- package/native/bin/pi-undo-fs-win32-arm64.exe +0 -0
- package/native/bin/pi-undo-fs-win32-x64.exe +0 -0
- package/package.json +1 -1
- package/src/controller.ts +27 -9
- package/src/durable-pack.ts +26 -11
- package/src/native-directory-scan.ts +19 -8
- package/src/path-safety.ts +42 -0
- package/src/pi-runtime.ts +3 -0
- package/src/resident-scan-client.ts +5 -1
- package/src/restore-engine.ts +20 -21
- package/src/snapshot-store.ts +180 -9
package/README.md
CHANGED
|
@@ -214,6 +214,15 @@ Performance gains come from:
|
|
|
214
214
|
- **Durable pack caching** — completed agent runs build and cache reverse/forward durable packs; the undo/redo hot path reuses the cached pack with manifest pin, avoiding redundant snapshot reads.
|
|
215
215
|
- **Indexed WAL record lookups** — mutation records are indexed by ordinal for O(1) direct access instead of linear scan.
|
|
216
216
|
- **Indexed path conflict resolution** — `exactExclusions` are stored as a Set for O(1) membership checks instead of O(n) `Array.includes`.
|
|
217
|
+
- **Private ODB compaction** — once 4,096 loose objects (or 256 MiB) have stayed above the threshold for 5 minutes, they are packed so `cat-file --batch-check` and blob reads use the pack index. Deferring avoids charging the repack (≈2.7 s for 20k objects) to the first undo after a large change; 10k-object `assertComplete` drops from ~331 ms to ~49 ms and 10k-blob reads from ~443–966 ms to ~158 ms once packed.
|
|
218
|
+
- **Indexed mutation path checks** — `MutationPathIndex` replaces the per-target-path `[...deletePaths, ...writePaths].some(...)` scan (10k × 10k ≈ 962 ms → < 1 ms).
|
|
219
|
+
- **Deduped batch blob reads** — object IDs are deduplicated with a `Set` instead of `Array.includes` (10k unique blobs ≈ 101 ms → 2.2 ms).
|
|
220
|
+
- **Shared, parallel parent validation** — native metadata inspect validates the deduplicated parent set once per request with 32-way concurrency (10k deep paths 111.7 ms → 71.0 ms).
|
|
221
|
+
- **Persistent resident scan cache** — `--serve` starts from the on-disk v2 directory cache and writes it back; idle timeout raised to 600 s. 20k-directory cold start 377 ms → 251 ms with a warm cache, steady-state requests remain 13–20 ms.
|
|
222
|
+
- **Racy-window baseline skip** — settled capture skips the guaranteed-failing `isBaselineFresh` probe within 2 s after a restore, removing ~227–240 ms from the settlement path.
|
|
223
|
+
- **Transferred pack payloads** — the restore path hands pack ownership of its freshly read blobs instead of copying them (64 MiB benchmark: +64.0 MiB → +0.01 MiB extra ArrayBuffers, +2.3 MiB RSS), and the pack checksum is computed during the write pass.
|
|
224
|
+
|
|
225
|
+
The ignored-present proof still runs `git ls-files --others --ignored --exclude-standard` on every full capture (~363 ms for 10k ignored files on the local benchmark). Reusing a previous proof needs directory-cache evidence to cross the helper boundary, which is tracked in `docs/native-performance.md` as the remaining follow-up.
|
|
217
226
|
- **Batch validated manifest blob reads** — durable pack reads coalesce blob fetches into a single batch per manifest with combined Git validation.
|
|
218
227
|
|
|
219
228
|
## How It Works
|
|
@@ -131,6 +131,138 @@ GitRunner 会对单次 Git stdout 应用 256 MiB 硬上限;超过上限时终
|
|
|
131
131
|
|
|
132
132
|
真实 Pi SDK、10,000 个 ignored 依赖包、单文件变更的三次 undo/redo 基准中,改动前 undo 中位数为 2,150 ms、每次 3 次完整拓扑发现;改动后中位数为 1,745 ms、每次 2 次发现。单次诊断中 safety 阶段从 504 ms 降至 90 ms:当 `changedPaths` 全部是叶子且工作区内容逐项匹配 checkpoint source 时,直接复用 source manifest;目录 scope、内容不匹配或无法证明时仍走原 scoped safety capture。运行时持有 workspace lock 后,visible-pre 的可见路径校验保留,但跳过其后的重复 topology 扫描;`restore-pre` 和 `restore-post` 两次检查仍保留。上述为本机样本,墙钟数据会受扫描缓存和负载影响,不构成其他工作区的速度保证。
|
|
133
133
|
|
|
134
|
+
## 私有 ODB 合并、父目录校验与 pack 内存
|
|
135
|
+
|
|
136
|
+
本轮不改恢复语义,只处理五类剩余成本:私有快照库长期保留 loose object、恢复路径上的两处 O(n²)、native metadata inspect 的重复父目录校验、常驻扫描重启后的冷启动,以及 durable pack 的整份 payload 副本。
|
|
137
|
+
|
|
138
|
+
### 私有 ODB 定期合并 loose object
|
|
139
|
+
|
|
140
|
+
私有仓库显式关闭了 `gc.auto` 与 `maintenance.auto`,此前没有任何 repack 路径,capture 写入的 blob/tree 会一直以 loose object 保留。`cat-file --batch-check` 与 `cat-file --batch` 因此需要逐个打开 loose 文件。
|
|
141
|
+
|
|
142
|
+
现在 `captureBaselineLocked` 在完整 capture 结束后检查 `git count-objects -v`,满足以下条件才合并:
|
|
143
|
+
|
|
144
|
+
- loose 对象达到 4,096 个或 256 MiB;
|
|
145
|
+
- 该状态已持续至少 5 分钟(首次越阈值只写 `pi-undo-compact.json` 观察时间,不立即 repack);
|
|
146
|
+
- 距离上次检查至少 60 秒(`count-objects` 在 2 万对象库上约 80–100 ms,不能每次 settle 都付)。
|
|
147
|
+
|
|
148
|
+
真正执行时按磁盘上的 loose object 列表调用 `git pack-objects`,再 `git prune-packed`。私有仓库没有 refs,`git repack -a` 不会把 manifest 引用的对象视为可达,因此不能直接使用 repack。维护窗口与 capture 共用 store 锁,调用方(settled capture)同时持有 workspace 独占锁;维护失败不影响已经完成的 capture。维护耗时单独报告为 `settled.compact`,不计入 `settled.fullCapture`。默认阈值可通过 `SnapshotStoreOptions.compactMinLooseObjects` / `compactMinAgeMs` 调整,`compactMinLooseObjects: 0` 表示禁用。
|
|
149
|
+
|
|
150
|
+
**为什么延迟 5 分钟**:A/B 复测发现“立即合并”会让大改动后的第一次 undo 变慢。10,000 个文件全部修改、settle 完整 capture 后再 `prepareDurableRestore` 的本机结果:
|
|
151
|
+
|
|
152
|
+
| 方案 | settle(含维护) | prepare(含 plan + pack) | 合计 |
|
|
153
|
+
| --- | ---: | ---: | ---: |
|
|
154
|
+
| 基线(不合并) | 3,997 ms | 2,490 ms | 6,487 ms |
|
|
155
|
+
| 立即合并(旧版本轮实现) | 6,074 ms(其中 repack 2,666 ms) | 1,140 ms | 7,214 ms |
|
|
156
|
+
| 延迟合并(当前实现) | 3,651–3,684 ms(检查约 134 ms) | 1,728–1,775 ms | 5,379–5,459 ms |
|
|
157
|
+
|
|
158
|
+
立即合并虽然把 `plan.complete` 从约 930 ms 降到约 299 ms,但 repack 成本(20,403 个对象约 2.7 s,其中 `prune-packed` 约 1 s)被提前到刚写完这批对象的时候,单次流程反而更慢。延迟到 5 分钟后,repack 摊销到后续多次 plan/verify:每次 undo 约省 600 ms,约 4–5 次后收回成本。
|
|
159
|
+
|
|
160
|
+
合并前后(10,000 个内容各异的文件,真实 `SnapshotStore`):
|
|
161
|
+
|
|
162
|
+
| 指标 | 合并前(loose) | 合并后(packed) |
|
|
163
|
+
| --- | ---: | ---: |
|
|
164
|
+
| `assertComplete`(10k blob) | 331 ms | 49 ms |
|
|
165
|
+
| `readBlobs`(10k blob,含 manifest 复验) | 443–966 ms | 158 ms |
|
|
166
|
+
| `count-objects` | `count: 10202, in-pack: 0` | `count: 0, in-pack: 10205` |
|
|
167
|
+
|
|
168
|
+
### 本轮端到端 A/B 复核
|
|
169
|
+
|
|
170
|
+
基线为合并前的 `ce0b71e5`;两侧在隔离 worktree 中顺序运行同一用例。
|
|
171
|
+
|
|
172
|
+
**大型工作区(10,000 个依赖包,修改 1 个文件)**:每次调用 3 次 undo/redo,三次调用结果:
|
|
173
|
+
|
|
174
|
+
| 指标 | 基线 | 本轮 |
|
|
175
|
+
| --- | ---: | ---: |
|
|
176
|
+
| undo 中位数 | 345 / 343 / 335 ms | 367 / 337 / 328 ms |
|
|
177
|
+
| redo 中位数 | 565 / 563 / 567 ms | 636 / 569 / 561 ms |
|
|
178
|
+
| 扫描次数 | 2 / 2 / 2 | 2 / 2 / 2 |
|
|
179
|
+
|
|
180
|
+
差异在运行波动内,**没有证明该场景的端到端提速**。该场景的可见文件只有 2 个,扫描已是常驻 13–20 ms 稳态,本轮改动主要作用在 10k 文件的捕获/校验路径,不改变这条主耗时。
|
|
181
|
+
|
|
182
|
+
**durable pack 准备(10,001 个文件全部修改,冷/热各三次)**:
|
|
183
|
+
|
|
184
|
+
| 场景 | 基线 | 本轮 | 变化 |
|
|
185
|
+
| --- | ---: | ---: | ---: |
|
|
186
|
+
| 冷缓存完整准备 | 2,598 / 2,449 / 2,431 ms | 1,697 / 1,666 / 1,666 ms | 约 -32% |
|
|
187
|
+
| 热缓存完整准备 | 1,852 / 1,852 / 1,808 ms | 1,039 / 1,060 / 1,060 ms | 约 -43% |
|
|
188
|
+
| `prepare.blobs` | 892–978 ms | 148–173 ms | 约 6× |
|
|
189
|
+
| `plan.complete`(loose) | 约 900 ms | 约 900 ms | 不变 |
|
|
190
|
+
|
|
191
|
+
主要收益来自 `readBlobs` 的 `Set` 去重(10k 唯一 blob 的 `Array.includes` 约 100 ms)与 `MutationPathIndex`;`plan.complete` 只有在 ODB 合并后才从约 930 ms 降到约 299 ms。
|
|
192
|
+
|
|
193
|
+
**native metadata inspect**:10,000 个深路径(20,001 个父目录)从 111.5–112.8 ms 降到 68.3–72.3 ms;10 个路径的小请求 4.25–4.27 ms → 4.16–4.32 ms,加入“小于 64 个父目录保持串行”后没有回归。
|
|
194
|
+
|
|
195
|
+
**常驻扫描**:20,002 个目录的 serve 冷启动 377 ms、带持久缓存 251 ms、稳态 13–20 ms。该收益在进程重启/长空闲后才体现;同一 session 内的连续 discover 本来就是稳态。
|
|
196
|
+
|
|
197
|
+
**racy 窗口 baseline 探测**:A/B 中强制 `skipBaselineProbe` 时 `settled.baselineValidation` 从 530 ms 降到 0 ms;实际只在 restore 后 2 秒内的下一次 settle 生效,不是每轮都有的收益。
|
|
198
|
+
|
|
199
|
+
### 恢复路径的 O(n²)
|
|
200
|
+
|
|
201
|
+
- `MutationPathIndex`(`src/path-safety.ts`):`reusableTargetPaths` 原实现对每个 target path 重新展开 `[...deletePaths, ...writePaths]` 并线性扫描,10k × 10k 时约 962 ms。现在一次构建精确集合与排序列表,单次判定近 O(log n),同数据形状 < 1 ms(构建 1.6 ms)。语义与 `mutationPath === path || mutationPath.startsWith(path + "/") || path.startsWith(mutationPath + "/")` 逐项等价,随机集合差分测试覆盖。
|
|
202
|
+
- `SnapshotStore.readBlobs`:同一 ODB 内的对象 ID 去重从 `Array.includes` 改为 `Set`,10k 个唯一 blob 从约 101 ms 降到 2.2 ms。该路径会在 durable pack 预制的 source/target 两个方向各执行一次。
|
|
203
|
+
|
|
204
|
+
### native metadata inspect 父目录校验
|
|
205
|
+
|
|
206
|
+
`inspect-batches` 原来按 batch 各自构建 `BTreeSet<PathBuf>` 父目录集合并前后各校验一遍;10,000 个 `node_modules/pkg/lib/index.js` 会收集 20,001 个父目录、执行 40,002 次 `lstat`。现在:
|
|
207
|
+
|
|
208
|
+
- 路径校验与父目录收集在整个请求内共享,parent 集合用 `HashSet<&str>` 借用请求中的路径字符串,避免为每个祖先分配 `PathBuf`;
|
|
209
|
+
- 父目录校验用与叶子 stat 相同的 32 路并发执行,前后仍然各一次;
|
|
210
|
+
- 保留原有 symlink/非目录诊断;macOS 上 `O_NOFOLLOW|O_DIRECTORY` 对 symlink 返回 `ENOTDIR` 的差异不再影响判定,因为改为直接 `lstat`。
|
|
211
|
+
|
|
212
|
+
本机 10k 深路径(20,001 个父目录)`--inspect-batches`:111.7 ms → 71.0 ms,约 36%;平铺 10k 路径的基线仍约 20 ms。
|
|
213
|
+
|
|
214
|
+
### 常驻扫描的持久缓存与闲置超时
|
|
215
|
+
|
|
216
|
+
`--serve` 握手新增可选 `cachePath`。启动时读取一次性扫描使用的 v2 目录缓存作为初始 `previous`,每个目录仍逐项核验元数据,漂移会重新枚举;初始扫描结束后把最终目录表原子写回,供下一次进程启动复用。闲置超时从 60 s 提高到 600 s,TS 客户端从 70 s 提高到 610 s,避免间歇使用时反复付冷启动。
|
|
217
|
+
|
|
218
|
+
本机 20,002 个目录:
|
|
219
|
+
|
|
220
|
+
| 场景 | 初始扫描 |
|
|
221
|
+
| --- | ---: |
|
|
222
|
+
| serve 冷启动(无缓存) | 377 ms |
|
|
223
|
+
| serve 使用持久缓存启动 | 251 ms |
|
|
224
|
+
| serve 稳态请求(无变化/单文件变化) | 13–20 ms |
|
|
225
|
+
|
|
226
|
+
### settled 的 baseline 探测
|
|
227
|
+
|
|
228
|
+
restore 成功后 2 s 内的 racy-clean 窗口内,文件 ctime 尚未稳定,`isBaselineFresh` 必然返回 false,但仍会完整执行 `assertObjectsComplete`、`collectVisibleLeaves`、ignored proof 和叶子复核。调用方在这个窗口内传入的 baseline 已确定失效,因此 `CaptureOptions.skipBaselineProbe` 直接进入 fullCapture。
|
|
229
|
+
|
|
230
|
+
控制器只在 `changedPaths.length > 0` 的 restore 后记录 target baseline;空 changedPaths 的 session-only undo 不写文件,探测仍可能成功,不跳过。多文件工作区实测 `settled.baselineValidation` 约 227–240 ms,本改动把这部分从结算路径上移除(诊断中保留 0 ms 阶段)。
|
|
231
|
+
|
|
232
|
+
### durable pack 的内存副本
|
|
233
|
+
|
|
234
|
+
`createDurablePack` 默认仍在校验前复制 payload,保证调用方之后改写输入不影响磁盘与驻留 pack。`restore-engine` 的 blob 来自本次操作独占的 store 读取,两个生产调用点改用 `payloadOwnership: "transfer"`,pack 直接引用这些 buffer。`loadDurablePack` 也不再为每个 variant 复制 payload,而是共享同一次 `readFile` 的 subarray,`leaf()` 返回时再复制。pack checksum 改为在 `writeAll` 按实际写出的字节增量计算,发布后不再完整遍历 payload。
|
|
235
|
+
|
|
236
|
+
本机独立进程 64 MiB payload 基准(阶段边界观测,不是瞬时峰值):
|
|
237
|
+
|
|
238
|
+
| 模式 | 额外 ArrayBuffer | 额外 RSS | `pack.checksum` |
|
|
239
|
+
| --- | ---: | ---: | ---: |
|
|
240
|
+
| copy(默认/旧行为) | 64.0 MiB | 66.8 MiB | 24 ms |
|
|
241
|
+
| transfer(生产路径) | 0.01 MiB | 2.3 MiB | ~0 ms |
|
|
242
|
+
|
|
243
|
+
`PI_UNDO_PACK_MEMORY_TRANSFER=1` 可复现 transfer 数值;输入 buffer 本身仍由调用方持有,因此该表只表示 pack 阶段的额外增量。
|
|
244
|
+
|
|
245
|
+
### 仍未处理:ignored-present proof 的全量枚举
|
|
246
|
+
|
|
247
|
+
`captureIgnoredPresentPaths` 每次完整 capture 仍执行 `git ls-files --others --ignored --exclude-standard`。本机 10,000 个 `node_modules` 忽略文件约 363 ms(`--directory` 可降到约 30 ms,但会丢失具体路径,无法直接替代 proof)。复用旧 proof 需要证明 ignored 子树的目录项集合未变,即把目录扫描的 cache 复用证据(或 FSEvents 代数)传递到 capture;这需要新的 helper 协议和缓存 schema 变更,本轮未实现,避免在恢复安全判定上引入未经复核的信任路径。当前只完成了同一条路径上 native inspect 部分的优化。
|
|
248
|
+
|
|
249
|
+
### 测试与复现
|
|
250
|
+
|
|
251
|
+
新增覆盖:MutationPathIndex 随机差分、readBlobs 重复请求只发一次 batch、`skipBaselineProbe` 直接完整捕获、自动 loose object 合并与阈值以下不重写 pack、transfer 所有权与 copy 逐字节一致且 `leaf()` 仍返回副本、serve handshake 的 cachePath(`PI_UNDO_RUN_RESIDENT_LIVE=1` 冒烟用例)。
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
npm run build:native
|
|
255
|
+
cp native/pi-undo-fs/target/release/pi-undo-fs native/bin/pi-undo-fs-darwin-arm64
|
|
256
|
+
npm run test:native
|
|
257
|
+
npm test
|
|
258
|
+
npm run typecheck
|
|
259
|
+
PI_UNDO_PACK_MEMORY_BENCHMARK=1 npx vitest run test/durable-pack-memory-benchmark.test.ts
|
|
260
|
+
PI_UNDO_PACK_MEMORY_BENCHMARK=1 PI_UNDO_PACK_MEMORY_TRANSFER=1 npx vitest run test/durable-pack-memory-benchmark.test.ts
|
|
261
|
+
PI_UNDO_RUN_RESIDENT_LIVE=1 npx vitest run test/resident-live.test.ts
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
本轮未在本机验证 Linux、其他架构或 Windows 的构建与运行;发布仍依赖现有六平台构建流程。
|
|
265
|
+
|
|
134
266
|
## 上一轮原生优化的历史数据
|
|
135
267
|
|
|
136
268
|
下表保留上一轮以 `d7d80230` 为起点的结果,便于追踪原生扫描及普通文件恢复的收益;这些数据不是本轮缓存改动的前后对比。单位为毫秒。
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
package/src/controller.ts
CHANGED
|
@@ -49,6 +49,7 @@ export interface ControllerDependencies {
|
|
|
49
49
|
baseline: SnapshotManifest,
|
|
50
50
|
onDiagnostics?: (diagnostic: CaptureDiagnostic) => void,
|
|
51
51
|
onTiming?: (phase: string, durationMs: number) => void,
|
|
52
|
+
options?: CaptureBaselineOptions,
|
|
52
53
|
) => Promise<SnapshotManifest>;
|
|
53
54
|
readonly captureSafety?: (
|
|
54
55
|
referenceManifestId: ManifestId,
|
|
@@ -87,6 +88,14 @@ export interface ControllerDependencies {
|
|
|
87
88
|
readonly onSettlement?: (diagnostic: SettlementDiagnostic) => void | Promise<void>;
|
|
88
89
|
}
|
|
89
90
|
|
|
91
|
+
export interface CaptureBaselineOptions {
|
|
92
|
+
/**
|
|
93
|
+
* 调用方已证明 baseline 一定不是当前状态(例如 restore 后仍在 racy-clean 窗口内),
|
|
94
|
+
* 跳过 isBaselineFresh 的完整探测,直接走 fullCapture。
|
|
95
|
+
*/
|
|
96
|
+
readonly skipBaselineProbe?: boolean;
|
|
97
|
+
}
|
|
98
|
+
|
|
90
99
|
export interface JournalPort {
|
|
91
100
|
prepare(descriptor: OperationDescriptor, plan: unknown): Promise<void>;
|
|
92
101
|
setPhase(
|
|
@@ -626,16 +635,21 @@ export class UndoControllerImpl implements UndoController {
|
|
|
626
635
|
try {
|
|
627
636
|
// settled 复用 run 开始时的 before;helper 在缺少 captureBaseline 时回退完整 capture。
|
|
628
637
|
// 撕裂捕获(并发写撞上断言窗口)是暂态的:短暂退避后重试一次,成功则不清空历史。
|
|
638
|
+
// racy-clean 窗口(2s)内文件 ctime 未稳定,isBaselineFresh 必然失败且会多跑一轮
|
|
639
|
+
// collectVisibleLeaves;只有超过 2s 后才能从 target manifest 受益。
|
|
640
|
+
const appliedTargetInRacyWindow = appliedTargetManifest !== undefined &&
|
|
641
|
+
appliedTargetAt !== undefined && performance.now() - appliedTargetAt < 2_000;
|
|
629
642
|
const after = await measure("settled.capture", () =>
|
|
630
643
|
this.captureSettledBaselineWithRetry(
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
(appliedTargetManifest !== undefined && appliedTargetAt !== undefined &&
|
|
634
|
-
performance.now() - appliedTargetAt >= 2_000)
|
|
644
|
+
appliedTargetManifest !== undefined && appliedTargetAt !== undefined &&
|
|
645
|
+
!appliedTargetInRacyWindow
|
|
635
646
|
? appliedTargetManifest
|
|
636
647
|
: staged.before,
|
|
637
648
|
(diagnostic) => captureDiagnostics.push(diagnostic),
|
|
638
649
|
recordCaptureTiming,
|
|
650
|
+
// 窗口内传入的 baseline 是 restore 前的 staged.before,已确定失效;
|
|
651
|
+
// 探测只会白跑一轮,直接进入 fullCapture。
|
|
652
|
+
{ skipBaselineProbe: appliedTargetInRacyWindow },
|
|
639
653
|
));
|
|
640
654
|
failureStage = "settled.changedPaths";
|
|
641
655
|
const changedPaths = await measure("settled.changedPaths", () =>
|
|
@@ -1045,8 +1059,10 @@ export class UndoControllerImpl implements UndoController {
|
|
|
1045
1059
|
await this.dependencies.journal.markCommitted(descriptor.opId);
|
|
1046
1060
|
});
|
|
1047
1061
|
this.lastSafetyManifestId = rollback.manifestId;
|
|
1048
|
-
|
|
1049
|
-
|
|
1062
|
+
if (checkpoint.changedPaths.length > 0) {
|
|
1063
|
+
this.lastAppliedTargetManifest = target;
|
|
1064
|
+
this.lastAppliedTargetAt = performance.now();
|
|
1065
|
+
}
|
|
1050
1066
|
return done(this.advanceHistory(action, checkpoint, { code: "ok", changedFiles: applied.verifiedPaths }));
|
|
1051
1067
|
} catch (error) {
|
|
1052
1068
|
if (this.recordUnsafeExit(error)) return done(this.recoveryResult());
|
|
@@ -1244,6 +1260,7 @@ export class UndoControllerImpl implements UndoController {
|
|
|
1244
1260
|
baseline: SnapshotManifest,
|
|
1245
1261
|
onDiagnostics?: (diagnostic: CaptureDiagnostic) => void,
|
|
1246
1262
|
onTiming?: (phase: string, durationMs: number) => void,
|
|
1263
|
+
options: CaptureBaselineOptions = {},
|
|
1247
1264
|
): Promise<SnapshotManifest> {
|
|
1248
1265
|
const captureBaseline = this.dependencies.captureBaseline;
|
|
1249
1266
|
if (captureBaseline === undefined) return this.captureWithWorkspaceLock();
|
|
@@ -1256,7 +1273,7 @@ export class UndoControllerImpl implements UndoController {
|
|
|
1256
1273
|
}
|
|
1257
1274
|
const captureStartedAt = performance.now();
|
|
1258
1275
|
try {
|
|
1259
|
-
return await captureBaseline(baseline, onDiagnostics, onTiming);
|
|
1276
|
+
return await captureBaseline(baseline, onDiagnostics, onTiming, options);
|
|
1260
1277
|
} catch (error) {
|
|
1261
1278
|
this.recordUnsafeExit(error);
|
|
1262
1279
|
throw error;
|
|
@@ -1275,16 +1292,17 @@ export class UndoControllerImpl implements UndoController {
|
|
|
1275
1292
|
baseline: SnapshotManifest,
|
|
1276
1293
|
onDiagnostics?: (diagnostic: CaptureDiagnostic) => void,
|
|
1277
1294
|
onTiming?: (phase: string, durationMs: number) => void,
|
|
1295
|
+
options: CaptureBaselineOptions = {},
|
|
1278
1296
|
): Promise<SnapshotManifest> {
|
|
1279
1297
|
try {
|
|
1280
|
-
return await this.captureBaselineWithWorkspaceLock(baseline, onDiagnostics, onTiming);
|
|
1298
|
+
return await this.captureBaselineWithWorkspaceLock(baseline, onDiagnostics, onTiming, options);
|
|
1281
1299
|
} catch (error) {
|
|
1282
1300
|
rethrowOperationFailure(error);
|
|
1283
1301
|
if (!isTransientCaptureFailure(error)) throw error;
|
|
1284
1302
|
const retryStartedAt = performance.now();
|
|
1285
1303
|
await sleep(250);
|
|
1286
1304
|
onTiming?.("settled.captureRetryDelay", performance.now() - retryStartedAt);
|
|
1287
|
-
return await this.captureBaselineWithWorkspaceLock(baseline, onDiagnostics, onTiming);
|
|
1305
|
+
return await this.captureBaselineWithWorkspaceLock(baseline, onDiagnostics, onTiming, options);
|
|
1288
1306
|
}
|
|
1289
1307
|
}
|
|
1290
1308
|
|
package/src/durable-pack.ts
CHANGED
|
@@ -34,6 +34,12 @@ export interface DurablePackInput {
|
|
|
34
34
|
readonly opId: string;
|
|
35
35
|
readonly planDigest: string;
|
|
36
36
|
readonly entries: readonly DurablePackEntryInput[];
|
|
37
|
+
/**
|
|
38
|
+
* "transfer" 表示调用方放弃 `variants[].bytes` 的所有权(通常是刚从 store 读出的
|
|
39
|
+
* 独占 buffer);pack 直接引用这些 buffer,不再复制一份完整 payload。
|
|
40
|
+
* 默认 "copy" 保留对调用方之后改写输入的隔离。
|
|
41
|
+
*/
|
|
42
|
+
readonly payloadOwnership?: "copy" | "transfer";
|
|
37
43
|
}
|
|
38
44
|
|
|
39
45
|
export interface DurablePackPreparationOptions {
|
|
@@ -170,7 +176,7 @@ export async function createDurablePack(
|
|
|
170
176
|
const planDigest = input.planDigest;
|
|
171
177
|
if (opId !== journal.operationId) throw new Error("durable pack opId 与 mutation journal 不匹配");
|
|
172
178
|
assertDigest(planDigest, "planDigest");
|
|
173
|
-
const entries = canonicalEntries(input.entries);
|
|
179
|
+
const entries = canonicalEntries(input.entries, true, input.payloadOwnership ?? "copy");
|
|
174
180
|
reportDurablePackTiming(options, "pack.validate", validationStarted);
|
|
175
181
|
const encodingStarted = performance.now();
|
|
176
182
|
const payloads: Buffer[] = [];
|
|
@@ -218,10 +224,13 @@ export async function createDurablePack(
|
|
|
218
224
|
reportDurablePackTiming(options, "pack.encode", encodingStarted);
|
|
219
225
|
let writeCalls = 0;
|
|
220
226
|
let handle;
|
|
227
|
+
// 写入时同步累积 pack checksum:writeAll 按实际写出的顺序哈希全部 buffer,
|
|
228
|
+
// 发布后不再完整遍历一遍 payload。
|
|
229
|
+
const digest = createHash("sha256");
|
|
221
230
|
try {
|
|
222
231
|
const writeStarted = performance.now();
|
|
223
232
|
handle = await open(temporary, "wx", 0o600);
|
|
224
|
-
writeCalls = await writeAll(handle, [MAGIC, lengthBytes, headerBytes, ...payloads]);
|
|
233
|
+
writeCalls = await writeAll(handle, [MAGIC, lengthBytes, headerBytes, ...payloads], digest);
|
|
225
234
|
reportDurablePackTiming(options, "pack.write", writeStarted);
|
|
226
235
|
checkOperation();
|
|
227
236
|
const syncStarted = performance.now();
|
|
@@ -240,11 +249,6 @@ export async function createDurablePack(
|
|
|
240
249
|
throw error;
|
|
241
250
|
}
|
|
242
251
|
const checksumStarted = performance.now();
|
|
243
|
-
const digest = createHash("sha256");
|
|
244
|
-
digest.update(MAGIC);
|
|
245
|
-
digest.update(lengthBytes);
|
|
246
|
-
digest.update(headerBytes);
|
|
247
|
-
for (const payload of payloads) digest.update(payload);
|
|
248
252
|
const packChecksum = digest.digest("hex");
|
|
249
253
|
reportDurablePackTiming(options, "pack.checksum", checksumStarted);
|
|
250
254
|
const residentStarted = performance.now();
|
|
@@ -303,11 +307,13 @@ export async function loadDurablePack(
|
|
|
303
307
|
if (start < payloadStart || end > bytes.length) throw new Error("durable pack payload 越界");
|
|
304
308
|
const content = bytes.subarray(start, end);
|
|
305
309
|
if (checksum(content) !== variant.dataChecksum) throw new Error("durable pack payload checksum 不匹配");
|
|
310
|
+
// payload 共享同一次 readFile 的 buffer;`leaf()` 返回时再复制,
|
|
311
|
+
// 避免为每个 variant 额外保留一份完整 payload。
|
|
306
312
|
leaf = {
|
|
307
313
|
kind: "file",
|
|
308
314
|
fingerprint: variant.fingerprint,
|
|
309
315
|
mode: variant.mode!,
|
|
310
|
-
bytes:
|
|
316
|
+
bytes: content,
|
|
311
317
|
};
|
|
312
318
|
}
|
|
313
319
|
const semanticFingerprint = leaf.kind === "absent"
|
|
@@ -490,6 +496,7 @@ async function assertSameFileIdentity(original: string, artifact: string, logica
|
|
|
490
496
|
function canonicalEntries(
|
|
491
497
|
entries: readonly DurablePackEntryInput[],
|
|
492
498
|
validateSemanticFingerprints = true,
|
|
499
|
+
payloadOwnership: "copy" | "transfer" = "copy",
|
|
493
500
|
): DurablePackEntryInput[] {
|
|
494
501
|
const sorted = [...entries].sort((left, right) => left.path.localeCompare(right.path));
|
|
495
502
|
let previous: string | undefined;
|
|
@@ -503,8 +510,9 @@ function canonicalEntries(
|
|
|
503
510
|
if (entry.targetArtifact !== null) assertArtifact(entry.path, entry.targetArtifact, "target");
|
|
504
511
|
assertDigest(entry.sourceFingerprint, "sourceFingerprint");
|
|
505
512
|
if (entry.targetFingerprint !== null) assertDigest(entry.targetFingerprint, "targetFingerprint");
|
|
506
|
-
//
|
|
507
|
-
|
|
513
|
+
// 默认在校验及任何回调、await 之前复制 payload;调用方之后改写输入不影响 pack。
|
|
514
|
+
// "transfer" 只用于内部刚读出的独占 buffer,避免再复制一份完整 payload。
|
|
515
|
+
const variants = entry.variants.map((variant): DurableLeafInput => variant.kind === "file" && payloadOwnership === "copy"
|
|
508
516
|
? { ...variant, bytes: new Uint8Array(variant.bytes) }
|
|
509
517
|
: { ...variant }).sort((left, right) => left.fingerprint.localeCompare(right.fingerprint));
|
|
510
518
|
const fingerprints = new Set<string>();
|
|
@@ -626,7 +634,11 @@ function durablePackFromOwnedEntries(
|
|
|
626
634
|
);
|
|
627
635
|
}
|
|
628
636
|
|
|
629
|
-
async function writeAll(
|
|
637
|
+
async function writeAll(
|
|
638
|
+
handle: FileHandle,
|
|
639
|
+
buffers: readonly Uint8Array[],
|
|
640
|
+
digest?: ReturnType<typeof createHash>,
|
|
641
|
+
): Promise<number> {
|
|
630
642
|
let calls = 0;
|
|
631
643
|
let batch: Uint8Array[] = [];
|
|
632
644
|
let batchBytes = 0;
|
|
@@ -642,12 +654,15 @@ async function writeAll(handle: FileHandle, buffers: readonly Uint8Array[]): Pro
|
|
|
642
654
|
}
|
|
643
655
|
remainingBytes -= bytesWritten;
|
|
644
656
|
// 短写可能停在任意 buffer 内,也可能跨过多个 buffer。
|
|
657
|
+
// digest 只累积实际写出的字节,顺序与文件内容一致。
|
|
645
658
|
let consumed = bytesWritten;
|
|
646
659
|
let index = 0;
|
|
647
660
|
while (index < remaining.length && consumed >= remaining[index]!.byteLength) {
|
|
661
|
+
digest?.update(remaining[index]!);
|
|
648
662
|
consumed -= remaining[index]!.byteLength;
|
|
649
663
|
index += 1;
|
|
650
664
|
}
|
|
665
|
+
if (consumed > 0) digest?.update(remaining[index]!.subarray(0, consumed));
|
|
651
666
|
remaining = remaining.slice(index);
|
|
652
667
|
if (consumed > 0) remaining[0] = remaining[0]!.subarray(consumed);
|
|
653
668
|
}
|
|
@@ -52,7 +52,11 @@ export class NativeDirectoryScanner implements NativeDirectoryScanPort {
|
|
|
52
52
|
private readonly executable: string | undefined;
|
|
53
53
|
private capabilities: Promise<ReadonlySet<string>> | undefined;
|
|
54
54
|
private cacheDirectory: Promise<string> | undefined;
|
|
55
|
-
private resident: {
|
|
55
|
+
private resident: {
|
|
56
|
+
readonly workspaceRoot: string;
|
|
57
|
+
readonly cachePath: string | undefined;
|
|
58
|
+
readonly client: ResidentScanClient;
|
|
59
|
+
} | undefined;
|
|
56
60
|
private prewarmInFlight: { readonly workspaceRoot: string; readonly promise: Promise<void> } | undefined;
|
|
57
61
|
|
|
58
62
|
constructor(executable = nativeExecutable()) {
|
|
@@ -86,23 +90,29 @@ export class NativeDirectoryScanner implements NativeDirectoryScanPort {
|
|
|
86
90
|
try {
|
|
87
91
|
const capabilities = await this.probeCapabilities(requestDirectory);
|
|
88
92
|
if (!capabilities.has("scan-serve-v1")) return;
|
|
89
|
-
const
|
|
93
|
+
const cachePath = capabilities.has("scan-serve-cache-v1")
|
|
94
|
+
? await this.directoryCachePath(workspaceRoot, temporaryRoot, requestDirectory)
|
|
95
|
+
: undefined;
|
|
96
|
+
const client = this.ensureResident(workspaceRoot, cachePath);
|
|
90
97
|
if (client !== undefined) await client.prewarm();
|
|
91
98
|
} finally {
|
|
92
99
|
await rm(requestDirectory, { recursive: true, force: true }).catch(() => {});
|
|
93
100
|
}
|
|
94
101
|
}
|
|
95
102
|
|
|
96
|
-
private ensureResident(workspaceRoot: string): ResidentScanClient | undefined {
|
|
103
|
+
private ensureResident(workspaceRoot: string, cachePath: string | undefined): ResidentScanClient | undefined {
|
|
97
104
|
if (this.executable === undefined) return undefined;
|
|
98
|
-
if (this.resident?.workspaceRoot === workspaceRoot
|
|
105
|
+
if (this.resident?.workspaceRoot === workspaceRoot && this.resident.cachePath === cachePath) {
|
|
106
|
+
return this.resident.client;
|
|
107
|
+
}
|
|
99
108
|
if (this.resident !== undefined) void this.resident.client.close();
|
|
100
109
|
const client = new ResidentScanClient(
|
|
101
110
|
this.executable,
|
|
102
111
|
workspaceRoot,
|
|
103
112
|
configuredTimeout("PI_UNDO_OPERATION_TIMEOUT_MS", 300_000),
|
|
113
|
+
cachePath,
|
|
104
114
|
);
|
|
105
|
-
this.resident = { workspaceRoot, client };
|
|
115
|
+
this.resident = { workspaceRoot, cachePath, client };
|
|
106
116
|
return client;
|
|
107
117
|
}
|
|
108
118
|
|
|
@@ -122,8 +132,11 @@ export class NativeDirectoryScanner implements NativeDirectoryScanPort {
|
|
|
122
132
|
: capabilities.has("scan-directories-v1") ? "v1" : undefined;
|
|
123
133
|
if (version === undefined) return undefined;
|
|
124
134
|
|
|
135
|
+
// 旧 helper 不认识握手里的 cachePath(deny_unknown_fields);能力未声明时不发送。
|
|
136
|
+
const cachePath = version === "v2" && capabilities.has("scan-serve-cache-v1")
|
|
137
|
+
? await this.directoryCachePath(workspaceRoot, temporaryRoot, requestDirectory) : undefined;
|
|
125
138
|
if (capabilities.has("scan-serve-v1")) {
|
|
126
|
-
const client = this.ensureResident(workspaceRoot);
|
|
139
|
+
const client = this.ensureResident(workspaceRoot, cachePath);
|
|
127
140
|
if (client !== undefined) {
|
|
128
141
|
const response = await client.scan();
|
|
129
142
|
checkOperation();
|
|
@@ -138,8 +151,6 @@ export class NativeDirectoryScanner implements NativeDirectoryScanPort {
|
|
|
138
151
|
|
|
139
152
|
const budget = operationProcessOptions(configuredTimeout("PI_UNDO_OPERATION_TIMEOUT_MS", 300_000));
|
|
140
153
|
const requestPath = join(requestDirectory, "request.json");
|
|
141
|
-
const cachePath = version === "v2"
|
|
142
|
-
? await this.directoryCachePath(workspaceRoot, temporaryRoot, requestDirectory) : undefined;
|
|
143
154
|
await writeFile(requestPath, JSON.stringify({
|
|
144
155
|
schemaVersion: version === "v2" ? 2 : 1,
|
|
145
156
|
workspaceRoot,
|
package/src/path-safety.ts
CHANGED
|
@@ -174,3 +174,45 @@ function hasErrorCode(error: unknown, code: string): error is NodeJS.ErrnoExcept
|
|
|
174
174
|
function fail(code: PathSafetyErrorCode, message: string): never {
|
|
175
175
|
throw new PathSafetyError(code, message);
|
|
176
176
|
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* mutation 影响判定索引。
|
|
180
|
+
*
|
|
181
|
+
* 原实现每次判定都重新展开 `[...deletePaths, ...writePaths]` 并线性扫描;
|
|
182
|
+
* 10k 目标路径 × 10k mutation 时是 O(n²)。这里把精确集合、mutation 祖先集合和
|
|
183
|
+
* 排序后的 mutation 列表一次建好,单次判定近 O(log n)。
|
|
184
|
+
* 语义与 `mutationPath === path || mutationPath.startsWith(path + "/") || path.startsWith(mutationPath + "/")` 完全一致。
|
|
185
|
+
*/
|
|
186
|
+
export class MutationPathIndex {
|
|
187
|
+
private readonly exact: ReadonlySet<string>;
|
|
188
|
+
private readonly sorted: readonly string[];
|
|
189
|
+
|
|
190
|
+
constructor(deletePaths: readonly string[], writePaths: readonly string[]) {
|
|
191
|
+
this.exact = new Set([...deletePaths, ...writePaths]);
|
|
192
|
+
this.sorted = [...this.exact].sort();
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
affects(path: string): boolean {
|
|
196
|
+
// mutation 是 path 本身,或是 path 内的后代。
|
|
197
|
+
if (this.exact.has(path) || sortedStartsWith(this.sorted, `${path}/`)) return true;
|
|
198
|
+
// path 位于某个 mutation 目录内:path 的严格祖先中存在 mutation。
|
|
199
|
+
let separator = path.lastIndexOf("/");
|
|
200
|
+
while (separator > 0) {
|
|
201
|
+
if (this.exact.has(path.slice(0, separator))) return true;
|
|
202
|
+
separator = path.lastIndexOf("/", separator - 1);
|
|
203
|
+
}
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** 排序数组中第一个以 prefix 开头的项;供 ignored proof 与 mutation 索引复用。 */
|
|
209
|
+
export function sortedStartsWith(sorted: readonly string[], prefix: string): boolean {
|
|
210
|
+
let low = 0;
|
|
211
|
+
let high = sorted.length;
|
|
212
|
+
while (low < high) {
|
|
213
|
+
const middle = (low + high) >>> 1;
|
|
214
|
+
if (sorted[middle]! < prefix) low = middle + 1;
|
|
215
|
+
else high = middle;
|
|
216
|
+
}
|
|
217
|
+
return low < sorted.length && sorted[low]!.startsWith(prefix);
|
|
218
|
+
}
|
package/src/pi-runtime.ts
CHANGED
|
@@ -9,6 +9,7 @@ import type {
|
|
|
9
9
|
|
|
10
10
|
import {
|
|
11
11
|
UndoControllerImpl,
|
|
12
|
+
type CaptureBaselineOptions,
|
|
12
13
|
type ControllerDependencies,
|
|
13
14
|
type ControllerInitialState,
|
|
14
15
|
type SettlementDiagnostic,
|
|
@@ -72,6 +73,7 @@ async function initializePiUndoRuntime(context: ExtensionContext, pi: ExtensionA
|
|
|
72
73
|
baseline: SnapshotManifest,
|
|
73
74
|
onDiagnostics?: (diagnostic: CaptureDiagnostic) => void,
|
|
74
75
|
onTiming?: (phase: string, durationMs: number) => void,
|
|
76
|
+
options: CaptureBaselineOptions = {},
|
|
75
77
|
) => {
|
|
76
78
|
const discoveryStartedAt = performance.now();
|
|
77
79
|
let topology: RootTopology;
|
|
@@ -85,6 +87,7 @@ async function initializePiUndoRuntime(context: ExtensionContext, pi: ExtensionA
|
|
|
85
87
|
}
|
|
86
88
|
return store.captureBaseline(topology, baseline, undefined, {
|
|
87
89
|
topologyAlreadyValidated: true,
|
|
90
|
+
...(options.skipBaselineProbe === true ? { skipBaselineProbe: true } : {}),
|
|
88
91
|
// captureBaselineWithWorkspaceLock 已持有工作区独占锁,且此处刚完成 discover;
|
|
89
92
|
// post-capture topology 校验在锁内是冗余的,跳过以节省一次全量目录扫描。
|
|
90
93
|
skipPostTopologyCheck: true,
|
|
@@ -34,6 +34,8 @@ export class ResidentScanClient {
|
|
|
34
34
|
private readonly executable: string,
|
|
35
35
|
private readonly workspaceRoot: string,
|
|
36
36
|
private readonly timeoutMs: number,
|
|
37
|
+
/** 与一次性 helper 共享的持久目录缓存;serve 启动时读取,初始扫描后写回。 */
|
|
38
|
+
private readonly cachePath?: string,
|
|
37
39
|
) {}
|
|
38
40
|
|
|
39
41
|
/** 建立进程并完成首次全量扫描;失败静默,正式扫描会回退。 */
|
|
@@ -93,6 +95,7 @@ export class ResidentScanClient {
|
|
|
93
95
|
const handshakeLine = await this.requestLine(child, JSON.stringify({
|
|
94
96
|
schemaVersion: 1,
|
|
95
97
|
workspaceRoot: this.workspaceRoot,
|
|
98
|
+
...(this.cachePath === undefined ? {} : { cachePath: this.cachePath }),
|
|
96
99
|
}), timeoutMs);
|
|
97
100
|
const handshake = handshakeLine === undefined ? undefined : parseObjectOrUndefined(handshakeLine);
|
|
98
101
|
if (handshake?.ok !== true || handshake.ready !== true) {
|
|
@@ -211,9 +214,10 @@ export class ResidentScanClient {
|
|
|
211
214
|
private scheduleIdle(): void {
|
|
212
215
|
if (this.idleTimer !== undefined) clearTimeout(this.idleTimer);
|
|
213
216
|
if (this.closed) return;
|
|
217
|
+
// 与 Rust 端 600s 看门狗保持同一量级;大工作区冷启动比重启进程更贵。
|
|
214
218
|
this.idleTimer = setTimeout(() => {
|
|
215
219
|
void this.close();
|
|
216
|
-
},
|
|
220
|
+
}, 610_000);
|
|
217
221
|
this.idleTimer.unref?.();
|
|
218
222
|
}
|
|
219
223
|
|
package/src/restore-engine.ts
CHANGED
|
@@ -27,6 +27,7 @@ import { recoverPackedMutations } from "./packed-recovery.ts";
|
|
|
27
27
|
import type { ManifestId, RestorePath, SnapshotManifest, SnapshotRoot } from "./model.ts";
|
|
28
28
|
import {
|
|
29
29
|
assertNoSymlinkEscape,
|
|
30
|
+
MutationPathIndex,
|
|
30
31
|
relativeSafePath,
|
|
31
32
|
sortDeletePaths,
|
|
32
33
|
sortWritePaths,
|
|
@@ -473,6 +474,8 @@ export class RestoreEngine {
|
|
|
473
474
|
const pack = await createDurablePack(cacheJournal, {
|
|
474
475
|
opId: cacheOpId,
|
|
475
476
|
planDigest: plan.planDigest,
|
|
477
|
+
// durablePackEntries 的 blob 是本次操作独占的 store 读取结果,可转移所有权。
|
|
478
|
+
payloadOwnership: "transfer",
|
|
476
479
|
entries: await this.durablePackEntries(
|
|
477
480
|
current,
|
|
478
481
|
target,
|
|
@@ -1019,6 +1022,8 @@ export class RestoreEngine {
|
|
|
1019
1022
|
durablePack = await createDurablePack(options.mutationJournal, {
|
|
1020
1023
|
opId: options.opId,
|
|
1021
1024
|
planDigest: plan.planDigest,
|
|
1025
|
+
// 同上:这里的 entries 只属于本次 pack,写入后不再被其他调用方使用。
|
|
1026
|
+
payloadOwnership: "transfer",
|
|
1022
1027
|
entries: await this.durablePackEntries(current, target, currentPaths, targetPaths, plan, options.opId),
|
|
1023
1028
|
});
|
|
1024
1029
|
}
|
|
@@ -1072,8 +1077,9 @@ export class RestoreEngine {
|
|
|
1072
1077
|
} finally {
|
|
1073
1078
|
reportRestoreTiming(options.onTiming, "apply.prefetch", t2);
|
|
1074
1079
|
}
|
|
1080
|
+
const mutationIndex = new MutationPathIndex(plan.deletePaths, plan.writePaths);
|
|
1075
1081
|
const reusableTargetPaths = new Set(
|
|
1076
|
-
[...targetPaths.keys()].filter((path) => !
|
|
1082
|
+
[...targetPaths.keys()].filter((path) => !mutationIndex.affects(path)),
|
|
1077
1083
|
);
|
|
1078
1084
|
const t3 = performance.now();
|
|
1079
1085
|
const preflightPromise = this.verifyKnownState(
|
|
@@ -2322,16 +2328,6 @@ function entriesEquivalent(left: OwnedPath | undefined, right: OwnedPath): boole
|
|
|
2322
2328
|
return left !== undefined && sameEntry(left.entry, right.entry);
|
|
2323
2329
|
}
|
|
2324
2330
|
|
|
2325
|
-
function isPathAffectedByMutations(
|
|
2326
|
-
path: string,
|
|
2327
|
-
deletePaths: readonly string[],
|
|
2328
|
-
writePaths: readonly string[],
|
|
2329
|
-
): boolean {
|
|
2330
|
-
return [...deletePaths, ...writePaths].some((mutationPath) =>
|
|
2331
|
-
mutationPath === path || mutationPath.startsWith(`${path}/`) || path.startsWith(`${mutationPath}/`),
|
|
2332
|
-
);
|
|
2333
|
-
}
|
|
2334
|
-
|
|
2335
2331
|
function entryFingerprint(owned: OwnedPath): string {
|
|
2336
2332
|
const path = owned.absolutePath;
|
|
2337
2333
|
if (owned.entry.kind === "directory") {
|
|
@@ -2514,17 +2510,20 @@ class IgnoredProofIndex {
|
|
|
2514
2510
|
if (this.exact.size === 0) return false;
|
|
2515
2511
|
// 排序成本只在第一次目录判定时付出,纯文件计划完全不触发。
|
|
2516
2512
|
this.sortedPaths ??= [...this.exact].sort();
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2513
|
+
return sortedStartsWith(this.sortedPaths, `${path}/`);
|
|
2514
|
+
}
|
|
2515
|
+
}
|
|
2516
|
+
|
|
2517
|
+
/** 排序数组中第一个以 prefix 开头的项;相同逻辑供 ignored proof 与 mutation 索引复用。 */
|
|
2518
|
+
function sortedStartsWith(sorted: readonly string[], prefix: string): boolean {
|
|
2519
|
+
let low = 0;
|
|
2520
|
+
let high = sorted.length;
|
|
2521
|
+
while (low < high) {
|
|
2522
|
+
const middle = (low + high) >>> 1;
|
|
2523
|
+
if (sorted[middle]! < prefix) low = middle + 1;
|
|
2524
|
+
else high = middle;
|
|
2527
2525
|
}
|
|
2526
|
+
return low < sorted.length && sorted[low]!.startsWith(prefix);
|
|
2528
2527
|
}
|
|
2529
2528
|
|
|
2530
2529
|
function rootBoundaryDirectories(root: string): string[] {
|
package/src/snapshot-store.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { BigIntStats } from "node:fs";
|
|
2
|
-
import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, realpath, rm, stat } from "node:fs/promises";
|
|
2
|
+
import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, realpath, rm, stat, writeFile } from "node:fs/promises";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
5
5
|
|
|
@@ -56,6 +56,17 @@ const BLOB_BATCH_MAX_BYTES = 16 * 1024 * 1024;
|
|
|
56
56
|
const BLOB_BATCH_MAX_ENTRIES = process.platform === "win32" ? 256 : 2_048;
|
|
57
57
|
const RACY_CLEAN_WINDOW_NS = 2_000_000_000n;
|
|
58
58
|
const LEAF_CACHE_FILE = "leaf-cache.json";
|
|
59
|
+
// 私有 ODB 的 loose object 达到阈值后合并为 pack;阈值取得足够大,避免把高频 capture
|
|
60
|
+
// 变成高频 repack,同时让大改动后的 cat-file 查询尽快走 pack 索引。
|
|
61
|
+
const COMPACT_MIN_LOOSE_OBJECTS = 4_096;
|
|
62
|
+
const COMPACT_MIN_LOOSE_KIB = 256 * 1024;
|
|
63
|
+
// 阈值达到后不立即打包:刚写入的大批对象正是下一次 undo 要校验的对象,
|
|
64
|
+
// 立刻 repack 只会把成本提前,单次操作反而更慢。连续 5 分钟维持高压后才合并,
|
|
65
|
+
// 让打包摊销到后续多次 plan/verify 上。
|
|
66
|
+
const COMPACT_MIN_AGE_MS = 5 * 60 * 1_000;
|
|
67
|
+
const COMPACT_PENDING_FILE = "pi-undo-compact.json";
|
|
68
|
+
// `git count-objects -v` 在 2 万 loose object 的库上约 80-100 ms;不需要每次 settle 都付。
|
|
69
|
+
const COMPACT_CHECK_INTERVAL_MS = 60_000;
|
|
59
70
|
|
|
60
71
|
interface PinRecord {
|
|
61
72
|
readonly schemaVersion: 1;
|
|
@@ -64,6 +75,11 @@ interface PinRecord {
|
|
|
64
75
|
readonly updatedAt: string;
|
|
65
76
|
}
|
|
66
77
|
|
|
78
|
+
interface CompactPendingRecord {
|
|
79
|
+
readonly schemaVersion: 1;
|
|
80
|
+
readonly observedAt: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
67
83
|
interface StoreGcRecord {
|
|
68
84
|
readonly schemaVersion: 1;
|
|
69
85
|
readonly lastUsedAt: number;
|
|
@@ -225,6 +241,10 @@ export interface SnapshotStoreOptions {
|
|
|
225
241
|
readonly lock?: WorkspaceLock;
|
|
226
242
|
readonly clock?: () => number;
|
|
227
243
|
readonly nativeMetadata?: NativeMetadataPort;
|
|
244
|
+
/** 仅供测试或受控调优:0 禁用 loose object 合并。 */
|
|
245
|
+
readonly compactMinLooseObjects?: number;
|
|
246
|
+
/** 阈值持续达到多久后才真正合并;默认 5 分钟。 */
|
|
247
|
+
readonly compactMinAgeMs?: number;
|
|
228
248
|
}
|
|
229
249
|
|
|
230
250
|
export interface CaptureOptions {
|
|
@@ -234,6 +254,11 @@ export interface CaptureOptions {
|
|
|
234
254
|
* 因此校验窗口内出现的漂移依然会被拒绝。仅限调用方与本次枚举之间没有文件变更时使用。
|
|
235
255
|
*/
|
|
236
256
|
readonly topologyAlreadyValidated?: boolean;
|
|
257
|
+
/**
|
|
258
|
+
* 调用方已证明 baseline 一定不是当前状态时跳过 baseline 探测;完整 capture 仍会
|
|
259
|
+
* 重新建立对象、ignored proof 与 leaves 缓存。
|
|
260
|
+
*/
|
|
261
|
+
readonly skipBaselineProbe?: boolean;
|
|
237
262
|
/**
|
|
238
263
|
* 跳过捕获或枚举结束后的 topology 二次校验(safety-snapshot post 扫描)。
|
|
239
264
|
* 仅在调用方已持有工作区独占锁、且刚刚完成 discover 并确认 topology 未变化时传 true。
|
|
@@ -302,6 +327,9 @@ export class SnapshotStore {
|
|
|
302
327
|
private readonly lock: WorkspaceLock;
|
|
303
328
|
private readonly clock: () => number;
|
|
304
329
|
private readonly nativeMetadata: NativeMetadataPort;
|
|
330
|
+
private readonly compactMinLooseObjects: number;
|
|
331
|
+
private readonly compactMinAgeMs: number;
|
|
332
|
+
private lastCompactCheckAt: number | undefined;
|
|
305
333
|
private readonly manifestLocations = new Map<string, string>();
|
|
306
334
|
// Tree 与 blob 都由 object ID 内容寻址;缓存只复用已从私有 ODB 读取的不可变内容。
|
|
307
335
|
private readonly treeEntriesCache = new Map<string, Promise<CapturedTreeEntry[]>>();
|
|
@@ -323,6 +351,8 @@ export class SnapshotStore {
|
|
|
323
351
|
this.lock = options.lock ?? new WorkspaceLock();
|
|
324
352
|
this.clock = options.clock ?? Date.now;
|
|
325
353
|
this.nativeMetadata = options.nativeMetadata ?? new NativeMetadataInspector();
|
|
354
|
+
this.compactMinLooseObjects = options.compactMinLooseObjects ?? COMPACT_MIN_LOOSE_OBJECTS;
|
|
355
|
+
this.compactMinAgeMs = options.compactMinAgeMs ?? COMPACT_MIN_AGE_MS;
|
|
326
356
|
}
|
|
327
357
|
|
|
328
358
|
static supportsValidatedBlobBatch(store: SnapshotStore): boolean {
|
|
@@ -521,9 +551,13 @@ export class SnapshotStore {
|
|
|
521
551
|
throw new SnapshotStoreError("capture_failed", `broken root 不能静默进入 baseline 校验: ${brokenRoots.join(", ")}`);
|
|
522
552
|
}
|
|
523
553
|
const baselineValidationStartedAt = performance.now();
|
|
524
|
-
let baselineFresh
|
|
554
|
+
let baselineFresh = false;
|
|
525
555
|
try {
|
|
526
|
-
|
|
556
|
+
// skipBaselineProbe 只由调用方对“已确定失效”的 baseline 传入;此时探测必然失败,
|
|
557
|
+
// 跳过它可以把一次完整 collectVisibleLeaves + ignored proof 从结算路径上省掉。
|
|
558
|
+
if (options.skipBaselineProbe !== true) {
|
|
559
|
+
baselineFresh = await this.isBaselineFresh(topology, baseline, scope, options, diagnostic);
|
|
560
|
+
}
|
|
527
561
|
} finally {
|
|
528
562
|
publishCaptureTiming(options.onTiming, "settled.baselineValidation", baselineValidationStartedAt);
|
|
529
563
|
}
|
|
@@ -538,11 +572,23 @@ export class SnapshotStore {
|
|
|
538
572
|
// 已完成一次捕获前 topology 校验;完整回退仍保留捕获后的校验。
|
|
539
573
|
if (diagnostic !== undefined) diagnostic.strategy = "baseline-fallback";
|
|
540
574
|
const fullCaptureStartedAt = performance.now();
|
|
575
|
+
let captured: SnapshotManifest;
|
|
541
576
|
try {
|
|
542
|
-
|
|
577
|
+
captured = await this.captureLocked(
|
|
578
|
+
topology,
|
|
579
|
+
scope,
|
|
580
|
+
{ ...options, topologyAlreadyValidated: true, skipPostTopologyCheck: options.skipPostTopologyCheck },
|
|
581
|
+
diagnostic,
|
|
582
|
+
);
|
|
543
583
|
} finally {
|
|
544
584
|
publishCaptureTiming(options.onTiming, "settled.fullCapture", fullCaptureStartedAt);
|
|
545
585
|
}
|
|
586
|
+
// 完整捕获刚写入一批 loose object;结算路径已持有 store 与 workspace 锁,
|
|
587
|
+
// 是唯一可以在不打断恢复读取的情况下做维护的窗口。维护时长单独报告。
|
|
588
|
+
const compactStartedAt = performance.now();
|
|
589
|
+
await this.compactLooseObjectsLocked(topology);
|
|
590
|
+
publishCaptureTiming(options.onTiming, "settled.compact", compactStartedAt);
|
|
591
|
+
return captured;
|
|
546
592
|
} catch (error) {
|
|
547
593
|
if (error instanceof SnapshotStoreError) {
|
|
548
594
|
throw error;
|
|
@@ -551,6 +597,106 @@ export class SnapshotStore {
|
|
|
551
597
|
}
|
|
552
598
|
}
|
|
553
599
|
|
|
600
|
+
/**
|
|
601
|
+
* 把私有 ODB 的 loose object 合并为 pack。
|
|
602
|
+
*
|
|
603
|
+
* 私有仓库没有 refs,`git repack -a` 不会把 manifest 引用的对象视为可达,
|
|
604
|
+
* 因此这里直接按磁盘上的 loose object 列表调用 pack-objects,再 prune-packed。
|
|
605
|
+
* 维护失败不影响已经完成的 capture;取消/超时仍按操作契约向上传递。
|
|
606
|
+
*/
|
|
607
|
+
private async compactLooseObjectsLocked(topology: RootTopology): Promise<void> {
|
|
608
|
+
if (this.compactMinLooseObjects <= 0) return;
|
|
609
|
+
const now = this.clock();
|
|
610
|
+
if (this.lastCompactCheckAt !== undefined && now - this.lastCompactCheckAt < COMPACT_CHECK_INTERVAL_MS) return;
|
|
611
|
+
this.lastCompactCheckAt = now;
|
|
612
|
+
const storeDirectory = this.storeDirectory(topology);
|
|
613
|
+
let roots: string[];
|
|
614
|
+
try {
|
|
615
|
+
roots = await readdir(join(storeDirectory, "roots"));
|
|
616
|
+
} catch {
|
|
617
|
+
return;
|
|
618
|
+
}
|
|
619
|
+
for (const root of roots) {
|
|
620
|
+
try {
|
|
621
|
+
await this.compactGitDirectory(join(storeDirectory, "roots", root, "git"));
|
|
622
|
+
} catch {
|
|
623
|
+
// 维护是 best-effort:capture 已经完成,repack 失败/取消不影响本次结果,
|
|
624
|
+
// 未合并的 loose object 会在下一次阈值检查时重试。
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
private async compactGitDirectory(gitDirectory: string): Promise<void> {
|
|
630
|
+
const environment = cleanGitEnvironment();
|
|
631
|
+
const output = await this.runGit(
|
|
632
|
+
["--git-dir", gitDirectory, "count-objects", "-v"],
|
|
633
|
+
{ env: environment },
|
|
634
|
+
);
|
|
635
|
+
const looseCount = parseCountObjectsField(output, "count");
|
|
636
|
+
const looseKiB = parseCountObjectsField(output, "size");
|
|
637
|
+
const pendingPath = join(gitDirectory, COMPACT_PENDING_FILE);
|
|
638
|
+
if (looseCount < this.compactMinLooseObjects && looseKiB < COMPACT_MIN_LOOSE_KIB) {
|
|
639
|
+
// 已经低于阈值(例如 store 被清理):撤销待合并状态。
|
|
640
|
+
await rm(pendingPath, { force: true }).catch(() => {});
|
|
641
|
+
return;
|
|
642
|
+
}
|
|
643
|
+
const pending = await readCompactPending(pendingPath);
|
|
644
|
+
const now = this.clock();
|
|
645
|
+
if (pending === undefined) {
|
|
646
|
+
if (this.compactMinAgeMs <= 0) {
|
|
647
|
+
// 测试或显式配置:立即合并。
|
|
648
|
+
} else {
|
|
649
|
+
// 只是延迟标记;不 fsync,丢失最多让合并再等一个窗口。
|
|
650
|
+
await writeFile(
|
|
651
|
+
pendingPath,
|
|
652
|
+
JSON.stringify({ schemaVersion: 1, observedAt: now } satisfies CompactPendingRecord),
|
|
653
|
+
{ mode: 0o600 },
|
|
654
|
+
).catch(() => {});
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
} else if (now - pending.observedAt < this.compactMinAgeMs) {
|
|
658
|
+
return;
|
|
659
|
+
}
|
|
660
|
+
const objects = await this.listLooseObjectIds(gitDirectory);
|
|
661
|
+
if (objects.length === 0) {
|
|
662
|
+
await rm(pendingPath, { force: true }).catch(() => {});
|
|
663
|
+
return;
|
|
664
|
+
}
|
|
665
|
+
await this.runGit(
|
|
666
|
+
["--git-dir", gitDirectory, "pack-objects", "-q", join(gitDirectory, "objects", "pack", "pack")],
|
|
667
|
+
{ env: environment, stdin: `${objects.join("\n")}\n` },
|
|
668
|
+
);
|
|
669
|
+
await this.runGit(["--git-dir", gitDirectory, "prune-packed"], { env: environment });
|
|
670
|
+
await rm(pendingPath, { force: true }).catch(() => {});
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
private async listLooseObjectIds(gitDirectory: string): Promise<string[]> {
|
|
674
|
+
const objectsDirectory = join(gitDirectory, "objects");
|
|
675
|
+
const result: string[] = [];
|
|
676
|
+
let directories: string[];
|
|
677
|
+
try {
|
|
678
|
+
directories = await readdir(objectsDirectory);
|
|
679
|
+
} catch {
|
|
680
|
+
return result;
|
|
681
|
+
}
|
|
682
|
+
for (const directory of directories) {
|
|
683
|
+
if (!/^[0-9a-f]{2}$/.test(directory)) continue;
|
|
684
|
+
let files: string[];
|
|
685
|
+
try {
|
|
686
|
+
files = await readdir(join(objectsDirectory, directory));
|
|
687
|
+
} catch {
|
|
688
|
+
continue;
|
|
689
|
+
}
|
|
690
|
+
for (const file of files) {
|
|
691
|
+
// 兼容 SHA-1(40 字符)与 SHA-256(64 字符)对象目录。
|
|
692
|
+
if (/^[0-9a-f]{38}$/.test(file) || /^[0-9a-f]{62}$/.test(file)) {
|
|
693
|
+
result.push(`${directory}${file}`);
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
return result.sort();
|
|
698
|
+
}
|
|
699
|
+
|
|
554
700
|
private async isBaselineFresh(
|
|
555
701
|
topology: RootTopology,
|
|
556
702
|
baseline: SnapshotManifest,
|
|
@@ -1024,11 +1170,15 @@ export class SnapshotStore {
|
|
|
1024
1170
|
pending.push({ gitDirectory, objectId: request.blobId });
|
|
1025
1171
|
}
|
|
1026
1172
|
const bytesById = new Map<string, Uint8Array>();
|
|
1027
|
-
|
|
1173
|
+
// 同一 ODB 的对象 ID 去重必须用 Set:10k 个唯一 blob 时 Array.includes 是 O(n²)。
|
|
1174
|
+
const byGitDirectory = new Map<string, Set<string>>();
|
|
1028
1175
|
for (const { gitDirectory, objectId } of pending) {
|
|
1029
|
-
|
|
1030
|
-
if (
|
|
1031
|
-
|
|
1176
|
+
let ids = byGitDirectory.get(gitDirectory);
|
|
1177
|
+
if (ids === undefined) {
|
|
1178
|
+
ids = new Set<string>();
|
|
1179
|
+
byGitDirectory.set(gitDirectory, ids);
|
|
1180
|
+
}
|
|
1181
|
+
ids.add(objectId);
|
|
1032
1182
|
}
|
|
1033
1183
|
for (const [gitDirectory, objectIds] of byGitDirectory) {
|
|
1034
1184
|
for (const objectId of objectIds) {
|
|
@@ -1036,7 +1186,7 @@ export class SnapshotStore {
|
|
|
1036
1186
|
if (cached === undefined) continue;
|
|
1037
1187
|
bytesById.set(objectId, await cached.promise);
|
|
1038
1188
|
}
|
|
1039
|
-
const missing = objectIds.filter((objectId) => !bytesById.has(objectId));
|
|
1189
|
+
const missing = [...objectIds].filter((objectId) => !bytesById.has(objectId));
|
|
1040
1190
|
if (missing.length === 0) continue;
|
|
1041
1191
|
// batch 头部自带长度,无需预先知道 size;按响应自身长度解析后逐条校验。
|
|
1042
1192
|
const loaded = parseBatchBlobOutputByOrder(
|
|
@@ -2828,6 +2978,27 @@ function parseObjectIdLines(output: string, expectedCount: number): string[] {
|
|
|
2828
2978
|
return lines;
|
|
2829
2979
|
}
|
|
2830
2980
|
|
|
2981
|
+
async function readCompactPending(path: string): Promise<CompactPendingRecord | undefined> {
|
|
2982
|
+
try {
|
|
2983
|
+
const value: unknown = JSON.parse(await readFile(path, "utf8"));
|
|
2984
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
|
|
2985
|
+
const record = value as Record<string, unknown>;
|
|
2986
|
+
if (record.schemaVersion !== 1 || typeof record.observedAt !== "number" ||
|
|
2987
|
+
!Number.isSafeInteger(record.observedAt)) return undefined;
|
|
2988
|
+
return { schemaVersion: 1, observedAt: record.observedAt };
|
|
2989
|
+
} catch {
|
|
2990
|
+
return undefined;
|
|
2991
|
+
}
|
|
2992
|
+
}
|
|
2993
|
+
|
|
2994
|
+
/** 解析 `git count-objects -v` 的整数字段;缺失或非法值按 0 处理(维护是 best-effort)。 */
|
|
2995
|
+
function parseCountObjectsField(output: string, field: string): number {
|
|
2996
|
+
const match = new RegExp(`^${field}: (\\d+)$`, "m").exec(output);
|
|
2997
|
+
if (match === null) return 0;
|
|
2998
|
+
const value = Number(match[1]);
|
|
2999
|
+
return Number.isSafeInteger(value) ? value : 0;
|
|
3000
|
+
}
|
|
3001
|
+
|
|
2831
3002
|
function treeDirectoryPaths(entries: readonly CapturedTreeEntry[]): string[] {
|
|
2832
3003
|
const directories: string[] = [];
|
|
2833
3004
|
let previousParts: readonly string[] = [];
|