@indigoai-us/hq-cloud 6.12.5 → 6.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +35 -2
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +49 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +39 -4
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +55 -0
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/share.js +15 -0
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +100 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +87 -3
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +204 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal.d.ts +8 -2
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +32 -1
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +34 -1
- package/dist/journal.test.js.map +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +7 -3
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +12 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/types.d.ts +11 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +61 -0
- package/src/bin/sync-runner.ts +57 -14
- package/src/cli/reindex.test.ts +60 -0
- package/src/cli/reindex.ts +47 -8
- package/src/cli/share.test.ts +119 -0
- package/src/cli/share.ts +21 -1
- package/src/cli/sync.test.ts +256 -0
- package/src/cli/sync.ts +97 -3
- package/src/journal.test.ts +44 -0
- package/src/journal.ts +37 -2
- package/src/skill-telemetry.test.ts +24 -0
- package/src/skill-telemetry.ts +6 -3
- package/src/types.ts +11 -1
package/src/cli/share.test.ts
CHANGED
|
@@ -2470,6 +2470,61 @@ describe("share", () => {
|
|
|
2470
2470
|
expect(journal.files["core/policies/old.md"]).toBeUndefined();
|
|
2471
2471
|
});
|
|
2472
2472
|
|
|
2473
|
+
it("currency-gated: propagates the S3 delete for a local-delete tombstone (pull-leg producer hand-off)", async () => {
|
|
2474
|
+
// The pull-leg producer stamped a persisting `local-delete` tombstone for a
|
|
2475
|
+
// file the user deleted (no respawn). The cross-leg hand-off: on the next
|
|
2476
|
+
// push the S3 object must actually be DELETED and the entry dropped, so the
|
|
2477
|
+
// deletion propagates to the cloud + every other surface.
|
|
2478
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2479
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2480
|
+
const journalPath = path.join(stateDir, "sync-journal.acme.json");
|
|
2481
|
+
fs.writeFileSync(
|
|
2482
|
+
journalPath,
|
|
2483
|
+
JSON.stringify({
|
|
2484
|
+
version: "2",
|
|
2485
|
+
lastSync: new Date().toISOString(),
|
|
2486
|
+
files: {
|
|
2487
|
+
"docs/deleted.md": {
|
|
2488
|
+
hash: "h",
|
|
2489
|
+
size: 100,
|
|
2490
|
+
syncedAt: new Date().toISOString(),
|
|
2491
|
+
direction: "down",
|
|
2492
|
+
remoteEtag: "abc123",
|
|
2493
|
+
removedAt: new Date().toISOString(),
|
|
2494
|
+
removedReason: "local-delete",
|
|
2495
|
+
},
|
|
2496
|
+
},
|
|
2497
|
+
pulls: [],
|
|
2498
|
+
}),
|
|
2499
|
+
);
|
|
2500
|
+
|
|
2501
|
+
// HEAD returns the same etag — this device had the current version, so the
|
|
2502
|
+
// currency-gated delete is safe to propagate.
|
|
2503
|
+
vi.mocked(headRemoteFile).mockResolvedValueOnce({
|
|
2504
|
+
lastModified: new Date(),
|
|
2505
|
+
etag: '"abc123"',
|
|
2506
|
+
size: 100,
|
|
2507
|
+
});
|
|
2508
|
+
|
|
2509
|
+
const result = await share({
|
|
2510
|
+
paths: [companyRoot],
|
|
2511
|
+
company: "acme",
|
|
2512
|
+
vaultConfig: mockConfig,
|
|
2513
|
+
hqRoot: tmpDir,
|
|
2514
|
+
skipUnchanged: true,
|
|
2515
|
+
propagateDeletes: true,
|
|
2516
|
+
propagateDeletePolicy: "currency-gated",
|
|
2517
|
+
});
|
|
2518
|
+
|
|
2519
|
+
expect(result.filesDeleted).toBe(1);
|
|
2520
|
+
expect(deleteRemoteFile).toHaveBeenCalledWith(
|
|
2521
|
+
expect.anything(),
|
|
2522
|
+
"docs/deleted.md",
|
|
2523
|
+
);
|
|
2524
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
2525
|
+
expect(journal.files["docs/deleted.md"]).toBeUndefined();
|
|
2526
|
+
});
|
|
2527
|
+
|
|
2473
2528
|
it("currency-gated: refuses delete + emits stale-etag event when remote moved since last sync", async () => {
|
|
2474
2529
|
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2475
2530
|
fs.mkdirSync(companyRoot, { recursive: true });
|
|
@@ -2531,6 +2586,70 @@ describe("share", () => {
|
|
|
2531
2586
|
expect(result.filesTombstoned).toBe(0);
|
|
2532
2587
|
});
|
|
2533
2588
|
|
|
2589
|
+
it("currency-gated: refuses delete for a divergent-local entry (journal-honesty — never delete a remote the local never genuinely matched)", async () => {
|
|
2590
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2591
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2592
|
+
// A keep-conflict recorded the REMOTE etag (so the conflict can't re-fire —
|
|
2593
|
+
// the #137 invariant) but the kept LOCAL content DIVERGED from that remote
|
|
2594
|
+
// (`localDiverges:true`). The local copy was then deleted. The currency HEAD
|
|
2595
|
+
// would MATCH the recorded etag — the trap — yet the etag never reflected a
|
|
2596
|
+
// genuine local==remote sync, so propagating the delete would destroy the
|
|
2597
|
+
// (different) remote version. Must be REFUSED, not propagated.
|
|
2598
|
+
const journalPath = path.join(stateDir, "sync-journal.acme.json");
|
|
2599
|
+
fs.writeFileSync(
|
|
2600
|
+
journalPath,
|
|
2601
|
+
JSON.stringify({
|
|
2602
|
+
version: "2",
|
|
2603
|
+
lastSync: new Date().toISOString(),
|
|
2604
|
+
files: {
|
|
2605
|
+
"core/policies/kept.md": {
|
|
2606
|
+
hash: "local-divergent-hash",
|
|
2607
|
+
size: 100,
|
|
2608
|
+
syncedAt: new Date().toISOString(),
|
|
2609
|
+
direction: "down",
|
|
2610
|
+
remoteEtag: "abc123",
|
|
2611
|
+
localDiverges: true,
|
|
2612
|
+
},
|
|
2613
|
+
},
|
|
2614
|
+
pulls: [],
|
|
2615
|
+
}),
|
|
2616
|
+
);
|
|
2617
|
+
|
|
2618
|
+
// Deliberately NO HEAD mock: the divergence guard short-circuits BEFORE the
|
|
2619
|
+
// currency HEAD, so `headRemoteFile` is never called for this entry — the
|
|
2620
|
+
// refusal costs no network round-trip (asserted below).
|
|
2621
|
+
const events: Array<{ type: string; path?: string; reason?: string }> = [];
|
|
2622
|
+
const result = await share({
|
|
2623
|
+
paths: [companyRoot],
|
|
2624
|
+
company: "acme",
|
|
2625
|
+
vaultConfig: mockConfig,
|
|
2626
|
+
hqRoot: tmpDir,
|
|
2627
|
+
skipUnchanged: true,
|
|
2628
|
+
propagateDeletes: true,
|
|
2629
|
+
propagateDeletePolicy: "currency-gated",
|
|
2630
|
+
onEvent: (e) => events.push(e as { type: string }),
|
|
2631
|
+
});
|
|
2632
|
+
|
|
2633
|
+
// No remote DELETE — the recorded etag never reflected a genuine sync.
|
|
2634
|
+
expect(result.filesDeleted).toBe(0);
|
|
2635
|
+
expect(deleteRemoteFile).not.toHaveBeenCalledWith(
|
|
2636
|
+
expect.anything(),
|
|
2637
|
+
"core/policies/kept.md",
|
|
2638
|
+
);
|
|
2639
|
+
// Refusal short-circuits before the currency HEAD — no network needed.
|
|
2640
|
+
expect(headRemoteFile).not.toHaveBeenCalled();
|
|
2641
|
+
// Journal entry preserved, not silently dropped.
|
|
2642
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
2643
|
+
expect(journal.files["core/policies/kept.md"]).toBeDefined();
|
|
2644
|
+
// Surfaced as a refusal with the divergent-local reason.
|
|
2645
|
+
const refused = events.find((e) => e.type === "delete-refused-stale-etag");
|
|
2646
|
+
expect(refused).toMatchObject({
|
|
2647
|
+
type: "delete-refused-stale-etag",
|
|
2648
|
+
path: "core/policies/kept.md",
|
|
2649
|
+
reason: "divergent-local",
|
|
2650
|
+
});
|
|
2651
|
+
});
|
|
2652
|
+
|
|
2534
2653
|
it("currency-gated: tombstones journal entry when remote returns 404 (out-of-band cleanup)", async () => {
|
|
2535
2654
|
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2536
2655
|
fs.mkdirSync(companyRoot, { recursive: true });
|
package/src/cli/share.ts
CHANGED
|
@@ -1987,7 +1987,11 @@ function resolveDeleteScopeRoots(
|
|
|
1987
1987
|
* tracked. `journalEtag` and `remoteEtag` are
|
|
1988
1988
|
* placeholder sentinels — do not display as ETags.
|
|
1989
1989
|
*/
|
|
1990
|
-
type RefusedStaleReason =
|
|
1990
|
+
type RefusedStaleReason =
|
|
1991
|
+
| "stale-etag"
|
|
1992
|
+
| "legacy-no-etag"
|
|
1993
|
+
| "bulk-asymmetry"
|
|
1994
|
+
| "divergent-local";
|
|
1991
1995
|
|
|
1992
1996
|
/**
|
|
1993
1997
|
* Bulk-asymmetry circuit-breaker — refuses to convert a suspiciously-large
|
|
@@ -2241,6 +2245,22 @@ async function computeDeletePlan(
|
|
|
2241
2245
|
// documentation of the policy-side intent.
|
|
2242
2246
|
if (isEphemeralPath(relativeKey)) continue;
|
|
2243
2247
|
|
|
2248
|
+
// Journal-honesty guard: an entry flagged `localDiverges` recorded the
|
|
2249
|
+
// remote etag only to silence a re-fired conflict (#137) — the local copy
|
|
2250
|
+
// NEVER matched that remote. Its currency would falsely "match" on HEAD, so
|
|
2251
|
+
// propagating a delete would destroy the divergent remote version. Refuse it
|
|
2252
|
+
// regardless of policy, and do NOT count it toward the bulk-asymmetry ratio
|
|
2253
|
+
// (it isn't a genuine mirror-loss candidate). Cleared by a real download.
|
|
2254
|
+
if (entry.localDiverges) {
|
|
2255
|
+
plan.refusedStale.push({
|
|
2256
|
+
key: relativeKey,
|
|
2257
|
+
journalEtag: entry.remoteEtag ?? "<divergent-local>",
|
|
2258
|
+
remoteEtag: "<not-checked>",
|
|
2259
|
+
reason: "divergent-local",
|
|
2260
|
+
});
|
|
2261
|
+
continue;
|
|
2262
|
+
}
|
|
2263
|
+
|
|
2244
2264
|
if (policy === "all") {
|
|
2245
2265
|
// policy:"all" is the explicit-opt-out emergency-reconcile mode; the
|
|
2246
2266
|
// bulk-asymmetry guard skips this branch (caller asserted intent).
|
package/src/cli/sync.test.ts
CHANGED
|
@@ -279,6 +279,76 @@ describe("sync", () => {
|
|
|
279
279
|
expect(fs.readFileSync(path.join(companyDocs, "handoff.md"), "utf-8")).toBe("local version");
|
|
280
280
|
});
|
|
281
281
|
|
|
282
|
+
it("conflict keep marks the entry localDiverges (journal-honesty: kept copy never matched the recorded remote etag)", async () => {
|
|
283
|
+
const companyDocs = path.join(tmpDir, "companies", "acme", "docs");
|
|
284
|
+
fs.mkdirSync(companyDocs, { recursive: true });
|
|
285
|
+
fs.writeFileSync(path.join(companyDocs, "handoff.md"), "local version");
|
|
286
|
+
fs.writeFileSync(
|
|
287
|
+
journalPath,
|
|
288
|
+
JSON.stringify({
|
|
289
|
+
version: "1",
|
|
290
|
+
lastSync: new Date().toISOString(),
|
|
291
|
+
files: {
|
|
292
|
+
"docs/handoff.md": {
|
|
293
|
+
hash: "old-hash-from-last-sync",
|
|
294
|
+
size: 20,
|
|
295
|
+
syncedAt: new Date(Date.now() - 3600000).toISOString(),
|
|
296
|
+
direction: "down",
|
|
297
|
+
},
|
|
298
|
+
},
|
|
299
|
+
}),
|
|
300
|
+
);
|
|
301
|
+
|
|
302
|
+
const result = await sync({
|
|
303
|
+
company: "acme",
|
|
304
|
+
onConflict: "keep",
|
|
305
|
+
vaultConfig: mockConfig,
|
|
306
|
+
hqRoot: tmpDir,
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
expect(result.conflicts).toBe(1);
|
|
310
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
311
|
+
// remoteEtag still recorded so the conflict can't re-fire (#137 invariant)…
|
|
312
|
+
expect(journal.files["docs/handoff.md"].remoteEtag).toBeTruthy();
|
|
313
|
+
// …but the kept-local copy is flagged divergent so a later currency-gated
|
|
314
|
+
// delete cannot fire on a journal that lies about local==remote.
|
|
315
|
+
expect(journal.files["docs/handoff.md"].localDiverges).toBe(true);
|
|
316
|
+
});
|
|
317
|
+
|
|
318
|
+
it("a genuine download clears a stale localDiverges flag (entry is now truly local==remote)", async () => {
|
|
319
|
+
const companyDocs = path.join(tmpDir, "companies", "acme", "docs");
|
|
320
|
+
fs.mkdirSync(companyDocs, { recursive: true });
|
|
321
|
+
fs.writeFileSync(
|
|
322
|
+
journalPath,
|
|
323
|
+
JSON.stringify({
|
|
324
|
+
version: "2",
|
|
325
|
+
lastSync: new Date().toISOString(),
|
|
326
|
+
files: {
|
|
327
|
+
"docs/handoff.md": {
|
|
328
|
+
hash: "old-divergent-hash",
|
|
329
|
+
size: 20,
|
|
330
|
+
syncedAt: new Date(Date.now() - 3600000).toISOString(),
|
|
331
|
+
direction: "down",
|
|
332
|
+
remoteEtag: "old-etag",
|
|
333
|
+
localDiverges: true,
|
|
334
|
+
},
|
|
335
|
+
},
|
|
336
|
+
pulls: [],
|
|
337
|
+
}),
|
|
338
|
+
);
|
|
339
|
+
// Remote advanced to a new etag; local absent → genuine download lands the
|
|
340
|
+
// remote bytes, so local==remote now and the divergence flag must clear.
|
|
341
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
342
|
+
{ key: "docs/handoff.md", size: 42, lastModified: new Date(), etag: '"new-etag"' },
|
|
343
|
+
]);
|
|
344
|
+
|
|
345
|
+
await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
|
|
346
|
+
|
|
347
|
+
expect(fs.existsSync(path.join(companyDocs, "handoff.md"))).toBe(true);
|
|
348
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
349
|
+
expect(journal.files["docs/handoff.md"]?.localDiverges).toBeFalsy();
|
|
350
|
+
});
|
|
351
|
+
|
|
282
352
|
it("emits a conflict event with path + resolution on hash mismatch", async () => {
|
|
283
353
|
const companyDocs = path.join(tmpDir, "companies", "acme", "docs");
|
|
284
354
|
fs.mkdirSync(companyDocs, { recursive: true });
|
|
@@ -884,6 +954,192 @@ describe("sync", () => {
|
|
|
884
954
|
expect(journal.files[trackedKey]).toBeUndefined();
|
|
885
955
|
});
|
|
886
956
|
|
|
957
|
+
it("local journal tombstone suppresses drift-restore: a deleted-but-still-remote key is NOT re-downloaded (no respawn)", async () => {
|
|
958
|
+
// Repro of the intermittent delete-respawn: the user deleted a clean file,
|
|
959
|
+
// its delete has not (yet) cleared the S3 object, and the SERVER
|
|
960
|
+
// FILE_TOMBSTONE fetch is degraded (empty map — the logged failure mode).
|
|
961
|
+
// The only intent record is the LOCAL journal tombstone (`removedAt`).
|
|
962
|
+
// The drift-restore path must honor it and NOT re-download the key.
|
|
963
|
+
const key = "docs/handoff.md";
|
|
964
|
+
const localPath = path.join(tmpDir, "companies", "acme", key);
|
|
965
|
+
|
|
966
|
+
// Remote still lists the object, etag matching the journal, lastModified
|
|
967
|
+
// BEFORE the local delete — a stale resurrection of a deleted key, not a
|
|
968
|
+
// genuine remote re-create.
|
|
969
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
970
|
+
{
|
|
971
|
+
key,
|
|
972
|
+
size: 42,
|
|
973
|
+
lastModified: new Date(Date.now() - 2 * 86_400_000),
|
|
974
|
+
etag: '"abc123"',
|
|
975
|
+
},
|
|
976
|
+
]);
|
|
977
|
+
// Degraded-fetch case: server FILE_TOMBSTONE map is EMPTY.
|
|
978
|
+
setupFetchMock({ tombstones: [] });
|
|
979
|
+
|
|
980
|
+
// Local file ABSENT (user deleted it). Journal entry is a TOMBSTONE,
|
|
981
|
+
// recent enough to survive TTL GC.
|
|
982
|
+
fs.writeFileSync(
|
|
983
|
+
journalPath,
|
|
984
|
+
JSON.stringify({
|
|
985
|
+
version: "2",
|
|
986
|
+
lastSync: "2026-06-19T00:00:00.000Z",
|
|
987
|
+
files: {
|
|
988
|
+
[key]: {
|
|
989
|
+
hash: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
|
|
990
|
+
size: 42,
|
|
991
|
+
syncedAt: "2026-06-19T00:00:00.000Z",
|
|
992
|
+
direction: "down",
|
|
993
|
+
remoteEtag: "abc123",
|
|
994
|
+
removedAt: new Date().toISOString(),
|
|
995
|
+
removedReason: "local-delete",
|
|
996
|
+
},
|
|
997
|
+
},
|
|
998
|
+
pulls: [],
|
|
999
|
+
}),
|
|
1000
|
+
);
|
|
1001
|
+
|
|
1002
|
+
await sync({
|
|
1003
|
+
company: "acme",
|
|
1004
|
+
vaultConfig: mockConfig,
|
|
1005
|
+
hqRoot: tmpDir,
|
|
1006
|
+
onEvent: () => {},
|
|
1007
|
+
});
|
|
1008
|
+
|
|
1009
|
+
// The intentionally-deleted key must NOT be re-materialised.
|
|
1010
|
+
expect(s3Module.downloadFile).not.toHaveBeenCalledWith(
|
|
1011
|
+
expect.anything(),
|
|
1012
|
+
key,
|
|
1013
|
+
expect.anything(),
|
|
1014
|
+
);
|
|
1015
|
+
expect(fs.existsSync(localPath)).toBe(false);
|
|
1016
|
+
});
|
|
1017
|
+
|
|
1018
|
+
it("pull producer: a single clean/current/non-divergent missing file is tombstoned (intentional delete), not drift-restored", async () => {
|
|
1019
|
+
// The user deleted a genuinely-synced file. Its entry is clean (etag matches
|
|
1020
|
+
// the current remote — currency holds), not divergent. The pull leg must
|
|
1021
|
+
// treat the absence as an intentional delete: stamp a persisting
|
|
1022
|
+
// `local-delete` tombstone and SKIP the re-download — not resurrect it.
|
|
1023
|
+
const key = "docs/gone.md";
|
|
1024
|
+
const localPath = path.join(tmpDir, "companies", "acme", key);
|
|
1025
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
1026
|
+
{ key, size: 42, lastModified: new Date(), etag: '"cur123"' },
|
|
1027
|
+
]);
|
|
1028
|
+
setupFetchMock({ tombstones: [] });
|
|
1029
|
+
fs.writeFileSync(
|
|
1030
|
+
journalPath,
|
|
1031
|
+
JSON.stringify({
|
|
1032
|
+
version: "2",
|
|
1033
|
+
lastSync: new Date().toISOString(),
|
|
1034
|
+
files: {
|
|
1035
|
+
[key]: {
|
|
1036
|
+
hash: "h",
|
|
1037
|
+
size: 42,
|
|
1038
|
+
syncedAt: new Date().toISOString(),
|
|
1039
|
+
direction: "down",
|
|
1040
|
+
remoteEtag: "cur123",
|
|
1041
|
+
},
|
|
1042
|
+
},
|
|
1043
|
+
pulls: [],
|
|
1044
|
+
}),
|
|
1045
|
+
);
|
|
1046
|
+
|
|
1047
|
+
await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
|
|
1048
|
+
|
|
1049
|
+
expect(s3Module.downloadFile).not.toHaveBeenCalledWith(
|
|
1050
|
+
expect.anything(),
|
|
1051
|
+
key,
|
|
1052
|
+
expect.anything(),
|
|
1053
|
+
);
|
|
1054
|
+
expect(fs.existsSync(localPath)).toBe(false);
|
|
1055
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
1056
|
+
expect(journal.files[key]?.removedAt).toBeTruthy();
|
|
1057
|
+
expect(journal.files[key]?.removedReason).toBe("local-delete");
|
|
1058
|
+
});
|
|
1059
|
+
|
|
1060
|
+
it("pull producer: a BULK disappearance is drift-restored, NOT tombstoned (ridge-incident guard in the pull leg)", async () => {
|
|
1061
|
+
// 12 clean files vanish at once (corrupt mirror / bulk rm / checkout). The
|
|
1062
|
+
// bulk-asymmetry guard refuses to treat this as intent — every file is
|
|
1063
|
+
// healed (re-downloaded), none tombstoned.
|
|
1064
|
+
const keys = Array.from({ length: 12 }, (_, i) => `docs/f${i}.md`);
|
|
1065
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce(
|
|
1066
|
+
keys.map((k) => ({ key: k, size: 10, lastModified: new Date(), etag: `"e-${k}"` })),
|
|
1067
|
+
);
|
|
1068
|
+
setupFetchMock({ tombstones: [] });
|
|
1069
|
+
const files: Record<string, unknown> = {};
|
|
1070
|
+
for (const k of keys) {
|
|
1071
|
+
files[k] = {
|
|
1072
|
+
hash: "h",
|
|
1073
|
+
size: 10,
|
|
1074
|
+
syncedAt: new Date().toISOString(),
|
|
1075
|
+
direction: "down",
|
|
1076
|
+
remoteEtag: `e-${k}`,
|
|
1077
|
+
};
|
|
1078
|
+
}
|
|
1079
|
+
fs.writeFileSync(
|
|
1080
|
+
journalPath,
|
|
1081
|
+
JSON.stringify({
|
|
1082
|
+
version: "2",
|
|
1083
|
+
lastSync: new Date().toISOString(),
|
|
1084
|
+
files,
|
|
1085
|
+
pulls: [],
|
|
1086
|
+
}),
|
|
1087
|
+
);
|
|
1088
|
+
|
|
1089
|
+
await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
|
|
1090
|
+
|
|
1091
|
+
// All restored, none tombstoned.
|
|
1092
|
+
expect(s3Module.downloadFile).toHaveBeenCalledTimes(12);
|
|
1093
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
1094
|
+
for (const k of keys) {
|
|
1095
|
+
expect(journal.files[k]?.removedAt).toBeUndefined();
|
|
1096
|
+
}
|
|
1097
|
+
});
|
|
1098
|
+
|
|
1099
|
+
it("pull producer: a local-delete tombstone persists across pulls (not consumed/removed), so the delete stays propagatable", async () => {
|
|
1100
|
+
// After the intentional-delete tombstone is stamped, a later pull must NOT
|
|
1101
|
+
// drop the entry (the push leg still needs it to propagate the S3 delete).
|
|
1102
|
+
// It stays skipped + preserved until the push propagates or TTL GC.
|
|
1103
|
+
const key = "docs/gone.md";
|
|
1104
|
+
const localPath = path.join(tmpDir, "companies", "acme", key);
|
|
1105
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
1106
|
+
{ key, size: 42, lastModified: new Date(Date.now() - 2 * 86_400_000), etag: '"cur123"' },
|
|
1107
|
+
]);
|
|
1108
|
+
setupFetchMock({ tombstones: [] });
|
|
1109
|
+
fs.writeFileSync(
|
|
1110
|
+
journalPath,
|
|
1111
|
+
JSON.stringify({
|
|
1112
|
+
version: "2",
|
|
1113
|
+
lastSync: new Date().toISOString(),
|
|
1114
|
+
files: {
|
|
1115
|
+
[key]: {
|
|
1116
|
+
hash: "h",
|
|
1117
|
+
size: 42,
|
|
1118
|
+
syncedAt: new Date().toISOString(),
|
|
1119
|
+
direction: "down",
|
|
1120
|
+
remoteEtag: "cur123",
|
|
1121
|
+
removedAt: new Date().toISOString(),
|
|
1122
|
+
removedReason: "local-delete",
|
|
1123
|
+
},
|
|
1124
|
+
},
|
|
1125
|
+
pulls: [],
|
|
1126
|
+
}),
|
|
1127
|
+
);
|
|
1128
|
+
|
|
1129
|
+
await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
|
|
1130
|
+
|
|
1131
|
+
expect(s3Module.downloadFile).not.toHaveBeenCalledWith(
|
|
1132
|
+
expect.anything(),
|
|
1133
|
+
key,
|
|
1134
|
+
expect.anything(),
|
|
1135
|
+
);
|
|
1136
|
+
expect(fs.existsSync(localPath)).toBe(false);
|
|
1137
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
1138
|
+
// Entry PRESERVED as a tombstone, not removed.
|
|
1139
|
+
expect(journal.files[key]?.removedAt).toBeTruthy();
|
|
1140
|
+
expect(journal.files[key]?.removedReason).toBe("local-delete");
|
|
1141
|
+
});
|
|
1142
|
+
|
|
887
1143
|
it("aborts on --on-conflict abort", async () => {
|
|
888
1144
|
const companyDocs = path.join(tmpDir, "companies", "acme", "docs");
|
|
889
1145
|
fs.mkdirSync(companyDocs, { recursive: true });
|
package/src/cli/sync.ts
CHANGED
|
@@ -34,6 +34,8 @@ import {
|
|
|
34
34
|
normalizeEtag,
|
|
35
35
|
migrateToV2,
|
|
36
36
|
gcTombstones,
|
|
37
|
+
isTombstone,
|
|
38
|
+
tombstoneEntry,
|
|
37
39
|
lastPullRecord,
|
|
38
40
|
appendPullRecord,
|
|
39
41
|
generatePullId,
|
|
@@ -213,7 +215,7 @@ export type SyncProgressEvent =
|
|
|
213
215
|
path: string;
|
|
214
216
|
journalEtag: string;
|
|
215
217
|
remoteEtag: string;
|
|
216
|
-
reason: "stale-etag" | "legacy-no-etag" | "bulk-asymmetry";
|
|
218
|
+
reason: "stale-etag" | "legacy-no-etag" | "bulk-asymmetry" | "divergent-local";
|
|
217
219
|
}
|
|
218
220
|
| {
|
|
219
221
|
/**
|
|
@@ -1209,6 +1211,14 @@ async function executeConflictItem(
|
|
|
1209
1211
|
remoteFile.etag,
|
|
1210
1212
|
item.localMtime.getTime(),
|
|
1211
1213
|
);
|
|
1214
|
+
// Journal-honesty: we recorded the remote etag so this conflict can't
|
|
1215
|
+
// re-fire (#137), but the KEPT local copy diverges from that remote — it
|
|
1216
|
+
// never matched it. Flag the entry so the currency-gated delete planner
|
|
1217
|
+
// refuses to propagate a delete for it (its currency would falsely match on
|
|
1218
|
+
// HEAD, and the delete would destroy the divergent remote version). Any
|
|
1219
|
+
// genuine future download clears the flag by replacing the entry.
|
|
1220
|
+
const keptEntry = getEntry(run.journal, remoteFile.key);
|
|
1221
|
+
if (keptEntry) keptEntry.localDiverges = true;
|
|
1212
1222
|
return null;
|
|
1213
1223
|
}
|
|
1214
1224
|
|
|
@@ -1738,6 +1748,15 @@ interface PullPlan {
|
|
|
1738
1748
|
* caller (`sync()`) is responsible for emitting the resulting plan event
|
|
1739
1749
|
* before iterating `items`.
|
|
1740
1750
|
*/
|
|
1751
|
+
// Pull-leg intentional-delete bulk-asymmetry breaker. Mirrors the push leg's
|
|
1752
|
+
// BULK_ASYMMETRY_* (share.ts): an abnormally large fraction of clean files
|
|
1753
|
+
// vanishing at once is a corrupt mirror / bulk op / unmount (the ridge-incident
|
|
1754
|
+
// class), not intent — so at/above the threshold the pull leg drift-restores
|
|
1755
|
+
// every candidate instead of tombstoning it. Below it, a missing clean file is
|
|
1756
|
+
// an intentional delete.
|
|
1757
|
+
const PULL_INTENTIONAL_DELETE_MIN_ABS = 10;
|
|
1758
|
+
const PULL_INTENTIONAL_DELETE_RATIO = 0.1;
|
|
1759
|
+
|
|
1741
1760
|
function computePullPlan(
|
|
1742
1761
|
remoteFiles: RemoteFile[],
|
|
1743
1762
|
journal: SyncJournal,
|
|
@@ -1758,6 +1777,12 @@ function computePullPlan(
|
|
|
1758
1777
|
fileTombstones: ReadonlyMap<string, CompanyTombstone> = new Map(),
|
|
1759
1778
|
): PullPlan {
|
|
1760
1779
|
const items: PullPlanItem[] = [];
|
|
1780
|
+
// Clean/current/non-divergent files missing locally — collected here, then
|
|
1781
|
+
// resolved post-loop by the bulk-asymmetry decision (tombstone vs. restore).
|
|
1782
|
+
const intentionalDeleteCandidates: Array<{
|
|
1783
|
+
remoteFile: RemoteFile;
|
|
1784
|
+
localPath: string;
|
|
1785
|
+
}> = [];
|
|
1761
1786
|
|
|
1762
1787
|
for (const remoteFile of remoteFiles) {
|
|
1763
1788
|
const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
|
|
@@ -1851,6 +1876,23 @@ function computePullPlan(
|
|
|
1851
1876
|
const tombstoneSuppresses =
|
|
1852
1877
|
tombstone !== undefined &&
|
|
1853
1878
|
!isRemoteRecreateAfterTombstone(remoteFile, tombstone);
|
|
1879
|
+
// LOCAL journal tombstone — this machine's own intentional-delete record,
|
|
1880
|
+
// independent of the (best-effort, intermittently-degraded) server
|
|
1881
|
+
// FILE_TOMBSTONE fetch above. When a clean local delete is classified, the
|
|
1882
|
+
// push leg stamps `removedAt` on the journal entry; honoring it here makes
|
|
1883
|
+
// the delete-vs-drift-restore outcome DETERMINISTIC, closing the
|
|
1884
|
+
// intermittent respawn where a degraded tombstone fetch let the pull leg
|
|
1885
|
+
// re-materialize an intentionally-deleted file. Reuses the same re-create
|
|
1886
|
+
// guard (a remote object newer than the local delete is a genuine re-create
|
|
1887
|
+
// and still downloads). Consulted ONLY on the `!localExists` re-download
|
|
1888
|
+
// path below — never to delete a file that is present locally.
|
|
1889
|
+
const localTombstoneSuppresses =
|
|
1890
|
+
journalEntry !== undefined &&
|
|
1891
|
+
isTombstone(journalEntry) &&
|
|
1892
|
+
journalEntry.removedAt !== undefined &&
|
|
1893
|
+
!isRemoteRecreateAfterTombstone(remoteFile, {
|
|
1894
|
+
deletedAt: journalEntry.removedAt,
|
|
1895
|
+
});
|
|
1854
1896
|
|
|
1855
1897
|
// lstat (not existsSync/statSync) handles three cases the legacy
|
|
1856
1898
|
// checks got wrong for symlinks:
|
|
@@ -2074,15 +2116,67 @@ function computePullPlan(
|
|
|
2074
2116
|
// re-create": the classic resurrection ("remote present → I'm behind →
|
|
2075
2117
|
// download") the planner must NOT do. Route to `tombstone-delete` so the
|
|
2076
2118
|
// executor drops any stale journal entry and the key stays gone, instead of
|
|
2077
|
-
// pulling the deleted object back in.
|
|
2078
|
-
|
|
2119
|
+
// pulling the deleted object back in. `localTombstoneSuppresses` adds the
|
|
2120
|
+
// same protection driven by this machine's own journal tombstone, so an
|
|
2121
|
+
// intentional delete is not resurrected even when the server FILE_TOMBSTONE
|
|
2122
|
+
// fetch is degraded.
|
|
2123
|
+
if (tombstoneSuppresses || localTombstoneSuppresses) {
|
|
2124
|
+
// A persisting `local-delete` tombstone (stamped by the intentional-delete
|
|
2125
|
+
// producer below) must KEEP its journal entry — the push leg still needs
|
|
2126
|
+
// it to propagate the S3 delete. Skip the re-download without dropping the
|
|
2127
|
+
// entry. Other tombstone kinds (scope-shrink / FILE_TOMBSTONE) drop it.
|
|
2128
|
+
if (journalEntry?.removedReason === "local-delete") {
|
|
2129
|
+
continue;
|
|
2130
|
+
}
|
|
2079
2131
|
items.push({ action: "tombstone-delete", remoteFile, localPath });
|
|
2080
2132
|
continue;
|
|
2081
2133
|
}
|
|
2082
2134
|
|
|
2135
|
+
// Intentional-delete producer: a clean, CURRENT (etag matches), non-divergent
|
|
2136
|
+
// file that is MISSING locally is the user deleting a genuinely-synced file.
|
|
2137
|
+
// Collect it; the post-loop bulk-asymmetry decision tombstones it (intent) or
|
|
2138
|
+
// restores it (mass loss = accident). A stale-etag, divergent, tombstoned, or
|
|
2139
|
+
// never-journaled missing file falls through to the normal download below.
|
|
2140
|
+
if (
|
|
2141
|
+
!localExists &&
|
|
2142
|
+
journalEntry !== undefined &&
|
|
2143
|
+
!isTombstone(journalEntry) &&
|
|
2144
|
+
!journalEntry.localDiverges &&
|
|
2145
|
+
journalEntry.remoteEtag !== undefined &&
|
|
2146
|
+
normalizeEtag(remoteFile.etag) === journalEntry.remoteEtag
|
|
2147
|
+
) {
|
|
2148
|
+
intentionalDeleteCandidates.push({ remoteFile, localPath });
|
|
2149
|
+
continue;
|
|
2150
|
+
}
|
|
2151
|
+
|
|
2083
2152
|
items.push({ action: "download", remoteFile, localPath, isNew: !localExists });
|
|
2084
2153
|
}
|
|
2085
2154
|
|
|
2155
|
+
// ── Intentional-delete bulk-asymmetry decision (pull-leg producer) ─────────
|
|
2156
|
+
// Below the breaker → stamp a persisting `local-delete` tombstone (the
|
|
2157
|
+
// suppression above keeps it from being re-pulled; the push leg propagates the
|
|
2158
|
+
// S3 delete on its next pass). At/above → drift-restore EVERY candidate: a
|
|
2159
|
+
// mass disappearance (corrupt mirror / bulk rm / checkout / unmount) is
|
|
2160
|
+
// accidental loss, never intent (the ridge-incident guard).
|
|
2161
|
+
const totalJournalEntries = Object.keys(journal.files).length;
|
|
2162
|
+
const intentionalDeleteIsBulk =
|
|
2163
|
+
intentionalDeleteCandidates.length >= PULL_INTENTIONAL_DELETE_MIN_ABS &&
|
|
2164
|
+
totalJournalEntries > 0 &&
|
|
2165
|
+
intentionalDeleteCandidates.length / totalJournalEntries >=
|
|
2166
|
+
PULL_INTENTIONAL_DELETE_RATIO;
|
|
2167
|
+
for (const cand of intentionalDeleteCandidates) {
|
|
2168
|
+
if (intentionalDeleteIsBulk) {
|
|
2169
|
+
items.push({
|
|
2170
|
+
action: "download",
|
|
2171
|
+
remoteFile: cand.remoteFile,
|
|
2172
|
+
localPath: cand.localPath,
|
|
2173
|
+
isNew: true,
|
|
2174
|
+
});
|
|
2175
|
+
} else {
|
|
2176
|
+
tombstoneEntry(journal, cand.remoteFile.key, "local-delete");
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2086
2180
|
let filesToDownload = 0;
|
|
2087
2181
|
let bytesToDownload = 0;
|
|
2088
2182
|
let filesToSkip = 0;
|
package/src/journal.test.ts
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
gcTombstones,
|
|
26
26
|
generatePullId,
|
|
27
27
|
TOMBSTONE_TTL_MS,
|
|
28
|
+
MAX_PULLS_PER_COMPANY,
|
|
28
29
|
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
29
30
|
migratePersonalVaultJournal,
|
|
30
31
|
listJournals,
|
|
@@ -463,6 +464,49 @@ describe("journal", () => {
|
|
|
463
464
|
"2026-05-20T00:00:00.000Z",
|
|
464
465
|
);
|
|
465
466
|
});
|
|
467
|
+
|
|
468
|
+
it("bounds pulls per company and preserves newest lastPullRecord", () => {
|
|
469
|
+
const j: SyncJournal = {
|
|
470
|
+
version: "2",
|
|
471
|
+
lastSync: "",
|
|
472
|
+
files: {},
|
|
473
|
+
pulls: [],
|
|
474
|
+
};
|
|
475
|
+
const completedAt = (n: number) =>
|
|
476
|
+
`2026-05-20T00:00:${String(n).padStart(2, "0")}.000Z`;
|
|
477
|
+
|
|
478
|
+
for (let i = 0; i < MAX_PULLS_PER_COMPANY + 5; i++) {
|
|
479
|
+
appendPullRecord(
|
|
480
|
+
j,
|
|
481
|
+
pull({
|
|
482
|
+
pullId: `pull-a-${String(i).padStart(2, "0")}`,
|
|
483
|
+
companyUid: "cmp_a",
|
|
484
|
+
completedAt: completedAt(i),
|
|
485
|
+
}),
|
|
486
|
+
);
|
|
487
|
+
}
|
|
488
|
+
for (let i = 0; i < 3; i++) {
|
|
489
|
+
appendPullRecord(
|
|
490
|
+
j,
|
|
491
|
+
pull({
|
|
492
|
+
pullId: `pull-b-${i}`,
|
|
493
|
+
companyUid: "cmp_b",
|
|
494
|
+
completedAt: completedAt(i),
|
|
495
|
+
}),
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
const pulls = j.pulls ?? [];
|
|
500
|
+
const companyA = pulls.filter((p) => p.companyUid === "cmp_a");
|
|
501
|
+
const companyB = pulls.filter((p) => p.companyUid === "cmp_b");
|
|
502
|
+
expect(companyA).toHaveLength(MAX_PULLS_PER_COMPANY);
|
|
503
|
+
expect(companyB).toHaveLength(3);
|
|
504
|
+
expect(companyA[0].pullId).toBe("pull-a-05");
|
|
505
|
+
expect(companyA.at(-1)?.pullId).toBe("pull-a-54");
|
|
506
|
+
expect(companyA.some((p) => p.pullId === "pull-a-00")).toBe(false);
|
|
507
|
+
expect(lastPullRecord(j, "cmp_a")?.pullId).toBe("pull-a-54");
|
|
508
|
+
expect(lastPullRecord(j, "cmp_b")?.pullId).toBe("pull-b-2");
|
|
509
|
+
});
|
|
466
510
|
});
|
|
467
511
|
|
|
468
512
|
describe("tombstoneEntry + isTombstone", () => {
|