@indigoai-us/hq-cloud 6.14.18 → 6.14.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/.github/workflows/ci.yml +35 -0
  2. package/dist/bin/sync-runner-company.d.ts +7 -0
  3. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  4. package/dist/bin/sync-runner-company.js +21 -1
  5. package/dist/bin/sync-runner-company.js.map +1 -1
  6. package/dist/bin/sync-runner-planning.d.ts.map +1 -1
  7. package/dist/bin/sync-runner-planning.js +8 -16
  8. package/dist/bin/sync-runner-planning.js.map +1 -1
  9. package/dist/bin/sync-runner-planning.test.js +138 -0
  10. package/dist/bin/sync-runner-planning.test.js.map +1 -1
  11. package/dist/bin/sync-runner-watch-loop.d.ts +10 -2
  12. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  13. package/dist/bin/sync-runner-watch-loop.js +201 -25
  14. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  15. package/dist/bin/sync-runner-watch-routes.d.ts +1 -1
  16. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
  17. package/dist/bin/sync-runner-watch-routes.js +14 -2
  18. package/dist/bin/sync-runner-watch-routes.js.map +1 -1
  19. package/dist/bin/sync-runner.d.ts +17 -2
  20. package/dist/bin/sync-runner.d.ts.map +1 -1
  21. package/dist/bin/sync-runner.js +36 -6
  22. package/dist/bin/sync-runner.js.map +1 -1
  23. package/dist/bin/sync-runner.test.js +550 -3
  24. package/dist/bin/sync-runner.test.js.map +1 -1
  25. package/dist/cli/rescue-core.js +27 -1
  26. package/dist/cli/rescue-core.js.map +1 -1
  27. package/dist/cli/rescue-drop-dir-symlink.test.d.ts +2 -0
  28. package/dist/cli/rescue-drop-dir-symlink.test.d.ts.map +1 -0
  29. package/dist/cli/rescue-drop-dir-symlink.test.js +206 -0
  30. package/dist/cli/rescue-drop-dir-symlink.test.js.map +1 -0
  31. package/dist/cli/share.d.ts +15 -0
  32. package/dist/cli/share.d.ts.map +1 -1
  33. package/dist/cli/share.js +100 -55
  34. package/dist/cli/share.js.map +1 -1
  35. package/dist/cli/share.test.js +119 -6
  36. package/dist/cli/share.test.js.map +1 -1
  37. package/dist/cli/sync.d.ts.map +1 -1
  38. package/dist/cli/sync.js +19 -25
  39. package/dist/cli/sync.js.map +1 -1
  40. package/dist/cli/sync.test.js +121 -0
  41. package/dist/cli/sync.test.js.map +1 -1
  42. package/dist/journal.d.ts +3 -1
  43. package/dist/journal.d.ts.map +1 -1
  44. package/dist/journal.js +13 -2
  45. package/dist/journal.js.map +1 -1
  46. package/dist/journal.test.js +26 -1
  47. package/dist/journal.test.js.map +1 -1
  48. package/dist/personal-vault.d.ts +12 -0
  49. package/dist/personal-vault.d.ts.map +1 -1
  50. package/dist/personal-vault.js +20 -0
  51. package/dist/personal-vault.js.map +1 -1
  52. package/dist/qmd-reindex.d.ts +88 -11
  53. package/dist/qmd-reindex.d.ts.map +1 -1
  54. package/dist/qmd-reindex.js +354 -65
  55. package/dist/qmd-reindex.js.map +1 -1
  56. package/dist/qmd-reindex.test.d.ts +4 -0
  57. package/dist/qmd-reindex.test.d.ts.map +1 -1
  58. package/dist/qmd-reindex.test.js +285 -3
  59. package/dist/qmd-reindex.test.js.map +1 -1
  60. package/dist/s3.d.ts +13 -0
  61. package/dist/s3.d.ts.map +1 -1
  62. package/dist/s3.js +34 -1
  63. package/dist/s3.js.map +1 -1
  64. package/dist/s3.test.js +60 -1
  65. package/dist/s3.test.js.map +1 -1
  66. package/dist/telemetry.d.ts +22 -0
  67. package/dist/telemetry.d.ts.map +1 -1
  68. package/dist/telemetry.js +79 -0
  69. package/dist/telemetry.js.map +1 -1
  70. package/dist/telemetry.test.js +117 -1
  71. package/dist/telemetry.test.js.map +1 -1
  72. package/dist/watcher.d.ts +26 -1
  73. package/dist/watcher.d.ts.map +1 -1
  74. package/dist/watcher.js +75 -12
  75. package/dist/watcher.js.map +1 -1
  76. package/package.json +1 -1
  77. package/src/bin/sync-runner-company.ts +29 -1
  78. package/src/bin/sync-runner-planning.test.ts +153 -0
  79. package/src/bin/sync-runner-planning.ts +10 -17
  80. package/src/bin/sync-runner-watch-loop.ts +322 -32
  81. package/src/bin/sync-runner-watch-routes.ts +14 -1
  82. package/src/bin/sync-runner.test.ts +662 -6
  83. package/src/bin/sync-runner.ts +65 -7
  84. package/src/cli/rescue-core.ts +26 -1
  85. package/src/cli/rescue-drop-dir-symlink.test.ts +224 -0
  86. package/src/cli/share.test.ts +150 -6
  87. package/src/cli/share.ts +143 -48
  88. package/src/cli/sync.test.ts +150 -1
  89. package/src/cli/sync.ts +25 -29
  90. package/src/journal.test.ts +45 -0
  91. package/src/journal.ts +18 -1
  92. package/src/personal-vault.ts +23 -0
  93. package/src/qmd-reindex.test.ts +324 -4
  94. package/src/qmd-reindex.ts +414 -76
  95. package/src/s3.test.ts +79 -0
  96. package/src/s3.ts +48 -1
  97. package/src/telemetry.test.ts +128 -0
  98. package/src/telemetry.ts +80 -0
  99. package/src/watcher.ts +111 -14
  100. package/test/e2e/watcher-real-chokidar.test.ts +62 -2
  101. package/test/e2e/watcher-recursive-backend.test.ts +67 -1
package/src/cli/share.ts CHANGED
@@ -41,7 +41,10 @@ import {
41
41
  wrapFilterWithPersonalVaultDefaults,
42
42
  type PersonalVaultExclusion,
43
43
  } from "../personal-vault-exclusions.js";
44
- import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
44
+ import {
45
+ isGeneratedCoreMirrorKey,
46
+ isPersonalVaultDeletionExcluded,
47
+ } from "../personal-vault.js";
45
48
  import { resolveConflict } from "./conflict.js";
46
49
  import type { ConflictStrategy } from "./conflict.js";
47
50
  import type { SyncProgressEvent } from "./sync.js";
@@ -590,6 +593,12 @@ export interface ShareOptions {
590
593
  * next push to erase it.
591
594
  */
592
595
  propagateDeletePolicy?: "currency-gated" | "owned-only" | "all";
596
+ /**
597
+ * Explicit vault-relative delete roots captured by a live watcher before a
598
+ * scoped push. These roots do not need to exist locally; delete execution
599
+ * still requires a current version-bound intent (or server tombstone).
600
+ */
601
+ deleteScopeRoots?: string[];
593
602
  /**
594
603
  * Hq-root-relative key prefixes whose journal entries should be
595
604
  * unconditionally decommissioned from the remote bucket and journal,
@@ -753,9 +762,27 @@ export interface ShareResult {
753
762
  * way to runner/UI consumers.
754
763
  */
755
764
  conflictPaths: string[];
765
+ /** Uncapped per-path delete outcomes used by scoped watcher publication. */
766
+ pathResults?: SharePathResult[];
756
767
  aborted: boolean;
757
768
  }
758
769
 
770
+ export type ShareDeleteRefusalReason =
771
+ | "stale-etag"
772
+ | "legacy-no-etag"
773
+ | "bulk-asymmetry"
774
+ | "divergent-local"
775
+ | "missing-delete-intent"
776
+ | "intent-changed"
777
+ | "transfer-error";
778
+
779
+ export interface SharePathResult {
780
+ path: string;
781
+ status: "accepted" | "refused";
782
+ operation: "delete" | "tombstone";
783
+ reason?: ShareDeleteRefusalReason;
784
+ }
785
+
759
786
  /**
760
787
  * A conditional-write fence rejection — the SDK's 412 (`name:
761
788
  * "PreconditionFailed"`) or the presigned transport's mirror of it. Means
@@ -838,6 +865,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
838
865
  const counters = createShareCounters();
839
866
  const filesRefusedStalePaths: string[] = [];
840
867
  const conflictPaths: string[] = [];
868
+ const pathResults: SharePathResult[] = [];
841
869
 
842
870
  const plans = await buildSharePlans(run);
843
871
  const uploadResult = await executeUploads(
@@ -852,6 +880,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
852
880
  counters,
853
881
  filesRefusedStalePaths,
854
882
  uploadResult.abortFlightConflictPaths,
883
+ pathResults,
855
884
  true,
856
885
  );
857
886
  }
@@ -862,11 +891,19 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
862
891
  plans.decommissionPlan,
863
892
  counters,
864
893
  filesRefusedStalePaths,
894
+ pathResults,
865
895
  );
866
896
  finalizeShareJournal(run);
867
897
  throwUploadWorkerErrors(uploadResult.workerErrors);
868
898
 
869
- return buildShareResult(run, counters, filesRefusedStalePaths, conflictPaths, false);
899
+ return buildShareResult(
900
+ run,
901
+ counters,
902
+ filesRefusedStalePaths,
903
+ conflictPaths,
904
+ pathResults,
905
+ false,
906
+ );
870
907
  }
871
908
 
872
909
  type DeletePolicy = NonNullable<ShareOptions["propagateDeletePolicy"]>;
@@ -1070,7 +1107,27 @@ async function buildSharePlans(run: PushRunContext): Promise<SharePlans> {
1070
1107
  // `companies/{slug}/…` keys, so this only applies in company mode.
1071
1108
  const filesToShare =
1072
1109
  run.options.personalMode === true
1073
- ? collected
1110
+ ? collected.filter((entry) => {
1111
+ // `core/<type>` symlinks are generated projections of canonical
1112
+ // personal/package content. Reindex or package wiring owns their
1113
+ // lifecycle, so uploading them creates remote objects that a later
1114
+ // pull can only materialize temporarily before reindex removes them.
1115
+ // Keep ordinary core files eligible: only a symlink is derived.
1116
+ const generatedCoreMirror =
1117
+ entry.kind === "symlink" && isGeneratedCoreMirrorKey(entry.relativePath)
1118
+ if (!generatedCoreMirror) return true;
1119
+
1120
+ // This is a type-aware companion to the path-only personal-vault
1121
+ // default filters. `collectFiles` is the first seam that knows the
1122
+ // entry is a symlink, so record it here for the same summary/result
1123
+ // accounting used by the static exclusions.
1124
+ if (!run.excludedSet.has(entry.relativePath)) {
1125
+ run.excludedSet.add(entry.relativePath);
1126
+ run.excludedById["generated-core-mirror"] =
1127
+ (run.excludedById["generated-core-mirror"] ?? 0) + 1;
1128
+ }
1129
+ return false;
1130
+ })
1074
1131
  : collected.filter((entry) => {
1075
1132
  if (!entry.relativePath.startsWith("companies/")) return true;
1076
1133
  run.emit({
@@ -1085,7 +1142,12 @@ async function buildSharePlans(run: PushRunContext): Promise<SharePlans> {
1085
1142
  run.skipUnchanged === true,
1086
1143
  );
1087
1144
  const deleteScopeRoots = run.propagateDeletes === true
1088
- ? resolveDeleteScopeRoots(run.paths, run.hqRoot, run.syncRoot)
1145
+ ? resolveDeleteScopeRoots(
1146
+ run.paths,
1147
+ run.hqRoot,
1148
+ run.syncRoot,
1149
+ run.options.deleteScopeRoots,
1150
+ )
1089
1151
  : [];
1090
1152
  const deletePlan: DeletePlan = run.propagateDeletes === true
1091
1153
  ? await computeDeletePlan(
@@ -1522,6 +1584,7 @@ async function executeDeletes(
1522
1584
  decommissionPlan: string[],
1523
1585
  counters: ShareCounters,
1524
1586
  filesRefusedStalePaths: string[],
1587
+ pathResults: SharePathResult[],
1525
1588
  ): Promise<void> {
1526
1589
  const deleteItems = [
1527
1590
  ...deletePlan.toDelete.map((item) => ({ ...item, decommission: false })),
@@ -1543,6 +1606,9 @@ async function executeDeletes(
1543
1606
  entry!.localDeleteIntent!.version !== item.intentVersion ||
1544
1607
  !isLocallyAbsent(path.join(run.syncRoot, relativePath)))
1545
1608
  ) {
1609
+ if (entry?.localDeleteIntent) {
1610
+ delete entry.localDeleteIntent;
1611
+ }
1546
1612
  counters.filesRefusedStale++;
1547
1613
  if (filesRefusedStalePaths.length < REFUSED_STALE_PATH_CAP) {
1548
1614
  filesRefusedStalePaths.push(relativePath);
@@ -1554,6 +1620,12 @@ async function executeDeletes(
1554
1620
  remoteEtag: "<not-checked>",
1555
1621
  reason: "intent-changed",
1556
1622
  });
1623
+ pathResults.push({
1624
+ path: relativePath,
1625
+ status: "refused",
1626
+ operation: "delete",
1627
+ reason: "intent-changed",
1628
+ });
1557
1629
  continue;
1558
1630
  }
1559
1631
  const size = entry?.size ?? 0;
@@ -1566,6 +1638,11 @@ async function executeDeletes(
1566
1638
  bytes: size,
1567
1639
  deleted: true,
1568
1640
  });
1641
+ pathResults.push({
1642
+ path: relativePath,
1643
+ status: "accepted",
1644
+ operation: "delete",
1645
+ });
1569
1646
  } catch (err) {
1570
1647
  if (err instanceof VaultAuthError) throw err;
1571
1648
  run.emit({
@@ -1573,6 +1650,12 @@ async function executeDeletes(
1573
1650
  path: relativePath,
1574
1651
  message: err instanceof Error ? err.message : String(err),
1575
1652
  });
1653
+ pathResults.push({
1654
+ path: relativePath,
1655
+ status: "refused",
1656
+ operation: "delete",
1657
+ reason: "transfer-error",
1658
+ });
1576
1659
  }
1577
1660
  }
1578
1661
  for (const relativePath of deletePlan.toTombstone) {
@@ -1585,6 +1668,11 @@ async function executeDeletes(
1585
1668
  deleted: true,
1586
1669
  message: "tombstone (remote already 404)",
1587
1670
  });
1671
+ pathResults.push({
1672
+ path: relativePath,
1673
+ status: "accepted",
1674
+ operation: "tombstone",
1675
+ });
1588
1676
  }
1589
1677
  const decommissionedSet =
1590
1678
  decommissionPlan.length > 0 ? new Set(decommissionPlan) : null;
@@ -1601,6 +1689,12 @@ async function executeDeletes(
1601
1689
  remoteEtag: refused.remoteEtag,
1602
1690
  reason: refused.reason,
1603
1691
  });
1692
+ pathResults.push({
1693
+ path: refused.key,
1694
+ status: "refused",
1695
+ operation: "delete",
1696
+ reason: refused.reason,
1697
+ });
1604
1698
  }
1605
1699
  }
1606
1700
 
@@ -1669,6 +1763,7 @@ function buildShareResult(
1669
1763
  counters: ShareCounters,
1670
1764
  filesRefusedStalePaths: string[],
1671
1765
  conflictPaths: string[],
1766
+ pathResults: SharePathResult[],
1672
1767
  aborted: boolean,
1673
1768
  ): ShareResult {
1674
1769
  return {
@@ -1684,6 +1779,7 @@ function buildShareResult(
1684
1779
  filesExcludedByScope: run.scopeExcludedSet.size,
1685
1780
  filesExcludedByIgnore: run.ignoreExcludedSet.size,
1686
1781
  conflictPaths,
1782
+ pathResults,
1687
1783
  aborted,
1688
1784
  };
1689
1785
  }
@@ -2043,8 +2139,26 @@ function resolveDeleteScopeRoots(
2043
2139
  paths: string[],
2044
2140
  hqRoot: string,
2045
2141
  syncRoot: string,
2142
+ explicitRoots: readonly string[] = [],
2046
2143
  ): string[] {
2047
2144
  const prefixes = new Set<string>();
2145
+ for (const root of explicitRoots) {
2146
+ const normalized = root
2147
+ .split("\\")
2148
+ .join("/")
2149
+ .replace(/^\.\/+/, "")
2150
+ .replace(/\/+$/, "");
2151
+ if (
2152
+ normalized === "" ||
2153
+ normalized === "." ||
2154
+ path.posix.isAbsolute(normalized) ||
2155
+ normalized === ".." ||
2156
+ normalized.startsWith("../")
2157
+ ) {
2158
+ continue;
2159
+ }
2160
+ prefixes.add(normalized);
2161
+ }
2048
2162
  for (const p of paths) {
2049
2163
  const absolutePath = path.isAbsolute(p) ? p : path.resolve(hqRoot, p);
2050
2164
  if (!fs.existsSync(absolutePath)) continue;
@@ -2296,23 +2410,19 @@ async function computeDeletePlan(
2296
2410
  // and the journal-mutation buckets are already settled before any I/O.
2297
2411
  type HeadCandidate = { key: string; journalEtag: string; intentVersion: 1 };
2298
2412
  const headCandidates: HeadCandidate[] = [];
2299
- // Litter drain bucket — kept separate from `plan.toDelete` so the
2300
- // bulk-asymmetry breaker (which moves toDelete + headCandidates into
2301
- // `refusedStale` when it trips) can't sweep these out. See
2413
+ // Litter drain bucket — kept separate from normal delete candidates so
2414
+ // the bulk-asymmetry accounting cannot sweep these in. See
2302
2415
  // `isVaultLitterArtifact` for the patterns and rationale: by construction
2303
2416
  // these aren't user content losses, so a high ratio of litter must not
2304
2417
  // refuse the drain — that's the whole point of the bypass. Merged into
2305
2418
  // `plan.toDelete` after the breaker check.
2306
2419
  const litterToDelete: string[] = [];
2307
- // Bulk-asymmetry tracking: count every in-scope journal entry (denominator)
2308
- // and every entry that would have been a delete-candidate before the guard
2309
- // (numerator). Numerator = headCandidates + owned-only/all toDelete picks +
2310
- // legacy-no-etag refusals. We do NOT count "ENOENT but ignore-filtered" or
2311
- // "ENOENT but ephemeral" — those drop out of the plan entirely on their own
2312
- // and don't reflect mirror-loss intent — and litter drains (see above),
2313
- // which are intentional ratchet-cleanup, not mass-delete intent.
2420
+ // Current watcher intents are affirmative evidence of an observed delete,
2421
+ // so only intent-less disappearances contribute to the mirror-loss breaker.
2422
+ // An unmounted/wiped tree produces no intents and still trips it.
2314
2423
  let inScopeJournalEntries = 0;
2315
- let bulkCandidatePicks = 0;
2424
+ let intentlessDisappearances = 0;
2425
+ const intentlessKeys = new Set<string>();
2316
2426
 
2317
2427
  for (const [relativeKey, entry] of Object.entries(journal.files)) {
2318
2428
  const inScope = scopeRoots.some(
@@ -2399,6 +2509,13 @@ async function computeDeletePlan(
2399
2509
  }
2400
2510
 
2401
2511
  if (!hasCurrentLocalDeleteIntent(entry)) {
2512
+ if (
2513
+ policy === "currency-gated" ||
2514
+ (policy === "owned-only" && entry.direction === "up")
2515
+ ) {
2516
+ intentlessDisappearances++;
2517
+ intentlessKeys.add(relativeKey);
2518
+ }
2402
2519
  plan.refusedStale.push({
2403
2520
  key: relativeKey,
2404
2521
  journalEtag: entry.remoteEtag ?? "<missing-delete-intent>",
@@ -2417,14 +2534,8 @@ async function computeDeletePlan(
2417
2534
  });
2418
2535
  continue;
2419
2536
  }
2420
- bulkCandidatePicks++;
2421
2537
  if (policy === "owned-only") {
2422
2538
  if (entry.direction !== "up") {
2423
- // Not a delete candidate under owned-only, but it WAS missing
2424
- // locally. Don't count it for the bulk guard — direction:'down'
2425
- // entries that vanish locally are silently ignored by this policy
2426
- // anyway, so they don't represent intent to mass-delete.
2427
- bulkCandidatePicks--;
2428
2539
  continue;
2429
2540
  }
2430
2541
  plan.toDelete.push({
@@ -2457,42 +2568,26 @@ async function computeDeletePlan(
2457
2568
  if (
2458
2569
  policy !== "all" &&
2459
2570
  !isBulkAsymmetryOverride() &&
2460
- bulkCandidatePicks >= BULK_ASYMMETRY_MIN_ABS &&
2571
+ intentlessDisappearances >= BULK_ASYMMETRY_MIN_ABS &&
2461
2572
  inScopeJournalEntries > 0 &&
2462
- bulkCandidatePicks / inScopeJournalEntries >= BULK_ASYMMETRY_RATIO
2573
+ intentlessDisappearances / inScopeJournalEntries >= BULK_ASYMMETRY_RATIO
2463
2574
  ) {
2464
- // Move every staged candidate (both already-bucketed toDelete from
2465
- // owned-only and queued headCandidates from currency-gated) into
2466
- // refusedStale with reason "bulk-asymmetry". Journal is not mutated;
2467
- // no DeleteObject is issued.
2468
2575
  const samplePaths: string[] = [];
2469
- const pushRefused = (key: string): void => {
2470
- plan.refusedStale.push({
2471
- key,
2472
- journalEtag: "<bulk-asymmetry>",
2473
- remoteEtag: "<not-checked>",
2474
- reason: "bulk-asymmetry",
2475
- });
2576
+ for (const refused of plan.refusedStale) {
2577
+ if (!intentlessKeys.has(refused.key)) continue;
2578
+ refused.journalEtag = "<bulk-asymmetry>";
2579
+ refused.remoteEtag = "<not-checked>";
2580
+ refused.reason = "bulk-asymmetry";
2476
2581
  if (samplePaths.length < BULK_ASYMMETRY_SAMPLE_CAP) {
2477
- samplePaths.push(key);
2582
+ samplePaths.push(refused.key);
2478
2583
  }
2479
- };
2480
- for (const item of plan.toDelete) pushRefused(item.key);
2481
- for (const c of headCandidates) pushRefused(c.key);
2482
- plan.toDelete = [];
2584
+ }
2483
2585
  plan.bulkAsymmetry = {
2484
- candidates: bulkCandidatePicks,
2586
+ candidates: intentlessDisappearances,
2485
2587
  inScope: inScopeJournalEntries,
2486
- ratio: bulkCandidatePicks / inScopeJournalEntries,
2588
+ ratio: intentlessDisappearances / inScopeJournalEntries,
2487
2589
  samplePaths,
2488
2590
  };
2489
- // Litter still drains even when the breaker trips — see `litterToDelete`
2490
- // declaration. The breaker protects against corrupt-mirror mass-deletes
2491
- // of user content; litter cleanup is orthogonal and should never be
2492
- // refused for the same reason new-litter producer-side exclusions
2493
- // shouldn't block it.
2494
- plan.toDelete.push(...litterToDelete.map((key) => ({ key, intentVersion: null })));
2495
- return plan;
2496
2591
  }
2497
2592
 
2498
2593
  // Stage 2: bounded-parallel HEAD pass. Promise.all over chunks of size
@@ -84,7 +84,10 @@ const mockVendResponse = {
84
84
  };
85
85
 
86
86
  function setupFetchMock(
87
- opts: { tombstones?: Array<{ key: string; deletedAt: string }> } = {},
87
+ opts: {
88
+ tombstones?: Array<{ key: string; deletedAt: string }>;
89
+ tombstoneFailure?: boolean;
90
+ } = {},
88
91
  ) {
89
92
  const fetchMock = vi.fn().mockImplementation(async (url: string) => {
90
93
  const urlStr = String(url);
@@ -92,6 +95,7 @@ function setupFetchMock(
92
95
  // every legacy test exercises the no-suppression path; tests that need
93
96
  // suppression pass `{ tombstones }`.
94
97
  if (urlStr.includes("/v1/files/tombstones")) {
98
+ if (opts.tombstoneFailure) throw new Error("tombstone lookup unavailable");
95
99
  return {
96
100
  ok: true,
97
101
  status: 200,
@@ -1085,6 +1089,53 @@ describe("sync", () => {
1085
1089
  expect(journal.files[key]?.removedAt).toBeUndefined();
1086
1090
  });
1087
1091
 
1092
+ it("tombstone lookup failure does not resurrect a current intent-backed delete", async () => {
1093
+ const key = "docs/watcher-deleted.md";
1094
+ const localPath = path.join(tmpDir, "companies", "acme", key);
1095
+ vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
1096
+ { key, size: 42, lastModified: new Date(), etag: '"cur123"' },
1097
+ ]);
1098
+ setupFetchMock({ tombstoneFailure: true });
1099
+ fs.writeFileSync(
1100
+ journalPath,
1101
+ JSON.stringify({
1102
+ version: "2",
1103
+ lastSync: new Date().toISOString(),
1104
+ files: {
1105
+ [key]: {
1106
+ hash: "synced-hash",
1107
+ size: 42,
1108
+ syncedAt: new Date().toISOString(),
1109
+ direction: "down",
1110
+ kind: "file",
1111
+ remoteEtag: "cur123",
1112
+ localDeleteIntent: {
1113
+ version: 1,
1114
+ remoteEtag: "cur123",
1115
+ localHash: "synced-hash",
1116
+ localKind: "file",
1117
+ },
1118
+ },
1119
+ },
1120
+ pulls: [],
1121
+ }),
1122
+ );
1123
+
1124
+ await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
1125
+
1126
+ expect(s3Module.downloadFile).not.toHaveBeenCalledWith(
1127
+ expect.anything(),
1128
+ key,
1129
+ expect.anything(),
1130
+ expect.anything(),
1131
+ );
1132
+ expect(fs.existsSync(localPath)).toBe(false);
1133
+ expect(
1134
+ JSON.parse(fs.readFileSync(journalPath, "utf8")).files[key]
1135
+ ?.removedReason,
1136
+ ).toBe("local-delete");
1137
+ });
1138
+
1088
1139
  it("pull producer: a BULK disappearance is drift-restored, NOT tombstoned (ridge-incident guard in the pull leg)", async () => {
1089
1140
  // 12 clean files vanish at once (corrupt mirror / bulk rm / checkout). The
1090
1141
  // bulk-asymmetry guard refuses to treat this as intent — every file is
@@ -3480,6 +3531,104 @@ describe("sync", () => {
3480
3531
  });
3481
3532
  });
3482
3533
 
3534
+ it("converges after reindex prunes a downloaded generated core mirror (DEV-1974)", async () => {
3535
+ // Legacy personal-vault objects may be generated links below core/<type>.
3536
+ // A pull materializes the link, then reindex removes it because its
3537
+ // canonical source is elsewhere. The second pull must recognize the
3538
+ // journaled symlink as generated state rather than downloading it again.
3539
+ const linkKey = "core/policies/legacy-mirror.md";
3540
+ const linkPath = path.join(tmpDir, linkKey);
3541
+ const linkTarget = "../../personal/policies/legacy-mirror.md";
3542
+ const remote = {
3543
+ key: linkKey,
3544
+ size: 48,
3545
+ lastModified: new Date(),
3546
+ etag: '"legacy-mirror"',
3547
+ };
3548
+
3549
+ vi.mocked(s3Module.listRemoteFiles).mockResolvedValue([remote]);
3550
+ vi.mocked(s3Module.downloadFile).mockImplementation(
3551
+ async (_ctx: unknown, _key: string, localPath: string) => {
3552
+ fs.mkdirSync(path.dirname(localPath), { recursive: true });
3553
+ fs.symlinkSync(linkTarget, localPath);
3554
+ return { metadata: {} };
3555
+ },
3556
+ );
3557
+
3558
+ const first = await sync({
3559
+ company: "acme",
3560
+ vaultConfig: mockConfig,
3561
+ hqRoot: tmpDir,
3562
+ personalMode: true,
3563
+ });
3564
+ expect(first.filesDownloaded).toBe(1);
3565
+ expect(fs.lstatSync(linkPath).isSymbolicLink()).toBe(true);
3566
+
3567
+ // The real post-sync reindex removes this retired mirror. The unit test
3568
+ // mocks reindex globally, so make that lifecycle transition explicit.
3569
+ fs.unlinkSync(linkPath);
3570
+
3571
+ const second = await sync({
3572
+ company: "acme",
3573
+ vaultConfig: mockConfig,
3574
+ hqRoot: tmpDir,
3575
+ personalMode: true,
3576
+ });
3577
+
3578
+ expect(second.filesDownloaded).toBe(0);
3579
+ expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(1);
3580
+ });
3581
+
3582
+ it("pulls a changed canonical object after a generated core mirror journal entry", async () => {
3583
+ const key = "core/policies/legacy-mirror.md";
3584
+ const localPath = path.join(tmpDir, key);
3585
+ const legacyMirror = {
3586
+ key,
3587
+ size: 48,
3588
+ lastModified: new Date(),
3589
+ etag: '"legacy-mirror"',
3590
+ };
3591
+ const canonicalReplacement = {
3592
+ ...legacyMirror,
3593
+ lastModified: new Date(Date.now() + 1_000),
3594
+ etag: '"canonical-replacement"',
3595
+ };
3596
+
3597
+ vi.mocked(s3Module.listRemoteFiles)
3598
+ .mockResolvedValueOnce([legacyMirror])
3599
+ .mockResolvedValueOnce([canonicalReplacement]);
3600
+ vi.mocked(s3Module.downloadFile)
3601
+ .mockImplementationOnce(async (_ctx: unknown, _key: string, destination: string) => {
3602
+ fs.mkdirSync(path.dirname(destination), { recursive: true });
3603
+ fs.symlinkSync("../../personal/policies/legacy-mirror.md", destination);
3604
+ return { metadata: {} };
3605
+ })
3606
+ .mockImplementationOnce(async (_ctx: unknown, _key: string, destination: string) => {
3607
+ fs.mkdirSync(path.dirname(destination), { recursive: true });
3608
+ fs.writeFileSync(destination, "canonical replacement\n");
3609
+ return { metadata: {} };
3610
+ });
3611
+
3612
+ await sync({
3613
+ company: "acme",
3614
+ vaultConfig: mockConfig,
3615
+ hqRoot: tmpDir,
3616
+ personalMode: true,
3617
+ });
3618
+ fs.unlinkSync(localPath);
3619
+
3620
+ const second = await sync({
3621
+ company: "acme",
3622
+ vaultConfig: mockConfig,
3623
+ hqRoot: tmpDir,
3624
+ personalMode: true,
3625
+ });
3626
+
3627
+ expect(second.filesDownloaded).toBe(1);
3628
+ expect(fs.readFileSync(localPath, "utf8")).toBe("canonical replacement\n");
3629
+ expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(2);
3630
+ });
3631
+
3483
3632
  it("does not crash when local is a real directory at a remote-key path (EISDIR guard)", async () => {
3484
3633
  // Codex round-9 P2 follow-up: pre-fix, the pull planner's
3485
3634
  // post-lstat branch always called hashFile (readFileSync) on a
package/src/cli/sync.ts CHANGED
@@ -48,7 +48,10 @@ import {
48
48
  PERSONAL_VAULT_JOURNAL_SLUG,
49
49
  migratePersonalVaultJournal,
50
50
  } from "../journal.js";
51
- import { PERSONAL_VAULT_MANIFEST_KEY } from "../personal-vault.js";
51
+ import {
52
+ isGeneratedCoreMirrorKey,
53
+ PERSONAL_VAULT_MANIFEST_KEY,
54
+ } from "../personal-vault.js";
52
55
  import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
53
56
  import {
54
57
  buildScopeShrinkPlan,
@@ -2122,33 +2125,6 @@ interface PullPlan {
2122
2125
  const PULL_INTENTIONAL_DELETE_MIN_ABS = 10;
2123
2126
  const PULL_INTENTIONAL_DELETE_RATIO = 0.1;
2124
2127
 
2125
- // The four release-shipped `core/<type>/` namespaces the personal-overlay
2126
- // mirror (reindex) materializes into. A stale mirror symlink pushed to the
2127
- // vault lands as a single object at exactly `core/<type>/<...>/<entry>`, and if
2128
- // a later release ships a REAL directory of the same name, pull sees a
2129
- // (local real dir) vs (remote single object) collision at that key forever.
2130
- const OVERLAY_MIRROR_CORE_TYPES = ["knowledge", "policies", "workers", "settings"];
2131
-
2132
- /**
2133
- * True when `key` is under a release-shipped personal-overlay-mirror core
2134
- * namespace (`core/{knowledge,policies,workers,settings}/<entry>/...`).
2135
- *
2136
- * A remote LIST entry is always a single S3 object. A REAL release-shipped core
2137
- * directory only ever syncs as child objects UNDER its key
2138
- * (`core/knowledge/public/agent-browser/INDEX.md`), never as one object AT the
2139
- * directory's own key — so a single object at such a key, colliding with a
2140
- * local real directory, is unambiguously a stale overlay symlink marker, not
2141
- * live core content. Pure path test — no I/O — safe inside the sync planner.
2142
- */
2143
- function isOverlayMirrorCoreKey(key: string): boolean {
2144
- const parts = key.split("/");
2145
- return (
2146
- parts.length >= 3 &&
2147
- parts[0] === "core" &&
2148
- OVERLAY_MIRROR_CORE_TYPES.includes(parts[1])
2149
- );
2150
- }
2151
-
2152
2128
  function computePullPlan(
2153
2129
  remoteFiles: RemoteFile[],
2154
2130
  journal: SyncJournal,
@@ -2305,6 +2281,26 @@ function computePullPlan(
2305
2281
 
2306
2282
  const journalEntry = getEntry(journal, remoteFile.key);
2307
2283
 
2284
+ // Reindex and package wiring own symlinks under these `core/<type>`
2285
+ // surfaces; their canonical sources are elsewhere. A prior pull records
2286
+ // the remote object as a symlink, then reindex removes the generated link.
2287
+ // Without this classification the next pull sees only an absent local path
2288
+ // and downloads the same marker forever. Retain regular core files: the
2289
+ // journal kind is the discriminator, because LIST exposes no object type.
2290
+ // An unchanged ETag is also required: a remote writer can replace a legacy
2291
+ // symlink with canonical file content at the same key, and that replacement
2292
+ // must still be pulled.
2293
+ if (
2294
+ personalMode &&
2295
+ journalEntry?.kind === "symlink" &&
2296
+ isGeneratedCoreMirrorKey(remoteFile.key) &&
2297
+ journalEntry.remoteEtag !== undefined &&
2298
+ !hasRemoteChanged(remoteFile, journalEntry)
2299
+ ) {
2300
+ items.push({ action: "skip-excluded-policy", remoteFile, localPath });
2301
+ continue;
2302
+ }
2303
+
2308
2304
  // ── FILE_TOMBSTONE consult (delete-resync) ───────────────────────────────
2309
2305
  // A remote object present in the LIST may be an intentionally-deleted key
2310
2306
  // (a peer re-pushed it, or its delete-marker hasn't propagated to this
@@ -2414,7 +2410,7 @@ function computePullPlan(
2414
2410
  // "rm -rf the local directory" advice below is actively dangerous here
2415
2411
  // (it would destroy release-shipped core content to pull inert derived
2416
2412
  // state), and it recurred on every sync. feedback_ce9aeede.
2417
- if (isOverlayMirrorCoreKey(remoteFile.key)) {
2413
+ if (isGeneratedCoreMirrorKey(remoteFile.key)) {
2418
2414
  console.error(
2419
2415
  ` Note: ${remoteFile.key} is a stale personal-overlay marker ` +
2420
2416
  `shadowing a release-shipped core directory; keeping the local ` +
@@ -29,6 +29,7 @@ import {
29
29
  PERSONAL_VAULT_JOURNAL_SLUG,
30
30
  migratePersonalVaultJournal,
31
31
  listJournals,
32
+ markLocalDeleteIntent,
32
33
  } from "./journal.js";
33
34
  import type { SyncJournal, PullRecord } from "./types.js";
34
35
 
@@ -745,6 +746,50 @@ describe("journal", () => {
745
746
  // successful sync because it read one path keyed off the HQ-ROOT, while the
746
747
  // engine writes slug-sharded journals (personal vault + per-company).
747
748
  // `listJournals` is the all-shard enumeration the status surface now uses.
749
+ it("binds a local delete intent to the captured remote revision", () => {
750
+ const journal: SyncJournal = {
751
+ version: "2",
752
+ lastSync: "",
753
+ pulls: [],
754
+ files: {
755
+ "docs/a.md": {
756
+ hash: "local-hash",
757
+ size: 1,
758
+ syncedAt: new Date().toISOString(),
759
+ direction: "down",
760
+ kind: "file",
761
+ remoteEtag: "new-etag",
762
+ },
763
+ },
764
+ };
765
+
766
+ expect(
767
+ markLocalDeleteIntent(
768
+ journal,
769
+ "docs/a.md",
770
+ "local-hash",
771
+ "file",
772
+ "old-etag",
773
+ ),
774
+ ).toBe(false);
775
+ expect(journal.files["docs/a.md"]?.localDeleteIntent).toBeUndefined();
776
+
777
+ expect(
778
+ markLocalDeleteIntent(
779
+ journal,
780
+ "docs/a.md",
781
+ "local-hash",
782
+ "file",
783
+ "new-etag",
784
+ ),
785
+ ).toBe(true);
786
+ expect(journal.files["docs/a.md"]?.localDeleteIntent).toMatchObject({
787
+ remoteEtag: "new-etag",
788
+ localHash: "local-hash",
789
+ localKind: "file",
790
+ });
791
+ });
792
+
748
793
  describe("listJournals", () => {
749
794
  function seed(slug: string, files: SyncJournal["files"]): void {
750
795
  writeJournal(slug, {