@indigoai-us/hq-cloud 6.11.11 → 6.11.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/dist/bin/sync-runner-company.d.ts +35 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -0
  3. package/dist/bin/sync-runner-company.js +290 -0
  4. package/dist/bin/sync-runner-company.js.map +1 -0
  5. package/dist/bin/sync-runner-events.d.ts +12 -0
  6. package/dist/bin/sync-runner-events.d.ts.map +1 -0
  7. package/dist/bin/sync-runner-events.js +12 -0
  8. package/dist/bin/sync-runner-events.js.map +1 -0
  9. package/dist/bin/sync-runner-planning.d.ts +53 -0
  10. package/dist/bin/sync-runner-planning.d.ts.map +1 -0
  11. package/dist/bin/sync-runner-planning.js +59 -0
  12. package/dist/bin/sync-runner-planning.js.map +1 -0
  13. package/dist/bin/sync-runner-rollup.d.ts +24 -0
  14. package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
  15. package/dist/bin/sync-runner-rollup.js +46 -0
  16. package/dist/bin/sync-runner-rollup.js.map +1 -0
  17. package/dist/bin/sync-runner-telemetry.d.ts +5 -0
  18. package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
  19. package/dist/bin/sync-runner-telemetry.js +5 -0
  20. package/dist/bin/sync-runner-telemetry.js.map +1 -0
  21. package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
  22. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
  23. package/dist/bin/sync-runner-watch-loop.js +372 -0
  24. package/dist/bin/sync-runner-watch-loop.js.map +1 -0
  25. package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
  26. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
  27. package/dist/bin/sync-runner-watch-routes.js +74 -0
  28. package/dist/bin/sync-runner-watch-routes.js.map +1 -0
  29. package/dist/bin/sync-runner.d.ts +5 -54
  30. package/dist/bin/sync-runner.d.ts.map +1 -1
  31. package/dist/bin/sync-runner.js +76 -978
  32. package/dist/bin/sync-runner.js.map +1 -1
  33. package/dist/bin/sync-runner.test.js +265 -11
  34. package/dist/bin/sync-runner.test.js.map +1 -1
  35. package/dist/cli/reindex.d.ts.map +1 -1
  36. package/dist/cli/reindex.js +34 -17
  37. package/dist/cli/reindex.js.map +1 -1
  38. package/dist/cli/reindex.test.js +39 -5
  39. package/dist/cli/reindex.test.js.map +1 -1
  40. package/dist/cli/rescue-classify-ordering.test.js +75 -0
  41. package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
  42. package/dist/cli/rescue-core.d.ts +45 -0
  43. package/dist/cli/rescue-core.d.ts.map +1 -1
  44. package/dist/cli/rescue-core.js +320 -170
  45. package/dist/cli/rescue-core.js.map +1 -1
  46. package/dist/cli/share.d.ts +2 -1
  47. package/dist/cli/share.d.ts.map +1 -1
  48. package/dist/cli/share.js +276 -660
  49. package/dist/cli/share.js.map +1 -1
  50. package/dist/cli/share.test.js +30 -0
  51. package/dist/cli/share.test.js.map +1 -1
  52. package/dist/cli/sync.d.ts +28 -1
  53. package/dist/cli/sync.d.ts.map +1 -1
  54. package/dist/cli/sync.js +541 -748
  55. package/dist/cli/sync.js.map +1 -1
  56. package/dist/cli/sync.test.js +382 -1
  57. package/dist/cli/sync.test.js.map +1 -1
  58. package/dist/cognito-auth.d.ts.map +1 -1
  59. package/dist/cognito-auth.js +55 -10
  60. package/dist/cognito-auth.js.map +1 -1
  61. package/dist/cognito-auth.test.js +61 -0
  62. package/dist/cognito-auth.test.js.map +1 -1
  63. package/dist/daemon-worker.d.ts +2 -2
  64. package/dist/daemon-worker.js +3 -3
  65. package/dist/daemon-worker.js.map +1 -1
  66. package/dist/index.d.ts +2 -1
  67. package/dist/index.d.ts.map +1 -1
  68. package/dist/index.js +1 -1
  69. package/dist/index.js.map +1 -1
  70. package/dist/journal.d.ts.map +1 -1
  71. package/dist/journal.js +93 -6
  72. package/dist/journal.js.map +1 -1
  73. package/dist/journal.test.js +59 -0
  74. package/dist/journal.test.js.map +1 -1
  75. package/dist/machine-auth.test.js +60 -2
  76. package/dist/machine-auth.test.js.map +1 -1
  77. package/dist/object-io.d.ts +37 -1
  78. package/dist/object-io.d.ts.map +1 -1
  79. package/dist/object-io.js +149 -30
  80. package/dist/object-io.js.map +1 -1
  81. package/dist/object-io.test.js +121 -0
  82. package/dist/object-io.test.js.map +1 -1
  83. package/dist/operation-lock.d.ts +8 -8
  84. package/dist/operation-lock.d.ts.map +1 -1
  85. package/dist/operation-lock.js +99 -32
  86. package/dist/operation-lock.js.map +1 -1
  87. package/dist/operation-lock.test.js +51 -4
  88. package/dist/operation-lock.test.js.map +1 -1
  89. package/dist/personal-vault.d.ts.map +1 -1
  90. package/dist/personal-vault.js +8 -2
  91. package/dist/personal-vault.js.map +1 -1
  92. package/dist/personal-vault.test.js +34 -0
  93. package/dist/personal-vault.test.js.map +1 -1
  94. package/dist/prefix-coalesce.d.ts +20 -9
  95. package/dist/prefix-coalesce.d.ts.map +1 -1
  96. package/dist/prefix-coalesce.js +124 -28
  97. package/dist/prefix-coalesce.js.map +1 -1
  98. package/dist/prefix-coalesce.test.js +57 -2
  99. package/dist/prefix-coalesce.test.js.map +1 -1
  100. package/dist/remote-pull.d.ts +8 -3
  101. package/dist/remote-pull.d.ts.map +1 -1
  102. package/dist/remote-pull.js +85 -16
  103. package/dist/remote-pull.js.map +1 -1
  104. package/dist/remote-pull.test.js +213 -2
  105. package/dist/remote-pull.test.js.map +1 -1
  106. package/dist/s3.d.ts +2 -0
  107. package/dist/s3.d.ts.map +1 -1
  108. package/dist/s3.js +197 -116
  109. package/dist/s3.js.map +1 -1
  110. package/dist/s3.test.js +109 -0
  111. package/dist/s3.test.js.map +1 -1
  112. package/dist/scope-shrink.d.ts +3 -2
  113. package/dist/scope-shrink.d.ts.map +1 -1
  114. package/dist/scope-shrink.js +1 -1
  115. package/dist/scope-shrink.js.map +1 -1
  116. package/dist/skill-telemetry.d.ts +1 -1
  117. package/dist/skill-telemetry.d.ts.map +1 -1
  118. package/dist/skill-telemetry.js +69 -9
  119. package/dist/skill-telemetry.js.map +1 -1
  120. package/dist/skill-telemetry.test.js +86 -0
  121. package/dist/skill-telemetry.test.js.map +1 -1
  122. package/dist/sync/event-sync.d.ts +6 -0
  123. package/dist/sync/event-sync.d.ts.map +1 -1
  124. package/dist/sync/event-sync.js +34 -1
  125. package/dist/sync/event-sync.js.map +1 -1
  126. package/dist/sync/event-sync.test.js +73 -0
  127. package/dist/sync/event-sync.test.js.map +1 -1
  128. package/dist/sync/metrics.d.ts +17 -1
  129. package/dist/sync/metrics.d.ts.map +1 -1
  130. package/dist/sync/metrics.js +32 -1
  131. package/dist/sync/metrics.js.map +1 -1
  132. package/dist/sync/metrics.test.js +74 -1
  133. package/dist/sync/metrics.test.js.map +1 -1
  134. package/dist/sync/pull-scope.d.ts.map +1 -1
  135. package/dist/sync/pull-scope.js +15 -7
  136. package/dist/sync/pull-scope.js.map +1 -1
  137. package/dist/sync/push-receiver.d.ts +12 -5
  138. package/dist/sync/push-receiver.d.ts.map +1 -1
  139. package/dist/sync/push-receiver.js +45 -17
  140. package/dist/sync/push-receiver.js.map +1 -1
  141. package/dist/sync/push-receiver.test.js +67 -1
  142. package/dist/sync/push-receiver.test.js.map +1 -1
  143. package/dist/sync-core.d.ts +27 -0
  144. package/dist/sync-core.d.ts.map +1 -0
  145. package/dist/sync-core.js +54 -0
  146. package/dist/sync-core.js.map +1 -0
  147. package/dist/telemetry.d.ts +1 -1
  148. package/dist/telemetry.d.ts.map +1 -1
  149. package/dist/telemetry.js +59 -6
  150. package/dist/telemetry.js.map +1 -1
  151. package/dist/telemetry.test.js +74 -0
  152. package/dist/telemetry.test.js.map +1 -1
  153. package/dist/types.d.ts +8 -0
  154. package/dist/types.d.ts.map +1 -1
  155. package/dist/vault-client.d.ts.map +1 -1
  156. package/dist/vault-client.js +284 -36
  157. package/dist/vault-client.js.map +1 -1
  158. package/dist/vault-client.test.js +59 -0
  159. package/dist/vault-client.test.js.map +1 -1
  160. package/dist/watcher.d.ts +38 -20
  161. package/dist/watcher.d.ts.map +1 -1
  162. package/dist/watcher.js +155 -143
  163. package/dist/watcher.js.map +1 -1
  164. package/dist/watcher.test.js +103 -0
  165. package/dist/watcher.test.js.map +1 -1
  166. package/package.json +1 -1
  167. package/src/bin/sync-runner-company.ts +350 -0
  168. package/src/bin/sync-runner-events.ts +25 -0
  169. package/src/bin/sync-runner-planning.ts +121 -0
  170. package/src/bin/sync-runner-rollup.ts +72 -0
  171. package/src/bin/sync-runner-telemetry.ts +8 -0
  172. package/src/bin/sync-runner-watch-loop.ts +443 -0
  173. package/src/bin/sync-runner-watch-routes.ts +86 -0
  174. package/src/bin/sync-runner.test.ts +298 -11
  175. package/src/bin/sync-runner.ts +99 -1054
  176. package/src/cli/reindex.test.ts +41 -3
  177. package/src/cli/reindex.ts +35 -19
  178. package/src/cli/rescue-classify-ordering.test.ts +81 -0
  179. package/src/cli/rescue-core.ts +400 -165
  180. package/src/cli/share.test.ts +38 -0
  181. package/src/cli/share.ts +420 -693
  182. package/src/cli/sync.test.ts +460 -1
  183. package/src/cli/sync.ts +788 -825
  184. package/src/cognito-auth.test.ts +77 -0
  185. package/src/cognito-auth.ts +73 -11
  186. package/src/daemon-worker.ts +3 -3
  187. package/src/index.ts +8 -0
  188. package/src/journal.test.ts +72 -0
  189. package/src/journal.ts +95 -8
  190. package/src/machine-auth.test.ts +64 -2
  191. package/src/object-io.test.ts +142 -0
  192. package/src/object-io.ts +183 -31
  193. package/src/operation-lock.test.ts +63 -4
  194. package/src/operation-lock.ts +99 -31
  195. package/src/personal-vault.test.ts +42 -0
  196. package/src/personal-vault.ts +8 -2
  197. package/src/prefix-coalesce.test.ts +71 -1
  198. package/src/prefix-coalesce.ts +155 -30
  199. package/src/remote-pull.test.ts +235 -1
  200. package/src/remote-pull.ts +106 -18
  201. package/src/s3.test.ts +126 -0
  202. package/src/s3.ts +237 -122
  203. package/src/scope-shrink.ts +6 -3
  204. package/src/skill-telemetry.test.ts +109 -0
  205. package/src/skill-telemetry.ts +82 -14
  206. package/src/sync/event-sync.test.ts +75 -0
  207. package/src/sync/event-sync.ts +54 -1
  208. package/src/sync/metrics.test.ts +81 -0
  209. package/src/sync/metrics.ts +59 -4
  210. package/src/sync/pull-scope.ts +23 -7
  211. package/src/sync/push-receiver.test.ts +73 -1
  212. package/src/sync/push-receiver.ts +56 -20
  213. package/src/sync-core.ts +58 -0
  214. package/src/telemetry.test.ts +85 -0
  215. package/src/telemetry.ts +69 -6
  216. package/src/types.ts +8 -0
  217. package/src/vault-client.test.ts +74 -0
  218. package/src/vault-client.ts +395 -43
  219. package/src/watcher.test.ts +117 -0
  220. package/src/watcher.ts +215 -174
package/dist/cli/sync.js CHANGED
@@ -11,13 +11,15 @@ import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, to
11
11
  import { readJournal, writeJournal, hashFile, hashSymlinkTarget, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
12
12
  import { PERSONAL_VAULT_MANIFEST_KEY } from "../personal-vault.js";
13
13
  import { buildScopeShrinkPlan, applyScopeShrink, ScopeShrinkBlockedError, ScopeShrinkLargePruneError, } from "../scope-shrink.js";
14
- import { coalescePrefixes, isCoveredByAny } from "../prefix-coalesce.js";
14
+ import { coalescePrefixes, isCoveredByAny, } from "../prefix-coalesce.js";
15
15
  import { createIgnoreFilter } from "../ignore.js";
16
+ import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransferConcurrency, } from "../sync-core.js";
16
17
  import { isEphemeralPath, isMalformedVaultKey } from "./share.js";
17
18
  import { resolveConflict } from "./conflict.js";
18
19
  import { buildConflictId, buildConflictPath, readShortMachineId, } from "../lib/conflict-file.js";
19
20
  import { appendConflictEntry } from "../lib/conflict-index.js";
20
21
  import { reindex } from "./reindex.js";
22
+ import { withOperationLock } from "../operation-lock.js";
21
23
  import { fetchCompanyTombstones, } from "./tombstones.js";
22
24
  /**
23
25
  * Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
@@ -36,6 +38,15 @@ export function resolveAutoPruneCap() {
36
38
  }
37
39
  /** Max time to wait on the best-effort new-files notification POST. */
38
40
  const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
41
+ /**
42
+ * Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
43
+ * an oversized batch wholesale, so the client MUST split a large report into
44
+ * chunks at or under this size — otherwise a first sync with more than this many
45
+ * new files reports none of them, and the same oversized batch re-triggers every
46
+ * sync cycle (wasted work + dropped notifications). Keep in lockstep with the
47
+ * server-side limit.
48
+ */
49
+ const NOTIFY_FILE_ADDED_MAX_BATCH = 1000;
39
50
  /**
40
51
  * Best-effort report of the files that were new to this drive during the sync,
41
52
  * so the HQ Sync app can show a persistent cross-session "new files" history.
@@ -44,17 +55,31 @@ const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
44
55
  * FILE_EVENT rows for the calling user (the one the files are new for). Fully
45
56
  * non-fatal: any error, non-2xx, or timeout is swallowed — the durable signal
46
57
  * is the synced file itself; this is only a notification mirror. Bounded by a
47
- * 5s timeout so a hung endpoint can't stall sync completion. No-op when there
48
- * are no new files.
58
+ * 5s timeout PER request so a hung endpoint can't stall sync completion. No-op
59
+ * when there are no new files.
60
+ *
61
+ * Large reports are split into chunks of at most NOTIFY_FILE_ADDED_MAX_BATCH
62
+ * files (the server's per-report cap). Each chunk is POSTed independently and
63
+ * best-effort, so one failing/oversized batch can never block the others or the
64
+ * sync. Exported only so the chunking can be unit-tested directly.
49
65
  */
50
- async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, files) {
66
+ export async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, files) {
51
67
  if (files.length === 0)
52
68
  return;
69
+ let token;
53
70
  try {
54
- const token = typeof vaultConfig.authToken === "function"
55
- ? await vaultConfig.authToken()
56
- : vaultConfig.authToken;
57
- const base = vaultConfig.apiUrl.replace(/\/+$/, "");
71
+ token =
72
+ typeof vaultConfig.authToken === "function"
73
+ ? await vaultConfig.authToken()
74
+ : vaultConfig.authToken;
75
+ }
76
+ catch (err) {
77
+ logNotifyFailure(err);
78
+ return;
79
+ }
80
+ const base = vaultConfig.apiUrl.replace(/\/+$/, "");
81
+ for (let i = 0; i < files.length; i += NOTIFY_FILE_ADDED_MAX_BATCH) {
82
+ const batch = files.slice(i, i + NOTIFY_FILE_ADDED_MAX_BATCH);
58
83
  const controller = new AbortController();
59
84
  const timer = setTimeout(() => controller.abort(), NOTIFY_FILE_ADDED_TIMEOUT_MS);
60
85
  try {
@@ -67,7 +92,7 @@ async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, file
67
92
  body: JSON.stringify({
68
93
  companyUid,
69
94
  companySlug,
70
- files: files.map((f) => ({
95
+ files: batch.map((f) => ({
71
96
  path: f.path,
72
97
  bytes: f.bytes,
73
98
  ...(f.addedBy ? { addedBy: f.addedBy } : {}),
@@ -76,211 +101,169 @@ async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, file
76
101
  signal: controller.signal,
77
102
  });
78
103
  }
104
+ catch (err) {
105
+ // Best-effort per chunk: never let notification reporting affect the sync
106
+ // result, and a failed chunk must not abort the remaining chunks.
107
+ logNotifyFailure(err);
108
+ }
79
109
  finally {
80
110
  clearTimeout(timer);
81
111
  }
82
112
  }
83
- catch (err) {
84
- // Best-effort: never let notification reporting affect the sync result.
85
- try {
86
- console.error(`[hq-sync] new-files notify report failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
87
- }
88
- catch {
89
- // swallow — logging must never break sync
90
- }
113
+ }
114
+ /** Log a non-fatal notify failure without ever throwing out of the logger. */
115
+ function logNotifyFailure(err) {
116
+ try {
117
+ console.error(`[hq-sync] new-files notify report failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
118
+ }
119
+ catch {
120
+ // swallow — logging must never break sync
91
121
  }
92
122
  }
93
123
  /**
94
124
  * Sync (pull) all allowed files from the entity vault.
95
125
  */
96
126
  export async function sync(options) {
97
- const { company, onConflict, vaultConfig, hqRoot } = options;
127
+ if (options.operationLockAlreadyHeld) {
128
+ return syncWithOperationLockHeld(options);
129
+ }
130
+ return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
131
+ }
132
+ async function syncWithOperationLockHeld(options) {
133
+ const run = await buildPullContext(options);
134
+ const plan = planPull(run);
135
+ emitPullPlan(run.emit, plan);
136
+ const scopePlan = planScopeShrink(run);
137
+ const scopeRun = executeScopeShrink(run, scopePlan);
138
+ const counters = createPullCounters();
139
+ const transferConcurrency = resolveTransferConcurrency();
140
+ const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
141
+ if (conflictRun.abortResult) {
142
+ return conflictRun.abortResult;
143
+ }
144
+ await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
145
+ await emitAndReportNewFiles(run, plan);
146
+ await verifyPlannedJournalTombstones(run, plan);
147
+ executeJournalTombstoneDeletes(run, plan, counters);
148
+ return finalizePullRun(run, plan, scopeRun, counters);
149
+ }
150
+ async function buildPullContext(options) {
151
+ const { company, vaultConfig, hqRoot } = options;
98
152
  const emit = options.onEvent ?? defaultConsoleLogger;
99
- // Resolve company
100
153
  const companyRef = company ?? resolveActiveCompany(hqRoot);
101
154
  if (!companyRef) {
102
155
  throw new Error("No company specified and no active company found. " +
103
156
  "Use --company <slug> or set up .hq/config.json.");
104
157
  }
105
- // Resolve entity context
106
- let ctx = await resolveEntityContext(companyRef, vaultConfig);
107
- // Every company's files land under companies/{slug}/ so fanning out multiple
108
- // companies into the same hqRoot doesn't cross-clobber files with overlapping
109
- // S3 keys (e.g. every company has a .hq/manifest.json). Remote keys stay
110
- // company-relative; the prefix lives only on disk.
111
- // In personalMode the journal slug + S3 keys are person-relative (e.g. "docs/foo.md");
112
- // the local target is `hqRoot` directly, NOT `<hqRoot>/companies/<personSlug>/`. This
113
- // keeps round-trip parity with the Rust personal first-push (Step 7) which sources
114
- // `<hqRoot>/docs/foo.md`.
158
+ const ctx = await resolveEntityContext(companyRef, vaultConfig);
115
159
  const companyRoot = options.personalMode === true
116
160
  ? hqRoot
117
161
  : path.join(hqRoot, "companies", ctx.slug);
118
162
  const shouldSync = createIgnoreFilter(hqRoot);
119
163
  const journalSlug = options.journalSlug ?? ctx.slug;
120
164
  const startedAt = new Date().toISOString();
121
- // Personal-vault callers must never start from an empty journal when only
122
- // the legacy `personal` file exists (mass re-download/etag churn). Seeding
123
- // here — inside the engine — covers every consumer (sync-runner already
124
- // seeds; hq-cli historically didn't, which split the vault's bookkeeping
125
- // across two journal files and re-flagged synced files as conflicts).
126
165
  if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG)
127
166
  migratePersonalVaultJournal();
128
- // Migrate v1 → v2 in place so the scope-shrink / pull-record machinery has
129
- // its fields, and GC any tombstones past the 30-day retention window before
130
- // we re-evaluate orphans (so a long-pruned path can re-download cleanly).
131
167
  const journal = migrateToV2(readJournal(journalSlug));
132
168
  gcTombstones(journal, Date.now());
133
- // ── Effective download scope (US-005) ─────────────────────────────────────
134
- // `all` → prefixSet `[""]`, which `isCoveredByAny` treats as "covers
135
- // everything" — so the download filter and the scope-shrink
136
- // comparison both become no-ops, preserving legacy full-bucket
137
- // behavior bit-for-bit.
138
- // `shared`/`custom` → the coalesced, company-relative prefix set the runner
139
- // resolved. An empty set means "nothing in scope" → download
140
- // nothing (the runner falls back to `all` on resolution errors, so
141
- // empty here is an intentional "nothing shared", never a failure).
142
169
  const syncMode = options.syncMode ?? "all";
143
170
  const currentPrefixSet = syncMode === "all" ? [""] : coalescePrefixes(options.prefixSet ?? []);
144
- // Authorship guard input (scope-shrink): the caller's own Cognito sub,
145
- // injected by the entry point (the runner sources it from its decoded
146
- // idToken claims — the same sub stamped onto uploads as `created-by-sub`).
147
- // Undefined degrades safely: own-author files lose their special shield, but
148
- // the `protectUnknownAuthors` conservative path below still prevents a
149
- // routine sync from deleting anything it can't prove is foreign.
150
- const callerSub = options.callerSub;
151
- let filesDownloaded = 0;
152
- let bytesDownloaded = 0;
153
- let filesSkipped = 0;
154
- let conflicts = 0;
155
- let filesTombstoned = 0;
156
- let filesOutOfScope = 0;
157
- const conflictPaths = [];
158
- // List all remote files (IAM session policy filters at the AWS layer)
159
171
  const remoteFiles = await listRemoteFiles(ctx);
160
- // Fetch the company's FILE_TOMBSTONE records so the planner can suppress
161
- // resurrection of an intentionally-deleted object (delete-resync). Done in
162
- // parallel intent with the LIST above conceptually, but kept serial here for
163
- // a clean read of `ctx`; best-effort — a failed read degrades to an empty map
164
- // (no suppression), preserving the pre-fix behavior. ctx.uid is the verified
165
- // companyUid the tombstone rows are keyed under.
166
- //
167
- // SKIP for the personal vault: its `ctx.uid` is a personUid (`prs_…`), but
168
- // `GET /v1/files/tombstones?company=…` is COMPANY-scoped server-side
169
- // (findCallerWithMembership), so a personal-vault request resolves
170
- // `company=prs_…` to no membership and is correctly rejected with
171
- // `403 "No active membership for caller in company prs_…"`. That 403 is
172
- // benign for the pull (it already degrades to the empty map below), but
173
- // hq-pro captures EVERY one as a Sentry warning — the per-personal-vault
174
- // no-membership cluster (one Sentry issue per signed-in user). Personal-vault
175
- // delete-resync was never a committed feature and there is no person-scoped
176
- // tombstone path, so for the personal target we skip the fetch and use an
177
- // empty map — byte-for-byte the current degraded behavior, minus the 403 spam.
178
- // FUTURE FOLLOW-UP (not built here): if personal-vault delete-resync is
179
- // wanted, it needs a real person-scoped tombstone endpoint + client read.
180
- const tombstones = options.personalMode === true
172
+ const fileTombstones = options.personalMode === true
181
173
  ? new Map()
182
174
  : await fetchCompanyTombstones(vaultConfig, ctx.uid);
183
- // Stage 1: classify every remote file against the journal + local disk.
184
- // Hashing happens here (not in the transfer loop) so the plan event below
185
- // carries an accurate denominator before any progress events fire.
186
- const plan = computePullPlan(remoteFiles, journal, companyRoot, shouldSync, options.personalMode === true, options.includeLocalCompanies === true, options.teamSyncedSlugs ?? null, currentPrefixSet, tombstones);
175
+ return {
176
+ options,
177
+ companyRef,
178
+ vaultConfig,
179
+ hqRoot,
180
+ emit,
181
+ ctx,
182
+ companyRoot,
183
+ shouldSync,
184
+ journalSlug,
185
+ startedAt,
186
+ journal,
187
+ remoteFiles,
188
+ syncMode,
189
+ currentPrefixSet,
190
+ fileTombstones,
191
+ };
192
+ }
193
+ function planPull(run) {
194
+ return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.includeLocalCompanies === true, run.options.teamSyncedSlugs ?? null, run.currentPrefixSet, run.fileTombstones);
195
+ }
196
+ function emitPullPlan(emit, plan) {
187
197
  emit({
188
198
  type: "plan",
189
199
  filesToDownload: plan.filesToDownload,
190
200
  bytesToDownload: plan.bytesToDownload,
191
- // sync() is pull-only; push counts are sourced from share()'s plan event.
192
201
  filesToUpload: 0,
193
202
  bytesToUpload: 0,
194
203
  filesToSkip: plan.filesToSkip,
195
204
  filesToConflict: plan.filesToConflict,
196
- // Authoritative FILE_TOMBSTONE suppressions (delete-resync) are the only
197
- // deletes known at plan time; the journal-vs-LIST tombstones are
198
- // HEAD-verified later and surfaced via the final filesTombstoned count.
199
205
  filesToDelete: plan.filesToTombstoneDelete,
200
206
  });
201
- // ── Scope-shrink cleanup (US-005) ─────────────────────────────────────────
202
- // If the effective scope narrowed since the last pull, files that were
203
- // pulled under the old scope but fall outside the new one are orphans. We
204
- // delete only CLEAN orphans (provably unchanged since last sync); dirty
205
- // (locally-modified) orphans are sacred. By default a dirty orphan aborts
206
- // the leg with a structured error the CLI renders; `forceScopeShrink` keeps
207
- // dirty files on disk and only tombstones their journal entries.
208
- //
209
- // `companyRoot` is passed as the module's `hqRoot` so its `path.join(root,
210
- // key)` resolves company-relative journal keys correctly (the scope-shrink
211
- // module is namespace-agnostic — root + keys + prefixSet must simply agree).
212
- //
213
- // Note: this is the durable selective-download fix for OWNERS. An owner's
214
- // STS is wide (role-bypass), so the remote LIST returns everything and the
215
- // AWS layer never narrows the pull. This client-side shrink is what makes
216
- // `hq sync mode shared` actually stick across re-syncs for an owner.
217
- const lastRecord = lastPullRecord(journal, ctx.uid);
218
- // A missing record, or a v1-migrated record with an empty prefixSet, means
219
- // "no recorded scope" → treat the last scope as full-bucket `all` (`[""]`),
220
- // per the PullRecord.prefixSet contract in types.ts.
207
+ }
208
+ function createPullCounters() {
209
+ return {
210
+ filesDownloaded: 0,
211
+ bytesDownloaded: 0,
212
+ filesSkipped: 0,
213
+ conflicts: 0,
214
+ filesTombstoned: 0,
215
+ filesOutOfScope: 0,
216
+ conflictPaths: [],
217
+ };
218
+ }
219
+ function planScopeShrink(run) {
220
+ const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
221
221
  const lastPrefixSet = lastRecord && lastRecord.prefixSet.length > 0
222
222
  ? lastRecord.prefixSet
223
223
  : [""];
224
224
  const shrinkPlan = buildScopeShrinkPlan({
225
- journal,
226
- hqRoot: companyRoot,
225
+ journal: run.journal,
226
+ hqRoot: run.companyRoot,
227
227
  lastPrefixSet,
228
- currentPrefixSet,
229
- callerSub,
230
- // Automatic pull: never auto-prune content the caller authored, and never
231
- // make a destructive guess about unknown-author (legacy) orphans. The
232
- // explicit `hq sync narrow` ritual opts out of the unknown-author shield.
228
+ currentPrefixSet: run.currentPrefixSet,
229
+ callerSub: run.options.callerSub,
233
230
  protectUnknownAuthors: true,
234
231
  });
235
- // Policy: the background menubar runner ("auto-recover") can take no
236
- // interactive flag, so it must never throw on a shrink — it self-heals
237
- // non-destructively (dirty kept on disk + un-tracked, clean quarantined).
238
- // A foreground `hq sync` ("block", the default) keeps the protective gate
239
- // but renders FOLLOWABLE advice. `autoRecover` implies force (proceed) and
240
- // bypasses the bulk-prune cap (quarantine is non-destructive, so a large
241
- // recovery move is safe). DEV-1768.
242
- const scopeShrinkPolicy = options.scopeShrinkPolicy ?? "block";
232
+ const scopeShrinkPolicy = run.options.scopeShrinkPolicy ?? "block";
243
233
  const autoRecover = scopeShrinkPolicy === "auto-recover";
244
234
  const adviceContext = autoRecover ? "runner" : "cli";
245
- const effectiveForce = options.forceScopeShrink === true || autoRecover;
235
+ const effectiveForce = run.options.forceScopeShrink === true || autoRecover;
236
+ return {
237
+ lastRecord,
238
+ shrinkPlan,
239
+ autoRecover,
240
+ adviceContext,
241
+ effectiveForce,
242
+ };
243
+ }
244
+ function executeScopeShrink(run, scopePlan) {
245
+ const { lastRecord, shrinkPlan, adviceContext, effectiveForce } = scopePlan;
246
246
  if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
247
- throw new ScopeShrinkBlockedError(ctx.uid, lastRecord?.syncMode ?? "unknown", syncMode, shrinkPlan.dirty, shrinkPlan.clean, adviceContext);
247
+ throw new ScopeShrinkBlockedError(run.ctx.uid, lastRecord?.syncMode ?? "unknown", run.syncMode, shrinkPlan.dirty, shrinkPlan.clean, adviceContext);
248
248
  }
249
- // Bulk guard: refuse to auto-move more than the safety cap of CLEAN files in
250
- // a single foreground sync. A deliberate large narrow goes through
251
- // `hq sync narrow --apply` (its own confirmation); `--force-scope-shrink` (or
252
- // raising HQ_SYNC_MAX_AUTO_PRUNE) overrides. Cap of 0 = unlimited. Skipped
253
- // under auto-recover — quarantine is non-destructive so a big recovery is
254
- // safe, and the runner has no way to act on a thrown cap. The engine moves
255
- // nothing when it throws here.
256
249
  const autoPruneCap = resolveAutoPruneCap();
257
250
  if (!effectiveForce &&
258
251
  autoPruneCap > 0 &&
259
252
  shrinkPlan.clean.length > autoPruneCap) {
260
- throw new ScopeShrinkLargePruneError(ctx.uid, syncMode, shrinkPlan.clean.length, autoPruneCap, adviceContext);
253
+ throw new ScopeShrinkLargePruneError(run.ctx.uid, run.syncMode, shrinkPlan.clean.length, autoPruneCap, adviceContext);
261
254
  }
262
- // Clean orphans are QUARANTINED (moved into `.hq/scope-quarantine/<slug>/`,
263
- // recoverable), never silently deleted — a background sync purging local
264
- // files unannounced was DEV-1768 fix #3. The quarantine root lives under the
265
- // real HQ root's `.hq/` (outside `companyRoot` and never pushed), so moved
266
- // files don't round-trip back through S3.
267
- const scopeQuarantineRoot = path.join(hqRoot, ".hq", "scope-quarantine", journalSlug);
255
+ const scopeQuarantineRoot = path.join(run.hqRoot, ".hq", "scope-quarantine", run.journalSlug);
268
256
  const shrinkResult = applyScopeShrink({
269
- journal,
257
+ journal: run.journal,
270
258
  plan: shrinkPlan,
271
- hqRoot: companyRoot,
259
+ hqRoot: run.companyRoot,
272
260
  forceScopeShrink: effectiveForce,
273
261
  reason: "scope_shrink",
274
262
  cleanDisposition: "quarantine",
275
263
  quarantineRoot: scopeQuarantineRoot,
276
264
  });
277
- // Surface each affected orphan explicitly (named path) so the prune is never
278
- // silent. Quarantined clean files render as `deleted: true` (removed from the
279
- // working tree, recoverable in quarantine); dirty files KEPT on disk render
280
- // as a non-deletion notice so the operator knows they were un-tracked, not
281
- // removed. The Rust menubar parser already handles `deleted: true`.
282
265
  for (const relPath of shrinkResult.quarantinedPaths) {
283
- emit({
266
+ run.emit({
284
267
  type: "progress",
285
268
  path: relPath,
286
269
  bytes: 0,
@@ -289,7 +272,7 @@ export async function sync(options) {
289
272
  });
290
273
  }
291
274
  for (const relPath of shrinkResult.removedPaths) {
292
- emit({
275
+ run.emit({
293
276
  type: "progress",
294
277
  path: relPath,
295
278
  bytes: 0,
@@ -298,477 +281,314 @@ export async function sync(options) {
298
281
  });
299
282
  }
300
283
  for (const relPath of shrinkResult.dirtyKeptPaths) {
301
- emit({
284
+ run.emit({
302
285
  type: "progress",
303
286
  path: relPath,
304
287
  bytes: 0,
305
288
  message: "scope-narrowed: locally-modified file KEPT on disk, un-tracked from sync (outside scope)",
306
289
  });
307
290
  }
308
- // "Removed from the working tree" = deleted OR quarantined; both vacate the
309
- // file's original path. Reported as `scopeOrphansRemoved` for back-compat.
310
- const scopeOrphansRemoved = shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined;
311
- // Stage 2: execute the plan. Per-item branching mirrors the pre-refactor
312
- // inline loop; the only structural change is that classification has
313
- // already happened (so `localHash` is reused instead of re-hashing).
314
- //
315
- // 5.36.0: download items go through a bounded-concurrent pool
316
- // (`TRANSFER_CONCURRENCY`, default 16, tunable via
317
- // `HQ_SYNC_TRANSFER_CONCURRENCY`) for 4-8x speedup on transfer-heavy
318
- // syncs. Conflict items stay serial — they may prompt the operator
319
- // and the abort path must short-circuit before any further work. We
320
- // partition the plan items into "conflict (serial)" and "download
321
- // (parallel)" buckets and run the serial pass first; the parallel pass
322
- // only runs if no conflict aborted.
323
- //
324
- // Per-file `progress` events fire at the moment each individual download
325
- // settles (inside the pool wrapper), NOT in plan-walk order. The cross-
326
- // file interleave is acceptable: the menubar stream parser already
327
- // handles per-company interleave, and the same shape applies within a
328
- // single company's pool. Per-file event-count correctness is preserved
329
- // (one progress per download, one error per failure).
330
- const TRANSFER_CONCURRENCY = (() => {
331
- const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
332
- if (raw === undefined || raw === "")
333
- return 16;
334
- const parsed = Number.parseInt(raw, 10);
335
- return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
336
- })();
337
- // First pass: serial walk for non-download outcomes (skips + conflicts).
338
- // Conflicts may set `aborted = true` and short-circuit the whole pull;
339
- // we detect that and skip the parallel pass. Download items are
340
- // collected into `downloadItems[]` for the pool pass below.
291
+ return {
292
+ shrinkPlan,
293
+ shrinkResult,
294
+ scopeOrphansRemoved: shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined,
295
+ };
296
+ }
297
+ async function refreshRunContextIfExpiring(run) {
298
+ if (isExpiringSoon(run.ctx.expiresAt)) {
299
+ run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
300
+ }
301
+ }
302
+ async function executeConflictExecutor(run, plan, scopeRun, counters) {
341
303
  const downloadItems = [];
342
- let aborted = false;
343
- let abortResult = null;
344
304
  for (const item of plan.items) {
345
- if (aborted)
346
- break;
347
305
  if (item.action === "skip-ignored" ||
348
306
  item.action === "skip-personal-mode" ||
349
307
  item.action === "skip-unchanged" ||
350
308
  item.action === "skip-local-only") {
351
- filesSkipped++;
309
+ counters.filesSkipped++;
352
310
  continue;
353
311
  }
354
312
  if (item.action === "skip-excluded-policy") {
355
- // Policy-excluded items count separately from `filesSkipped` so the
356
- // pull result mirrors the push side's `filesExcludedByPolicy`
357
- // counter — `filesSkipped` stays a measure of "unchanged on this
358
- // run", not a catch-all for everything we didn't download.
359
313
  continue;
360
314
  }
361
315
  if (item.action === "skip-out-of-scope") {
362
- // Outside the effective `syncMode` scope (US-005). Counted on its own
363
- // axis so `filesSkipped` keeps meaning "unchanged on this run" — these
364
- // are "deliberately not downloaded because of your sync scope".
365
- filesOutOfScope++;
316
+ counters.filesOutOfScope++;
366
317
  continue;
367
318
  }
368
319
  if (item.action === "tombstone-delete") {
369
- // Authoritative FILE_TOMBSTONE delete (delete-resync): the remote object
370
- // is present but a tombstone marks the key intentionally deleted and it is
371
- // not a newer re-create. Delete any local copy and drop the journal entry
372
- // so it stays gone — the mirror of the journal-vs-LIST tombstone executor
373
- // below, but WITHOUT the HEAD-verify (the remote object is present by
374
- // definition; the FILE_TOMBSTONE is the deletion authority). The planner
375
- // already routed any divergent local copy to `conflict`, so a local file
376
- // reaching here matches the deleted baseline and is safe to remove.
377
- const tombstoneKey = item.remoteFile.key;
378
- // Same Windows-backslash landmine guard as the journal-tombstone executor:
379
- // a malformed key must never reach fs.unlinkSync (path.join collapses the
380
- // backslashes onto a REAL POSIX file). Drop the poisoned journal entry
381
- // without touching disk.
382
- if (isMalformedVaultKey(tombstoneKey)) {
383
- removeEntry(journal, tombstoneKey);
384
- continue;
385
- }
386
- try {
387
- const lstat = fs.lstatSync(item.localPath);
388
- if (lstat.isSymbolicLink() || lstat.isFile()) {
389
- fs.unlinkSync(item.localPath);
390
- }
391
- // A directory at the key: don't recursively rm-rf the operator's dir;
392
- // just drop the journal entry (safe-by-default, same as the other path).
393
- }
394
- catch (err) {
395
- const code = err && typeof err === "object" && "code" in err
396
- ? err.code
397
- : undefined;
398
- // ENOENT → local already absent (the common case: a fresh machine that
399
- // never held the file, or a prior pull already removed it) → drop the
400
- // journal entry and converge. Other errors (EACCES/EPERM/…) leave the
401
- // file in place; surface and KEEP the journal entry so the next sync
402
- // retries rather than forgetting the delete.
403
- if (code !== "ENOENT") {
404
- emit({
405
- type: "error",
406
- path: tombstoneKey,
407
- message: `tombstone-suppress unlink failed: ${err instanceof Error ? err.message : String(err)}`,
408
- });
409
- continue;
410
- }
411
- }
412
- removeEntry(journal, tombstoneKey);
413
- filesTombstoned++;
414
- emit({ type: "progress", path: tombstoneKey, bytes: 0 });
320
+ executeFileTombstoneDelete(run, item, counters);
415
321
  continue;
416
322
  }
417
323
  if (item.action === "download") {
418
324
  downloadItems.push(item);
419
325
  continue;
420
326
  }
421
- const { remoteFile, localPath } = item;
422
- // Auto-refresh context if credentials expiring (kept in execute phase
423
- // because Stage 1 is fast — no need to refresh just to classify).
424
- if (isExpiringSoon(ctx.expiresAt)) {
425
- ctx = await refreshEntityContext(companyRef, vaultConfig);
327
+ const abortResult = await executeConflictItem(run, plan, scopeRun, counters, downloadItems, item);
328
+ if (abortResult) {
329
+ return { downloadItems, abortResult };
426
330
  }
427
- if (item.action === "conflict") {
428
- // ── Convergence guard ────────────────────────────────────────────
429
- // The planner flags a conflict purely from journal-relative deltas:
430
- // local hash != journal hash AND remote etag != journal etag. It can
431
- // NEVER compare local bytes against remote bytes directly — the remote
432
- // LIST only carries {key, size, etag, lastModified}, and ListObjectsV2
433
- // returns no content hash. So when the journal baseline goes stale,
434
- // both deltas fire even though local and remote are byte-for-byte
435
- // identical. Stale-baseline triggers seen in the wild: a shared-journal
436
- // cross-root collision (personal vault + companies/personal sharing one
437
- // journal), an mtime-rounding fast-path miss (journal stamps 1700..351,
438
- // the FS returns 1700..350.96, the `===` fast-path misses and re-hashes
439
- // — harmless on its own but combines with the etag delta), KMS/multipart
440
- // etag churn on a no-op re-upload, a second machine advancing S3 + its
441
- // own journal, or a manual revert. Materializing such a false positive
442
- // as a conflict litters a useless byte-identical `.conflict-*` mirror
443
- // and, under `--on-conflict abort`, halts the WHOLE sync over zero real
444
- // divergence. (One live run produced 140 byte-identical mirrors this way.)
445
- //
446
- // We have no remote content hash up front, so prove convergence by
447
- // fetching the remote bytes once — to the very path the conflict mirror
448
- // would occupy and hashing them. Identical bytes are not a conflict:
449
- // re-stamp the journal baseline (so neither side looks changed next run)
450
- // and skip. Genuine divergence reuses the already-fetched bytes as the
451
- // inspection mirror, so the common keep/skip path costs no extra I/O.
452
- const detectedAt = new Date().toISOString();
453
- const machineId = readShortMachineId(hqRoot);
454
- const originalRelative = path.relative(hqRoot, localPath);
455
- const conflictRelative = buildConflictPath(originalRelative, detectedAt, machineId);
456
- const conflictAbs = path.join(hqRoot, conflictRelative);
457
- let remoteFetched = false;
458
- let converged = false;
459
- try {
460
- await downloadFile(ctx, remoteFile.key, conflictAbs);
461
- remoteFetched = true;
462
- // Hash the fetched remote exactly the way the planner hashed local
463
- // (symlink-aware) so the two hashes are directly comparable. A
464
- // symlink record round-trips to a symlink on disk; hashing its
465
- // target string matches `hashSymlinkTarget(localPath)`.
466
- const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
467
- ? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
468
- : hashFile(conflictAbs);
469
- converged = remoteHash === item.localHash;
470
- }
471
- catch (probeErr) {
472
- // Couldn't fetch or hash the remote — fail safe by falling through to
473
- // the conventional conflict path (converged stays false). No mirror
474
- // is on disk in this case.
475
- emit({
476
- type: "error",
477
- path: remoteFile.key,
478
- message: `conflict convergence probe failed: ${probeErr instanceof Error ? probeErr.message : String(probeErr)}`,
479
- });
480
- }
481
- if (converged) {
482
- // False positive: remote == local. Drop the byte-identical mirror and
483
- // re-stamp the baseline (current localHash + current remoteEtag) so
484
- // the next sync sees "no change on either side". Counts as a skip.
485
- if (remoteFetched) {
486
- try {
487
- fs.rmSync(conflictAbs, { force: true });
488
- }
489
- catch {
490
- /* best-effort cleanup; a stray identical mirror is harmless */
491
- }
492
- }
493
- updateEntry(journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
494
- emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
495
- filesSkipped++;
496
- continue;
497
- }
498
- // ── Genuine divergence ───────────────────────────────────────────
499
- conflicts++;
500
- conflictPaths.push(remoteFile.key);
501
- const resolution = await resolveConflict({
502
- path: remoteFile.key,
503
- localHash: item.localHash,
504
- remoteModified: remoteFile.lastModified,
505
- // Use the lstat-mtime captured by the planner — statSync
506
- // here would follow a dangling symlink and throw ENOENT,
507
- // aborting the pull before resolveConflict could prompt.
508
- localModified: item.localMtime,
509
- direction: "pull",
510
- }, onConflict);
511
- emit({
512
- type: "conflict",
513
- path: remoteFile.key,
514
- direction: "pull",
515
- resolution,
516
- });
517
- // The remote bytes were already fetched to `conflictAbs` by the
518
- // convergence probe. For "keep"/"skip" they become the
519
- // `<original>.conflict-<ts>-<machine>.<ext>` inspection mirror — just
520
- // index it (no second download). For "abort" (user gave up) and
521
- // "overwrite" (cloud bytes are about to replace local) the mirror is
522
- // redundant, so discard it. Best-effort: failure here only emits an
523
- // error, doesn't break the sync.
524
- if (resolution !== "abort" && resolution !== "overwrite") {
525
- if (remoteFetched) {
526
- try {
527
- appendConflictEntry(hqRoot, {
528
- id: buildConflictId(originalRelative, detectedAt),
529
- originalPath: originalRelative,
530
- conflictPath: conflictRelative,
531
- detectedAt,
532
- side: "pull",
533
- machineId,
534
- localHash: item.localHash,
535
- remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
536
- });
537
- }
538
- catch (mirrorErr) {
539
- emit({
540
- type: "error",
541
- path: remoteFile.key,
542
- message: `conflict mirror index write failed: ${mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)}`,
543
- });
544
- }
545
- }
546
- // If the probe download failed (!remoteFetched) there is no mirror on
547
- // disk; the probe already emitted the error. The conflict is still
548
- // surfaced and journal-stamped below so it doesn't re-fire silently.
549
- }
550
- else if (remoteFetched) {
551
- try {
552
- fs.rmSync(conflictAbs, { force: true });
553
- }
554
- catch {
555
- /* best-effort; a leftover mirror is cosmetic, not corrupting */
556
- }
557
- }
558
- if (resolution === "abort") {
559
- emit({ type: "new-files", files: [] });
560
- writeJournal(journalSlug, journal);
561
- aborted = true;
562
- abortResult = {
563
- filesDownloaded,
564
- bytesDownloaded,
565
- filesSkipped,
566
- conflicts,
567
- conflictPaths,
568
- aborted: true,
569
- newFiles: plan.newFiles,
570
- newFilesCount: plan.newFilesCount,
571
- filesExcludedByPolicy: plan.filesExcludedByPolicy,
572
- // Abort short-circuits before the tombstone loop runs; report
573
- // 0 so the field shape stays stable for consumers that
574
- // destructure it.
575
- filesTombstoned: 0,
576
- // Scope-shrink ran before execution, so its counts are real even on
577
- // a conflict abort. `filesOutOfScope` reflects how far the serial
578
- // pass got before the abort; that's acceptable for an abort result.
579
- filesOutOfScope,
580
- scopeOrphansRemoved,
581
- scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
582
- };
583
- break;
584
- }
585
- if (resolution === "keep" || resolution === "skip") {
586
- filesSkipped++;
587
- // Stamp the journal with the new baseline so the same conflict
588
- // doesn't re-fire on every subsequent sync. After "keep", local
589
- // wins — the user has accepted that the cloud version we just
590
- // mirrored is what cloud is at this etag, and they don't want
591
- // it. Recording (current localHash + current remoteEtag) tells
592
- // the next sync "no change on either side" until something new
593
- // diverges. Without this, both `localChanged` and `remoteChanged`
594
- // stay true forever and the conflict is sticky.
595
- // Stamp from planner-captured size (symlink-aware), NOT
596
- // statSync — which would follow a dangling symlink and
597
- // throw ENOENT, get swallowed, and leave the journal
598
- // stale so this conflict would re-fire on every sync
599
- // forever. localSize is sourced from the same lstat that
600
- // computed localMtime + localHash above.
601
- updateEntry(journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
602
- continue;
603
- }
604
- // "overwrite" falls through to download — re-route through the pool
605
- // so it benefits from parallelism too. Synthesize a download item
606
- // pointing at the same remoteFile/localPath; isNew=false because
607
- // there was a conflict-eligible local file present.
608
- downloadItems.push({
609
- action: "download",
610
- remoteFile,
611
- localPath,
612
- isNew: false,
331
+ }
332
+ return { downloadItems, abortResult: null };
333
+ }
334
+ function executeFileTombstoneDelete(run, item, counters) {
335
+ const tombstoneKey = item.remoteFile.key;
336
+ const tombstonePath = resolveContainedVaultPath(run.companyRoot, tombstoneKey);
337
+ if (tombstonePath === null)
338
+ return;
339
+ try {
340
+ const lstat = fs.lstatSync(tombstonePath);
341
+ if (tombstoneTargetDiverged(run.journal, tombstoneKey, tombstonePath, lstat)) {
342
+ return;
343
+ }
344
+ if (lstat.isSymbolicLink() || lstat.isFile()) {
345
+ fs.unlinkSync(tombstonePath);
346
+ }
347
+ }
348
+ catch (err) {
349
+ const code = err && typeof err === "object" && "code" in err
350
+ ? err.code
351
+ : undefined;
352
+ if (code !== "ENOENT") {
353
+ run.emit({
354
+ type: "error",
355
+ path: tombstoneKey,
356
+ message: `tombstone-suppress unlink failed: ${err instanceof Error ? err.message : String(err)}`,
613
357
  });
614
- continue;
358
+ return;
615
359
  }
616
360
  }
617
- // Early-return on conflict abort BEFORE running the parallel download
618
- // pool — the abort intent is "stop the pull now", not "stop new work but
619
- // finish what's in flight". Since the pool hasn't started yet, this is
620
- // a clean drain (zero items in flight) by construction.
621
- if (aborted && abortResult) {
622
- return abortResult;
361
+ removeEntry(run.journal, tombstoneKey);
362
+ counters.filesTombstoned++;
363
+ run.emit({ type: "progress", path: tombstoneKey, bytes: 0 });
364
+ }
365
+ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems, item) {
366
+ const { remoteFile, localPath } = item;
367
+ await refreshRunContextIfExpiring(run);
368
+ const detectedAt = new Date().toISOString();
369
+ const machineId = readShortMachineId(run.hqRoot);
370
+ const originalRelative = path.relative(run.hqRoot, localPath);
371
+ const conflictRelative = buildConflictPath(originalRelative, detectedAt, machineId);
372
+ const conflictAbs = path.join(run.hqRoot, conflictRelative);
373
+ const conflictKey = toPosixKey(path.relative(run.companyRoot, conflictAbs));
374
+ if (!isDownloadWritePathStillContained(run.companyRoot, conflictKey, conflictAbs)) {
375
+ counters.filesSkipped++;
376
+ run.emit({
377
+ type: "error",
378
+ path: remoteFile.key,
379
+ message: "conflict mirror skipped: local parent escaped the sync root",
380
+ });
381
+ return null;
382
+ }
383
+ let remoteFetched = false;
384
+ let converged = false;
385
+ try {
386
+ const downloaded = await downloadFile(run.ctx, remoteFile.key, conflictAbs);
387
+ remoteFetched = true;
388
+ const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
389
+ ? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
390
+ : (downloaded.contentHash ?? hashFile(conflictAbs));
391
+ converged = remoteHash === item.localHash;
623
392
  }
624
- // ── Parallel download pool (5.36.0) ───────────────────────────────────
625
- // Bounded concurrency: TRANSFER_CONCURRENCY simultaneous downloads. Same
626
- // race-based shape as the HEAD-verify pool above but applied to the body
627
- // of each download. Per-item progress events fire at file-settle time
628
- // (inside the wrapper), so cross-file interleave is expected and the
629
- // menubar's stream parser already handles it.
630
- if (downloadItems.length > 0) {
631
- // Batch pre-mint GET URLs for every download in one shot (chunked server-
632
- // side) so the pool below — and the new-files HEAD enrichment that follows,
633
- // which re-reads the same keys — reuse them instead of presigning per file.
634
- // On a large initial pull this is the difference between ~ceil(N/100)
635
- // presign calls and N (which would 429 past the 100-req/hr limit). No-op
636
- // on the S3 SDK transport; best-effort (failure falls back to per-file).
637
- await primeObjectTransport(ctx, "get", downloadItems.map((d) => d.remoteFile.key));
638
- const queue = [...downloadItems];
639
- const inFlight = new Set();
640
- const downloadOne = async (downloadItem) => {
641
- const { remoteFile, localPath } = downloadItem;
642
- // Auto-refresh context if credentials expiring. Each task checks
643
- // independently — refresh is idempotent on the same context object.
644
- if (isExpiringSoon(ctx.expiresAt)) {
645
- ctx = await refreshEntityContext(companyRef, vaultConfig);
646
- }
393
+ catch (probeErr) {
394
+ run.emit({
395
+ type: "error",
396
+ path: remoteFile.key,
397
+ message: `conflict convergence probe failed: ${probeErr instanceof Error ? probeErr.message : String(probeErr)}`,
398
+ });
399
+ }
400
+ if (converged) {
401
+ if (remoteFetched) {
647
402
  try {
648
- const { metadata } = await downloadFile(ctx, remoteFile.key, localPath);
649
- const author = metadata?.["created-by"] ?? null;
650
- // Author sub for the scope-shrink authorship guard — same field the
651
- // upload side stamps, read straight off the GET response metadata.
652
- const createdBySub = metadata?.["created-by-sub"];
653
- // Symlink records materialize as real symlinks on disk. lstat
654
- // (does not follow) lets us detect that case so the journal stamp
655
- // mirrors what the push side would emit on the next tick:
656
- // hash = sha256(readlink target string)
657
- // size = 0
658
- // Without this check, hashFile would follow the link and stamp the
659
- // target file's contents — a value the next push would never
660
- // produce — which makes skipUnchanged perpetually re-upload every
661
- // symlink, defeating the point of the gate.
662
- const localLstat = fs.lstatSync(localPath);
663
- const isLocalSymlink = localLstat.isSymbolicLink();
664
- const hash = isLocalSymlink
665
- ? hashSymlinkTarget(fs.readlinkSync(localPath))
666
- : hashFile(localPath);
667
- const size = isLocalSymlink ? 0 : fs.statSync(localPath).size;
668
- // Capture the listing's ETag so subsequent syncs can detect remote
669
- // drift independently of mtime drift. Stamp mtimeMs from localLstat
670
- // (5.36.0) so the next push planner's lstat fast-path can skip the
671
- // SHA256 for this file without reading its bytes.
672
- //
673
- // 5.37.0 ordering invariant: downloadFile applies hq-mtime via
674
- // utimesSync AFTER its byte write but BEFORE returning, and this
675
- // lstat runs AFTER downloadFile resolves — so localLstat.mtimeMs
676
- // already reflects the source-stamped mtime, not the wall-clock
677
- // write-time. The journal therefore matches what the next push's
678
- // lstat fast-path will see, and the file is correctly skipped on
679
- // re-sync instead of being hashed every tick. Do not move this
680
- // lstat earlier; do not stamp the journal from any pre-download
681
- // mtime.
682
- updateEntry(journal, remoteFile.key, hash, size, "down", remoteFile.etag, localLstat.mtimeMs, createdBySub);
683
- // Attach message from the prior journal entry if present (set by a
684
- // previous `share` operation that included a --message).
685
- const priorEntry = getEntry(journal, remoteFile.key);
686
- const remoteJournalMessage = priorEntry?.message;
687
- emit({
688
- type: "progress",
689
- path: remoteFile.key,
690
- bytes: size,
691
- ...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
692
- ...(author ? { author } : {}),
693
- });
694
- filesDownloaded++;
695
- bytesDownloaded += size;
403
+ fs.rmSync(conflictAbs, { force: true });
696
404
  }
697
- catch (err) {
698
- // STS session policy may deny access to some paths — this is expected
699
- // for guest members with allowedPrefixes
700
- if (isAccessDenied(err)) {
701
- filesSkipped++;
702
- }
703
- else {
704
- emit({
705
- type: "error",
706
- path: remoteFile.key,
707
- message: err instanceof Error ? err.message : String(err),
708
- });
709
- }
405
+ catch {
406
+ /* best-effort cleanup; a stray identical mirror is harmless */
710
407
  }
711
- };
712
- // Codex P1 (5.36.x): worker promises wrapped in .catch so an
713
- // unhandled rejection inside downloadOne (e.g. refreshEntityContext
714
- // before the per-item try/catch, or lstatSync after the download
715
- // succeeded but before journal stamping) cannot escape
716
- // `Promise.race(inFlight)` and unwind the drain mid-flight. Without
717
- // this wrap, sibling downloads kept running after share()/sync()
718
- // had already failed, their files materialized on disk without
719
- // matching journal entries, and the next sync re-downloaded
720
- // everything. Errors are collected and surfaced after the pool
721
- // fully drains — see workerErrors throw below.
722
- const workerErrors = [];
723
- while (queue.length > 0 || inFlight.size > 0) {
724
- while (inFlight.size < TRANSFER_CONCURRENCY && queue.length > 0) {
725
- const downloadItem = queue.shift();
726
- const p = downloadOne(downloadItem)
727
- .catch((err) => {
728
- workerErrors.push(err instanceof Error ? err : new Error(String(err)));
729
- })
730
- .finally(() => {
731
- inFlight.delete(p);
408
+ }
409
+ updateEntry(run.journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
410
+ run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
411
+ counters.filesSkipped++;
412
+ return null;
413
+ }
414
+ counters.conflicts++;
415
+ counters.conflictPaths.push(remoteFile.key);
416
+ const resolution = await resolveConflict({
417
+ path: remoteFile.key,
418
+ localHash: item.localHash,
419
+ remoteModified: remoteFile.lastModified,
420
+ localModified: item.localMtime,
421
+ direction: "pull",
422
+ }, run.options.onConflict);
423
+ run.emit({
424
+ type: "conflict",
425
+ path: remoteFile.key,
426
+ direction: "pull",
427
+ resolution,
428
+ });
429
+ if (resolution !== "abort" && resolution !== "overwrite") {
430
+ if (remoteFetched) {
431
+ try {
432
+ appendConflictEntry(run.hqRoot, {
433
+ id: buildConflictId(originalRelative, detectedAt),
434
+ originalPath: originalRelative,
435
+ conflictPath: conflictRelative,
436
+ detectedAt,
437
+ side: "pull",
438
+ machineId,
439
+ localHash: item.localHash,
440
+ remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
732
441
  });
733
- inFlight.add(p);
734
442
  }
735
- if (inFlight.size > 0) {
736
- // Wait for at least one in-flight task to settle before topping up
737
- // the pool. allSettled-style semantics via Promise.race — the
738
- // .catch wrap above guarantees no worker promise can reject.
739
- await Promise.race(Array.from(inFlight));
443
+ catch (mirrorErr) {
444
+ run.emit({
445
+ type: "error",
446
+ path: remoteFile.key,
447
+ message: `conflict mirror index write failed: ${mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)}`,
448
+ });
740
449
  }
741
450
  }
742
- // Pool drained. If any worker rejected, write the journal first
743
- // (so the lstat fast-path stamps for successfully-downloaded files
744
- // persist) then throw the first error, preserving its stack.
745
- if (workerErrors.length > 0) {
746
- writeJournal(journalSlug, journal);
747
- const first = workerErrors[0];
748
- if (workerErrors.length > 1) {
749
- first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
750
- }
751
- throw first;
451
+ }
452
+ else if (remoteFetched) {
453
+ try {
454
+ fs.rmSync(conflictAbs, { force: true });
455
+ }
456
+ catch {
457
+ /* best-effort; a leftover mirror is cosmetic, not corrupting */
458
+ }
459
+ }
460
+ if (resolution === "abort") {
461
+ run.emit({ type: "new-files", files: [] });
462
+ writeJournal(run.journalSlug, run.journal);
463
+ return {
464
+ filesDownloaded: counters.filesDownloaded,
465
+ bytesDownloaded: counters.bytesDownloaded,
466
+ filesSkipped: counters.filesSkipped,
467
+ conflicts: counters.conflicts,
468
+ conflictPaths: counters.conflictPaths,
469
+ aborted: true,
470
+ newFiles: plan.newFiles,
471
+ newFilesCount: plan.newFilesCount,
472
+ filesExcludedByPolicy: plan.filesExcludedByPolicy,
473
+ filesTombstoned: 0,
474
+ filesOutOfScope: counters.filesOutOfScope,
475
+ scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
476
+ scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
477
+ };
478
+ }
479
+ if (resolution === "keep" || resolution === "skip") {
480
+ counters.filesSkipped++;
481
+ updateEntry(run.journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
482
+ return null;
483
+ }
484
+ downloadItems.push({
485
+ action: "download",
486
+ remoteFile,
487
+ localPath,
488
+ isNew: false,
489
+ });
490
+ return null;
491
+ }
492
+ async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
493
+ if (downloadItems.length === 0)
494
+ return;
495
+ await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
496
+ const queue = [...downloadItems];
497
+ const inFlight = new Set();
498
+ const workerErrors = [];
499
+ while (queue.length > 0 || inFlight.size > 0) {
500
+ while (inFlight.size < transferConcurrency && queue.length > 0) {
501
+ const downloadItem = queue.shift();
502
+ const p = downloadOne(run, downloadItem, counters)
503
+ .catch((err) => {
504
+ workerErrors.push(err instanceof Error ? err : new Error(String(err)));
505
+ })
506
+ .finally(() => {
507
+ inFlight.delete(p);
508
+ });
509
+ inFlight.add(p);
510
+ }
511
+ if (inFlight.size > 0) {
512
+ await Promise.race(Array.from(inFlight));
513
+ }
514
+ }
515
+ if (workerErrors.length > 0) {
516
+ writeJournal(run.journalSlug, run.journal);
517
+ const first = workerErrors[0];
518
+ if (workerErrors.length > 1) {
519
+ first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
520
+ }
521
+ throw first;
522
+ }
523
+ }
524
+ async function downloadOne(run, downloadItem, counters) {
525
+ const { remoteFile, localPath } = downloadItem;
526
+ await refreshRunContextIfExpiring(run);
527
+ if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
528
+ counters.filesSkipped++;
529
+ run.emit({
530
+ type: "error",
531
+ path: remoteFile.key,
532
+ message: "download skipped: local parent escaped the sync root",
533
+ });
534
+ return;
535
+ }
536
+ try {
537
+ const { metadata, contentHash, contentSize } = await downloadFile(run.ctx, remoteFile.key, localPath);
538
+ const author = metadata?.["created-by"] ?? null;
539
+ const createdBySub = metadata?.["created-by-sub"];
540
+ const localLstat = fs.lstatSync(localPath);
541
+ const isLocalSymlink = localLstat.isSymbolicLink();
542
+ const hash = isLocalSymlink
543
+ ? hashSymlinkTarget(fs.readlinkSync(localPath))
544
+ : (contentHash ?? hashFile(localPath));
545
+ const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
546
+ updateEntry(run.journal, remoteFile.key, hash, size, "down", remoteFile.etag, localLstat.mtimeMs, createdBySub);
547
+ const priorEntry = getEntry(run.journal, remoteFile.key);
548
+ const remoteJournalMessage = priorEntry?.message;
549
+ run.emit({
550
+ type: "progress",
551
+ path: remoteFile.key,
552
+ bytes: size,
553
+ ...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
554
+ ...(author ? { author } : {}),
555
+ });
556
+ counters.filesDownloaded++;
557
+ counters.bytesDownloaded += size;
558
+ }
559
+ catch (err) {
560
+ if (isAccessDenied(err)) {
561
+ counters.filesSkipped++;
562
+ }
563
+ else {
564
+ run.emit({
565
+ type: "error",
566
+ path: remoteFile.key,
567
+ message: err instanceof Error ? err.message : String(err),
568
+ });
752
569
  }
753
570
  }
754
- // ── New-files attribution (US-002) ─────────────────────────────────────
755
- // Enrich plan.newFiles with `addedBy` from S3 user metadata. HeadObject
756
- // calls are best-effort and capped at 5 concurrent to avoid hammering S3.
571
+ }
572
+ async function emitAndReportNewFiles(run, plan) {
757
573
  const enrichedNewFiles = [];
574
+ // Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
575
+ // created-by HEADs below reuse the cache instead of each minting its own
576
+ // presign. Without this, a big catch-up pull (hundreds of new files) bursts
577
+ // the presign endpoint, trips the circuit breaker, and every enrichment HEAD
578
+ // then fails. Mirrors the tombstone HEAD-verify pre-prime.
579
+ await primeObjectTransport(run.ctx, "get", plan.newFiles.map((nf) => nf.path));
758
580
  const HEAD_CONCURRENCY = 5;
759
581
  for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
760
582
  const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
761
583
  const results = await Promise.all(batch.map(async (nf) => {
762
584
  let addedBy = null;
763
585
  try {
764
- const head = await headRemoteFile(ctx, nf.path);
586
+ const head = await headRemoteFile(run.ctx, nf.path);
765
587
  if (head?.metadata?.["created-by"]) {
766
588
  addedBy = head.metadata["created-by"];
767
589
  }
768
590
  }
769
591
  catch (headErr) {
770
- // Best-effort: log to console (Sentry captures via global handler)
771
- // and fall through with addedBy = null.
772
592
  try {
773
593
  console.error(`[hq-sync] HeadObject failed for ${nf.path}: ${headErr instanceof Error ? headErr.message : String(headErr)}`);
774
594
  }
@@ -780,210 +600,124 @@ export async function sync(options) {
780
600
  }));
781
601
  enrichedNewFiles.push(...results);
782
602
  }
783
- emit({ type: "new-files", files: enrichedNewFiles });
784
- // Report new files to the notification service so they persist as a
785
- // cross-session "new files" history in the HQ Sync app (POST
786
- // /v1/notify/file-added → per-recipient FILE_EVENT rows for THIS user, who is
787
- // the one the files are new for). Best-effort and bounded: a failure or a
788
- // hung request must never delay or break the sync — the durable signal is the
789
- // synced file itself, this is only a notification mirror.
790
- await reportNewFilesToNotify(vaultConfig, ctx.uid, ctx.slug, enrichedNewFiles);
791
- // Codex P1 (PR #24 follow-up): scope-gate tombstone candidates with
792
- // a per-object HEAD before unlinking. `listRemoteFiles` is STS-scoped
793
- // (guest sessions with `allowedPrefixes`, role downgrade, custom
794
- // sync-mode prefix sets see the AccessDenied branch in the download
795
- // loop above), so a journal entry's absence from LIST does not prove
796
- // the object was deleted; it may simply be invisible to this session.
797
- // HEAD each candidate:
798
- // - HEAD returns metadata → object exists → NOT in our LIST scope →
799
- // skip the tombstone (peer didn't delete it; we just can't see it).
800
- // - HEAD returns null (NotFound) → confirmed deleted → tombstone.
801
- // - HEAD throws AccessDenied → can't tell → defensive skip; journal
802
- // stays so next sync (with broader scope) can re-evaluate.
803
- // - HEAD throws transient → defensive skip + emit error.
804
- // Bounded concurrency mirrors the new-files attribution pass above.
805
- if (plan.tombstones.length > 0) {
806
- // Pre-mint GET URLs for the tombstone HEAD-verify probes below (headRemote
807
- // File presigns a GET), so a large delete set doesn't add N presign calls.
808
- await primeObjectTransport(ctx, "get", plan.tombstones);
809
- const HEAD_VERIFY_CONCURRENCY = 5;
810
- const verified = [];
811
- for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
812
- const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
813
- const results = await Promise.all(batch.map(async (key) => {
814
- try {
815
- const head = await headRemoteFile(ctx, key);
816
- return head === null ? key : null;
817
- }
818
- catch (err) {
819
- if (isAccessDenied(err))
820
- return null;
821
- emit({
822
- type: "error",
823
- path: key,
824
- message: `tombstone HEAD verify failed (deferring): ${err instanceof Error ? err.message : String(err)}`,
825
- });
603
+ run.emit({ type: "new-files", files: enrichedNewFiles });
604
+ await reportNewFilesToNotify(run.vaultConfig, run.ctx.uid, run.ctx.slug, enrichedNewFiles);
605
+ }
606
+ async function verifyPlannedJournalTombstones(run, plan) {
607
+ if (plan.tombstones.length === 0)
608
+ return;
609
+ await primeObjectTransport(run.ctx, "get", plan.tombstones);
610
+ const HEAD_VERIFY_CONCURRENCY = 5;
611
+ const verified = [];
612
+ for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
613
+ const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
614
+ const results = await Promise.all(batch.map(async (key) => {
615
+ try {
616
+ const head = await headRemoteFile(run.ctx, key);
617
+ return head === null ? key : null;
618
+ }
619
+ catch (err) {
620
+ if (isAccessDenied(err))
826
621
  return null;
827
- }
828
- }));
829
- for (const k of results) {
830
- if (k !== null)
831
- verified.push(k);
622
+ run.emit({
623
+ type: "error",
624
+ path: key,
625
+ message: `tombstone HEAD verify failed (deferring): ${err instanceof Error ? err.message : String(err)}`,
626
+ });
627
+ return null;
832
628
  }
629
+ }));
630
+ for (const k of results) {
631
+ if (k !== null)
632
+ verified.push(k);
833
633
  }
834
- plan.tombstones = verified;
835
634
  }
836
- // Bug #9 — apply cross-machine delete propagation. Each tombstone is a
837
- // key the journal records as previously synced but the remote LIST no
838
- // longer contains. We delete the local file (or symlink, or empty dir
839
- // remnant) and drop the journal entry so the next sync's planner stays
840
- // converged. Failures are reported but non-fatal — the entry stays in
841
- // the journal and the next run retries.
635
+ plan.tombstones = verified;
636
+ }
637
+ function executeJournalTombstoneDeletes(run, plan, counters) {
842
638
  for (const key of plan.tombstones) {
843
- // Last line of defense against the Windows backslash-key landmine: a
844
- // malformed key must NEVER reach fs.unlinkSync. path.join(companyRoot, key)
845
- // collapses the backslashes and resolves onto the REAL POSIX file, so
846
- // unlinking here destroys live data (ridge incident, feedback_b8d09d0f).
847
- // The planner already refuses to enqueue malformed keys; if one still
848
- // arrives, drop the poisoned journal entry without touching disk —
849
- // normalizeJournalKeys rewrites it to its POSIX form on load.
850
- if (isMalformedVaultKey(key)) {
851
- removeEntry(journal, key);
639
+ const localPath = resolveContainedVaultPath(run.companyRoot, key);
640
+ if (localPath === null)
852
641
  continue;
853
- }
854
- const localPath = path.join(companyRoot, key);
855
642
  let removedSomething = false;
856
643
  try {
857
644
  const lstat = fs.lstatSync(localPath);
645
+ if (tombstoneTargetDiverged(run.journal, key, localPath, lstat)) {
646
+ continue;
647
+ }
858
648
  if (lstat.isSymbolicLink() || lstat.isFile()) {
859
649
  fs.unlinkSync(localPath);
860
650
  removedSomething = true;
861
651
  }
862
652
  else if (lstat.isDirectory()) {
863
- // A dir at a key likely from a (local-dir, cloud-file) historic
864
- // state. Don't recursively rm-rf the operator's dir; just drop
865
- // the journal entry so we converge with reality.
653
+ // A dir at a key is converged by dropping only the journal entry.
866
654
  }
867
655
  }
868
656
  catch (err) {
869
657
  const code = err && typeof err === "object" && "code" in err
870
658
  ? err.code
871
659
  : undefined;
872
- // ENOENT → local already gone; safe to drop the journal entry.
873
- // Other errors (EACCES/EPERM/EBUSY/etc.) leave the local file in
874
- // place — if we dropped the journal entry anyway, the pull side
875
- // would forget the peer's delete and a later push could re-upload
876
- // the still-present local file, silently undoing the peer's delete.
877
- // Surface the error and KEEP the journal entry so the next sync
878
- // retries the unlink after the operator fixes the permission.
879
660
  if (code !== "ENOENT") {
880
- emit({
661
+ run.emit({
881
662
  type: "error",
882
663
  path: key,
883
664
  message: `tombstone unlink failed: ${err instanceof Error ? err.message : String(err)}`,
884
665
  });
885
- // Skip removeEntry / filesTombstoned / progress event — the
886
- // tombstone hasn't actually been honored. Next sync retries.
887
666
  continue;
888
667
  }
889
668
  }
890
- removeEntry(journal, key);
891
- filesTombstoned++;
892
- emit({
669
+ removeEntry(run.journal, key);
670
+ counters.filesTombstoned++;
671
+ run.emit({
893
672
  type: "progress",
894
673
  path: key,
895
674
  bytes: 0,
896
675
  deleted: true,
897
- // Suffix differentiates a tombstone from a normal delete in the
898
- // tty stream — matches the push-side `defaultConsoleLogger`
899
- // tombstone surface in share.ts.
900
676
  message: removedSomething ? "tombstone (cross-machine delete)" : "tombstone (already absent locally)",
901
677
  });
902
678
  }
903
- // Record this pull's boundary (US-005) so the NEXT pull can diff its scope
904
- // against ours and detect a shrink. Append before the journal write so it
905
- // persists. `prefixSet` is stored in the same company-relative namespace as
906
- // the journal keys; `all` mode records `[""]` (covers everything).
907
- appendPullRecord(journal, {
679
+ }
680
+ function finalizePullRun(run, plan, scopeRun, counters) {
681
+ appendPullRecord(run.journal, {
908
682
  pullId: generatePullId(),
909
- companyUid: ctx.uid,
910
- startedAt,
683
+ companyUid: run.ctx.uid,
684
+ startedAt: run.startedAt,
911
685
  completedAt: new Date().toISOString(),
912
- syncMode,
913
- prefixSet: currentPrefixSet,
914
- scopeChangeDetected: shrinkPlan.scopeChangeDetected,
915
- orphansRemoved: scopeOrphansRemoved,
916
- orphansBlocked: shrinkResult.dirtyTombstoned,
686
+ syncMode: run.syncMode,
687
+ prefixSet: run.currentPrefixSet,
688
+ scopeChangeDetected: scopeRun.shrinkPlan.scopeChangeDetected,
689
+ orphansRemoved: scopeRun.scopeOrphansRemoved,
690
+ orphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
917
691
  });
918
- // Stamp lastSync on every successful run so the menubar's "Last sync · X ago"
919
- // ticks even when nothing transferred. updateEntry only fires on actual
920
- // downloads; without this, a no-op sync leaves lastSync at the time of the
921
- // last file change, which is misleading.
922
- journal.lastSync = new Date().toISOString();
923
- writeJournal(journalSlug, journal);
924
- // When the pull actually changed on-disk sources (new files, tombstoned
925
- // removals, or scope-orphan cleanups), refresh the generated skill wrappers,
926
- // personal-overlay mirrors, and workers registry. reindex is idempotent and
927
- // best-effort — it must never fail a sync, and is skipped on no-op syncs
928
- // (the common daemon case) and when the caller opts out via skipReindex.
929
- const changedOnDisk = filesDownloaded > 0 ||
930
- filesTombstoned > 0 ||
931
- scopeOrphansRemoved > 0;
932
- if (!options.skipReindex && changedOnDisk) {
692
+ run.journal.lastSync = new Date().toISOString();
693
+ writeJournal(run.journalSlug, run.journal);
694
+ const changedOnDisk = counters.filesDownloaded > 0 ||
695
+ counters.filesTombstoned > 0 ||
696
+ scopeRun.scopeOrphansRemoved > 0;
697
+ if (!run.options.skipReindex && changedOnDisk) {
933
698
  try {
934
- // skipLock: the surrounding sync run already holds this root's operation
935
- // lock; reindex re-acquiring would refuse against our own live PID.
936
- reindex({ repoRoot: hqRoot, skipLock: true });
699
+ reindex({ repoRoot: run.hqRoot, skipLock: true });
937
700
  }
938
701
  catch {
939
702
  // best-effort: a post-sync refresh failure never fails the sync
940
703
  }
941
704
  }
942
705
  return {
943
- filesDownloaded,
944
- bytesDownloaded,
945
- filesSkipped,
946
- conflicts,
947
- conflictPaths,
706
+ filesDownloaded: counters.filesDownloaded,
707
+ bytesDownloaded: counters.bytesDownloaded,
708
+ filesSkipped: counters.filesSkipped,
709
+ conflicts: counters.conflicts,
710
+ conflictPaths: counters.conflictPaths,
948
711
  aborted: false,
949
712
  newFiles: plan.newFiles,
950
713
  newFilesCount: plan.newFilesCount,
951
714
  filesExcludedByPolicy: plan.filesExcludedByPolicy,
952
- filesTombstoned,
953
- filesOutOfScope,
954
- scopeOrphansRemoved,
955
- scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
715
+ filesTombstoned: counters.filesTombstoned,
716
+ filesOutOfScope: counters.filesOutOfScope,
717
+ scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
718
+ scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
956
719
  };
957
720
  }
958
- /**
959
- * Resolve active company from .hq/config.json.
960
- */
961
- function resolveActiveCompany(hqRoot) {
962
- const configPath = path.join(hqRoot, ".hq", "config.json");
963
- if (fs.existsSync(configPath)) {
964
- try {
965
- const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
966
- return config.activeCompany ?? config.companySlug;
967
- }
968
- catch {
969
- // Ignore parse errors
970
- }
971
- }
972
- return undefined;
973
- }
974
- /**
975
- * Returns true when the remote object appears to have moved since the
976
- * journal entry's last-recorded sync. Prefers ETag equality; falls back to
977
- * `lastModified > syncedAt` for legacy entries written before remoteEtag
978
- * was tracked. Conservative on tie (`<=` skews "remote unchanged").
979
- */
980
- function hasRemoteChanged(remote, entry) {
981
- if (entry.remoteEtag) {
982
- return normalizeEtag(remote.etag) !== entry.remoteEtag;
983
- }
984
- const syncedAt = new Date(entry.syncedAt).getTime();
985
- return remote.lastModified.getTime() > syncedAt;
986
- }
987
721
  /**
988
722
  * Decide whether a remote object present in the LIST is a GENUINE RE-CREATE
989
723
  * written AFTER a FILE_TOMBSTONE — in which case the tombstone is stale and the
@@ -1012,6 +746,85 @@ function isRemoteRecreateAfterTombstone(remote, tombstone) {
1012
746
  return true; // no remote timestamp → don't suppress
1013
747
  return remoteMs > deletedAtMs;
1014
748
  }
749
+ function hasTraversalSegment(key) {
750
+ return key.split("/").some((segment) => segment === "..");
751
+ }
752
+ function isPathWithin(root, candidate) {
753
+ const relative = path.relative(root, candidate);
754
+ return (relative === "" ||
755
+ (!relative.startsWith("..") && !path.isAbsolute(relative)));
756
+ }
757
+ function deepestExistingAncestor(start) {
758
+ let current = start;
759
+ for (;;) {
760
+ try {
761
+ fs.lstatSync(current);
762
+ return current;
763
+ }
764
+ catch (err) {
765
+ const code = err && typeof err === "object" && "code" in err
766
+ ? err.code
767
+ : undefined;
768
+ if (code !== "ENOENT" && code !== "ENOTDIR")
769
+ return null;
770
+ }
771
+ const parent = path.dirname(current);
772
+ if (parent === current)
773
+ return null;
774
+ current = parent;
775
+ }
776
+ }
777
+ function resolveContainedVaultPath(root, key) {
778
+ if (isMalformedVaultKey(key) || hasTraversalSegment(key))
779
+ return null;
780
+ const resolvedRoot = path.resolve(root);
781
+ const resolvedLocal = path.resolve(resolvedRoot, key);
782
+ if (!isPathWithin(resolvedRoot, resolvedLocal))
783
+ return null;
784
+ let realRoot;
785
+ try {
786
+ realRoot = fs.realpathSync.native(resolvedRoot);
787
+ }
788
+ catch {
789
+ // If the vault root does not exist yet, no below-root symlink component can
790
+ // already exist to redirect this key. Preserve first-pull behavior.
791
+ return resolvedLocal;
792
+ }
793
+ const existingAncestor = deepestExistingAncestor(path.dirname(resolvedLocal));
794
+ if (existingAncestor === null)
795
+ return null;
796
+ try {
797
+ const realAncestor = fs.realpathSync.native(existingAncestor);
798
+ if (!isPathWithin(realRoot, realAncestor))
799
+ return null;
800
+ }
801
+ catch {
802
+ return null;
803
+ }
804
+ return resolvedLocal;
805
+ }
806
+ function isDownloadWritePathStillContained(root, key, localPath) {
807
+ const resolved = resolveContainedVaultPath(root, key);
808
+ return resolved !== null && path.resolve(resolved) === path.resolve(localPath);
809
+ }
810
+ function tombstoneTargetDiverged(journal, key, localPath, lstat) {
811
+ const journalEntry = journal.files[key];
812
+ if (!journalEntry?.hash) {
813
+ return lstat.isSymbolicLink() || lstat.isFile();
814
+ }
815
+ try {
816
+ if (lstat.isSymbolicLink()) {
817
+ return hashSymlinkTarget(fs.readlinkSync(localPath)) !== journalEntry.hash;
818
+ }
819
+ if (lstat.isFile()) {
820
+ return hashFile(localPath) !== journalEntry.hash;
821
+ }
822
+ }
823
+ catch {
824
+ return true;
825
+ }
826
+ return false;
827
+ }
1015
828
  /**
1016
829
  * Stage-1 planning pass: classify every remote file into download / skip /
1017
830
  * conflict buckets without performing any S3 transfers. Local hashes are
@@ -1037,7 +850,11 @@ prefixSet,
1037
850
  fileTombstones = new Map()) {
1038
851
  const items = [];
1039
852
  for (const remoteFile of remoteFiles) {
1040
- const localPath = path.join(companyRoot, remoteFile.key);
853
+ const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
854
+ if (localPath === null) {
855
+ items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
856
+ continue;
857
+ }
1041
858
  // Ephemeral-mirror filter — symmetric with the push-side walker. Bug #2
1042
859
  // in the 5.33.0 deep-test: the push side has refused to upload conflict
1043
860
  // mirrors since 5.33.0, but the pull side downloaded them freely from
@@ -1048,16 +865,6 @@ fileTombstones = new Map()) {
1048
865
  items.push({ action: "skip-excluded-policy", remoteFile, localPath });
1049
866
  continue;
1050
867
  }
1051
- // Malformed-key filter — keys with backslash separators pushed by
1052
- // pre-5.47.2 Windows clients. Downloading one materializes a junk local
1053
- // file whose NAME contains backslashes (it is not a path on POSIX), which
1054
- // then churns conflict mirrors forever. Refuse at planning time, same
1055
- // policy bucket as the ephemeral filter above. The bogus keys themselves
1056
- // are cleaned server-side; this keeps clean trees clean in the meantime.
1057
- if (isMalformedVaultKey(remoteFile.key)) {
1058
- items.push({ action: "skip-excluded-policy", remoteFile, localPath });
1059
- continue;
1060
- }
1061
868
  if (personalMode &&
1062
869
  remoteFile.key.startsWith("companies/") &&
1063
870
  // EXEMPTION: companies/manifest.yaml is the routing source-of-truth
@@ -1403,12 +1210,8 @@ fileTombstones = new Map()) {
1403
1210
  const posixKey = toPosixKey(key);
1404
1211
  if (remoteKeySet.has(posixKey))
1405
1212
  continue;
1406
- // Never tombstone-delete via a malformed (backslash) key: the executor's
1407
- // path.join(companyRoot, key) collapses backslashes back onto the REAL
1408
- // POSIX file and unlinks live data. Leave it for normalizeJournalKeys to
1409
- // rewrite to POSIX on the next write; the canonical key is re-evaluated
1410
- // (and correctly tombstoned if genuinely remote-deleted) on a later pull.
1411
- if (isMalformedVaultKey(key))
1213
+ const localPath = resolveContainedVaultPath(companyRoot, key);
1214
+ if (localPath === null)
1412
1215
  continue;
1413
1216
  // PersonalMode key gating — mirror the download branch.
1414
1217
  if (personalMode && key.startsWith("companies/")) {
@@ -1424,7 +1227,6 @@ fileTombstones = new Map()) {
1424
1227
  // Honor the current ignore filter — if a path was previously synced
1425
1228
  // but is now ignored (operator edited .hqignore), do NOT delete
1426
1229
  // the local copy. They're keeping it deliberately.
1427
- const localPath = path.join(companyRoot, key);
1428
1230
  if (!shouldSync(localPath, false) && !shouldSync(localPath, true))
1429
1231
  continue;
1430
1232
  // Codex P1 (PR #24 round 3): detect local edits before tombstoning.
@@ -1497,15 +1299,6 @@ fileTombstones = new Map()) {
1497
1299
  tombstones,
1498
1300
  };
1499
1301
  }
1500
- /**
1501
- * Check if an error is an S3 access denied (expected for filtered guests).
1502
- */
1503
- function isAccessDenied(err) {
1504
- if (err && typeof err === "object" && "name" in err) {
1505
- return err.name === "AccessDenied" || err.name === "Forbidden";
1506
- }
1507
- return false;
1508
- }
1509
1302
  /**
1510
1303
  * Default human-readable event rendering. Preserves the exact output format
1511
1304
  * that `hq sync` emitted before SyncProgressEvent was introduced, so callers