@indigoai-us/hq-cloud 6.11.12 → 6.11.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/bin/sync-runner-company.d.ts +35 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -0
  3. package/dist/bin/sync-runner-company.js +290 -0
  4. package/dist/bin/sync-runner-company.js.map +1 -0
  5. package/dist/bin/sync-runner-events.d.ts +12 -0
  6. package/dist/bin/sync-runner-events.d.ts.map +1 -0
  7. package/dist/bin/sync-runner-events.js +12 -0
  8. package/dist/bin/sync-runner-events.js.map +1 -0
  9. package/dist/bin/sync-runner-planning.d.ts +53 -0
  10. package/dist/bin/sync-runner-planning.d.ts.map +1 -0
  11. package/dist/bin/sync-runner-planning.js +59 -0
  12. package/dist/bin/sync-runner-planning.js.map +1 -0
  13. package/dist/bin/sync-runner-rollup.d.ts +24 -0
  14. package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
  15. package/dist/bin/sync-runner-rollup.js +46 -0
  16. package/dist/bin/sync-runner-rollup.js.map +1 -0
  17. package/dist/bin/sync-runner-telemetry.d.ts +5 -0
  18. package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
  19. package/dist/bin/sync-runner-telemetry.js +5 -0
  20. package/dist/bin/sync-runner-telemetry.js.map +1 -0
  21. package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
  22. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
  23. package/dist/bin/sync-runner-watch-loop.js +372 -0
  24. package/dist/bin/sync-runner-watch-loop.js.map +1 -0
  25. package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
  26. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
  27. package/dist/bin/sync-runner-watch-routes.js +74 -0
  28. package/dist/bin/sync-runner-watch-routes.js.map +1 -0
  29. package/dist/bin/sync-runner.d.ts +3 -54
  30. package/dist/bin/sync-runner.d.ts.map +1 -1
  31. package/dist/bin/sync-runner.js +73 -1154
  32. package/dist/bin/sync-runner.js.map +1 -1
  33. package/dist/cli/reindex.d.ts.map +1 -1
  34. package/dist/cli/reindex.js +34 -17
  35. package/dist/cli/reindex.js.map +1 -1
  36. package/dist/cli/reindex.test.js +39 -5
  37. package/dist/cli/reindex.test.js.map +1 -1
  38. package/dist/cli/rescue-classify-ordering.test.js +17 -0
  39. package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
  40. package/dist/cli/rescue-core.d.ts +45 -0
  41. package/dist/cli/rescue-core.d.ts.map +1 -1
  42. package/dist/cli/rescue-core.js +197 -170
  43. package/dist/cli/rescue-core.js.map +1 -1
  44. package/dist/cli/share.d.ts.map +1 -1
  45. package/dist/cli/share.js +224 -676
  46. package/dist/cli/share.js.map +1 -1
  47. package/dist/cli/sync.d.ts.map +1 -1
  48. package/dist/cli/sync.js +399 -726
  49. package/dist/cli/sync.js.map +1 -1
  50. package/dist/cli/sync.test.js +20 -0
  51. package/dist/cli/sync.test.js.map +1 -1
  52. package/dist/daemon-worker.d.ts +2 -2
  53. package/dist/daemon-worker.js +3 -3
  54. package/dist/daemon-worker.js.map +1 -1
  55. package/dist/object-io.js +1 -1
  56. package/dist/object-io.js.map +1 -1
  57. package/dist/remote-pull.d.ts +2 -2
  58. package/dist/remote-pull.d.ts.map +1 -1
  59. package/dist/remote-pull.js +23 -3
  60. package/dist/remote-pull.js.map +1 -1
  61. package/dist/remote-pull.test.js +24 -2
  62. package/dist/remote-pull.test.js.map +1 -1
  63. package/dist/sync/push-receiver.d.ts +6 -0
  64. package/dist/sync/push-receiver.d.ts.map +1 -1
  65. package/dist/sync/push-receiver.js +32 -2
  66. package/dist/sync/push-receiver.js.map +1 -1
  67. package/dist/sync/push-receiver.test.js +31 -0
  68. package/dist/sync/push-receiver.test.js.map +1 -1
  69. package/dist/sync-core.d.ts +27 -0
  70. package/dist/sync-core.d.ts.map +1 -0
  71. package/dist/sync-core.js +54 -0
  72. package/dist/sync-core.js.map +1 -0
  73. package/dist/vault-client.d.ts.map +1 -1
  74. package/dist/vault-client.js +284 -36
  75. package/dist/vault-client.js.map +1 -1
  76. package/dist/vault-client.test.js +59 -0
  77. package/dist/vault-client.test.js.map +1 -1
  78. package/dist/watcher.d.ts +2 -20
  79. package/dist/watcher.d.ts.map +1 -1
  80. package/dist/watcher.js +3 -113
  81. package/dist/watcher.js.map +1 -1
  82. package/package.json +1 -1
  83. package/src/bin/sync-runner-company.ts +350 -0
  84. package/src/bin/sync-runner-events.ts +25 -0
  85. package/src/bin/sync-runner-planning.ts +121 -0
  86. package/src/bin/sync-runner-rollup.ts +72 -0
  87. package/src/bin/sync-runner-telemetry.ts +8 -0
  88. package/src/bin/sync-runner-watch-loop.ts +443 -0
  89. package/src/bin/sync-runner-watch-routes.ts +86 -0
  90. package/src/bin/sync-runner.ts +96 -1253
  91. package/src/cli/reindex.test.ts +41 -3
  92. package/src/cli/reindex.ts +35 -19
  93. package/src/cli/rescue-classify-ordering.test.ts +20 -0
  94. package/src/cli/rescue-core.ts +252 -176
  95. package/src/cli/share.ts +363 -705
  96. package/src/cli/sync.test.ts +25 -0
  97. package/src/cli/sync.ts +612 -802
  98. package/src/daemon-worker.ts +3 -3
  99. package/src/object-io.ts +1 -1
  100. package/src/remote-pull.test.ts +30 -1
  101. package/src/remote-pull.ts +29 -4
  102. package/src/sync/push-receiver.test.ts +35 -0
  103. package/src/sync/push-receiver.ts +41 -2
  104. package/src/sync-core.ts +58 -0
  105. package/src/vault-client.test.ts +74 -0
  106. package/src/vault-client.ts +395 -43
  107. package/src/watcher.ts +6 -141
package/dist/cli/sync.js CHANGED
@@ -13,6 +13,7 @@ import { PERSONAL_VAULT_MANIFEST_KEY } from "../personal-vault.js";
13
13
  import { buildScopeShrinkPlan, applyScopeShrink, ScopeShrinkBlockedError, ScopeShrinkLargePruneError, } from "../scope-shrink.js";
14
14
  import { coalescePrefixes, isCoveredByAny, } from "../prefix-coalesce.js";
15
15
  import { createIgnoreFilter } from "../ignore.js";
16
+ import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransferConcurrency, } from "../sync-core.js";
16
17
  import { isEphemeralPath, isMalformedVaultKey } from "./share.js";
17
18
  import { resolveConflict } from "./conflict.js";
18
19
  import { buildConflictId, buildConflictPath, readShortMachineId, } from "../lib/conflict-file.js";
@@ -129,193 +130,140 @@ export async function sync(options) {
129
130
  return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
130
131
  }
131
132
  async function syncWithOperationLockHeld(options) {
132
- const { company, onConflict, vaultConfig, hqRoot } = options;
133
+ const run = await buildPullContext(options);
134
+ const plan = planPull(run);
135
+ emitPullPlan(run.emit, plan);
136
+ const scopePlan = planScopeShrink(run);
137
+ const scopeRun = executeScopeShrink(run, scopePlan);
138
+ const counters = createPullCounters();
139
+ const transferConcurrency = resolveTransferConcurrency();
140
+ const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
141
+ if (conflictRun.abortResult) {
142
+ return conflictRun.abortResult;
143
+ }
144
+ await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
145
+ await emitAndReportNewFiles(run, plan);
146
+ await verifyPlannedJournalTombstones(run, plan);
147
+ executeJournalTombstoneDeletes(run, plan, counters);
148
+ return finalizePullRun(run, plan, scopeRun, counters);
149
+ }
150
+ async function buildPullContext(options) {
151
+ const { company, vaultConfig, hqRoot } = options;
133
152
  const emit = options.onEvent ?? defaultConsoleLogger;
134
- // Resolve company
135
153
  const companyRef = company ?? resolveActiveCompany(hqRoot);
136
154
  if (!companyRef) {
137
155
  throw new Error("No company specified and no active company found. " +
138
156
  "Use --company <slug> or set up .hq/config.json.");
139
157
  }
140
- // Resolve entity context
141
- let ctx = await resolveEntityContext(companyRef, vaultConfig);
142
- // Every company's files land under companies/{slug}/ so fanning out multiple
143
- // companies into the same hqRoot doesn't cross-clobber files with overlapping
144
- // S3 keys (e.g. every company has a .hq/manifest.json). Remote keys stay
145
- // company-relative; the prefix lives only on disk.
146
- // In personalMode the journal slug + S3 keys are person-relative (e.g. "docs/foo.md");
147
- // the local target is `hqRoot` directly, NOT `<hqRoot>/companies/<personSlug>/`. This
148
- // keeps round-trip parity with the Rust personal first-push (Step 7) which sources
149
- // `<hqRoot>/docs/foo.md`.
158
+ const ctx = await resolveEntityContext(companyRef, vaultConfig);
150
159
  const companyRoot = options.personalMode === true
151
160
  ? hqRoot
152
161
  : path.join(hqRoot, "companies", ctx.slug);
153
162
  const shouldSync = createIgnoreFilter(hqRoot);
154
163
  const journalSlug = options.journalSlug ?? ctx.slug;
155
164
  const startedAt = new Date().toISOString();
156
- // Personal-vault callers must never start from an empty journal when only
157
- // the legacy `personal` file exists (mass re-download/etag churn). Seeding
158
- // here — inside the engine — covers every consumer (sync-runner already
159
- // seeds; hq-cli historically didn't, which split the vault's bookkeeping
160
- // across two journal files and re-flagged synced files as conflicts).
161
165
  if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG)
162
166
  migratePersonalVaultJournal();
163
- // Migrate v1 → v2 in place so the scope-shrink / pull-record machinery has
164
- // its fields, and GC any tombstones past the 30-day retention window before
165
- // we re-evaluate orphans (so a long-pruned path can re-download cleanly).
166
167
  const journal = migrateToV2(readJournal(journalSlug));
167
168
  gcTombstones(journal, Date.now());
168
- // ── Effective download scope (US-005) ─────────────────────────────────────
169
- // `all` → prefixSet `[""]`, which `isCoveredByAny` treats as "covers
170
- // everything" — so the download filter and the scope-shrink
171
- // comparison both become no-ops, preserving legacy full-bucket
172
- // behavior bit-for-bit.
173
- // `shared`/`custom` → the coalesced, company-relative prefix set the runner
174
- // resolved. An empty set means "nothing in scope" → download
175
- // nothing (the runner falls back to `all` on resolution errors, so
176
- // empty here is an intentional "nothing shared", never a failure).
177
169
  const syncMode = options.syncMode ?? "all";
178
170
  const currentPrefixSet = syncMode === "all" ? [""] : coalescePrefixes(options.prefixSet ?? []);
179
- // Authorship guard input (scope-shrink): the caller's own Cognito sub,
180
- // injected by the entry point (the runner sources it from its decoded
181
- // idToken claims — the same sub stamped onto uploads as `created-by-sub`).
182
- // Undefined degrades safely: own-author files lose their special shield, but
183
- // the `protectUnknownAuthors` conservative path below still prevents a
184
- // routine sync from deleting anything it can't prove is foreign.
185
- const callerSub = options.callerSub;
186
- let filesDownloaded = 0;
187
- let bytesDownloaded = 0;
188
- let filesSkipped = 0;
189
- let conflicts = 0;
190
- let filesTombstoned = 0;
191
- let filesOutOfScope = 0;
192
- const conflictPaths = [];
193
- // List all remote files (IAM session policy filters at the AWS layer)
194
171
  const remoteFiles = await listRemoteFiles(ctx);
195
- // Fetch the company's FILE_TOMBSTONE records so the planner can suppress
196
- // resurrection of an intentionally-deleted object (delete-resync). Done in
197
- // parallel intent with the LIST above conceptually, but kept serial here for
198
- // a clean read of `ctx`; best-effort — a failed read degrades to an empty map
199
- // (no suppression), preserving the pre-fix behavior. ctx.uid is the verified
200
- // companyUid the tombstone rows are keyed under.
201
- //
202
- // SKIP for the personal vault: its `ctx.uid` is a personUid (`prs_…`), but
203
- // `GET /v1/files/tombstones?company=…` is COMPANY-scoped server-side
204
- // (findCallerWithMembership), so a personal-vault request resolves
205
- // `company=prs_…` to no membership and is correctly rejected with
206
- // `403 "No active membership for caller in company prs_…"`. That 403 is
207
- // benign for the pull (it already degrades to the empty map below), but
208
- // hq-pro captures EVERY one as a Sentry warning — the per-personal-vault
209
- // no-membership cluster (one Sentry issue per signed-in user). Personal-vault
210
- // delete-resync was never a committed feature and there is no person-scoped
211
- // tombstone path, so for the personal target we skip the fetch and use an
212
- // empty map — byte-for-byte the current degraded behavior, minus the 403 spam.
213
- // FUTURE FOLLOW-UP (not built here): if personal-vault delete-resync is
214
- // wanted, it needs a real person-scoped tombstone endpoint + client read.
215
- const tombstones = options.personalMode === true
172
+ const fileTombstones = options.personalMode === true
216
173
  ? new Map()
217
174
  : await fetchCompanyTombstones(vaultConfig, ctx.uid);
218
- // Stage 1: classify every remote file against the journal + local disk.
219
- // Hashing happens here (not in the transfer loop) so the plan event below
220
- // carries an accurate denominator before any progress events fire.
221
- const plan = computePullPlan(remoteFiles, journal, companyRoot, shouldSync, options.personalMode === true, options.includeLocalCompanies === true, options.teamSyncedSlugs ?? null, currentPrefixSet, tombstones);
175
+ return {
176
+ options,
177
+ companyRef,
178
+ vaultConfig,
179
+ hqRoot,
180
+ emit,
181
+ ctx,
182
+ companyRoot,
183
+ shouldSync,
184
+ journalSlug,
185
+ startedAt,
186
+ journal,
187
+ remoteFiles,
188
+ syncMode,
189
+ currentPrefixSet,
190
+ fileTombstones,
191
+ };
192
+ }
193
+ function planPull(run) {
194
+ return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.includeLocalCompanies === true, run.options.teamSyncedSlugs ?? null, run.currentPrefixSet, run.fileTombstones);
195
+ }
196
+ function emitPullPlan(emit, plan) {
222
197
  emit({
223
198
  type: "plan",
224
199
  filesToDownload: plan.filesToDownload,
225
200
  bytesToDownload: plan.bytesToDownload,
226
- // sync() is pull-only; push counts are sourced from share()'s plan event.
227
201
  filesToUpload: 0,
228
202
  bytesToUpload: 0,
229
203
  filesToSkip: plan.filesToSkip,
230
204
  filesToConflict: plan.filesToConflict,
231
- // Authoritative FILE_TOMBSTONE suppressions (delete-resync) are the only
232
- // deletes known at plan time; the journal-vs-LIST tombstones are
233
- // HEAD-verified later and surfaced via the final filesTombstoned count.
234
205
  filesToDelete: plan.filesToTombstoneDelete,
235
206
  });
236
- // ── Scope-shrink cleanup (US-005) ─────────────────────────────────────────
237
- // If the effective scope narrowed since the last pull, files that were
238
- // pulled under the old scope but fall outside the new one are orphans. We
239
- // delete only CLEAN orphans (provably unchanged since last sync); dirty
240
- // (locally-modified) orphans are sacred. By default a dirty orphan aborts
241
- // the leg with a structured error the CLI renders; `forceScopeShrink` keeps
242
- // dirty files on disk and only tombstones their journal entries.
243
- //
244
- // `companyRoot` is passed as the module's `hqRoot` so its `path.join(root,
245
- // key)` resolves company-relative journal keys correctly (the scope-shrink
246
- // module is namespace-agnostic — root + keys + prefixSet must simply agree).
247
- //
248
- // Note: this is the durable selective-download fix for OWNERS. An owner's
249
- // STS is wide (role-bypass), so the remote LIST returns everything and the
250
- // AWS layer never narrows the pull. This client-side shrink is what makes
251
- // `hq sync mode shared` actually stick across re-syncs for an owner.
252
- const lastRecord = lastPullRecord(journal, ctx.uid);
253
- // A missing record, or a v1-migrated record with an empty prefixSet, means
254
- // "no recorded scope" → treat the last scope as full-bucket `all` (`[""]`),
255
- // per the PullRecord.prefixSet contract in types.ts.
207
+ }
208
+ function createPullCounters() {
209
+ return {
210
+ filesDownloaded: 0,
211
+ bytesDownloaded: 0,
212
+ filesSkipped: 0,
213
+ conflicts: 0,
214
+ filesTombstoned: 0,
215
+ filesOutOfScope: 0,
216
+ conflictPaths: [],
217
+ };
218
+ }
219
+ function planScopeShrink(run) {
220
+ const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
256
221
  const lastPrefixSet = lastRecord && lastRecord.prefixSet.length > 0
257
222
  ? lastRecord.prefixSet
258
223
  : [""];
259
224
  const shrinkPlan = buildScopeShrinkPlan({
260
- journal,
261
- hqRoot: companyRoot,
225
+ journal: run.journal,
226
+ hqRoot: run.companyRoot,
262
227
  lastPrefixSet,
263
- currentPrefixSet,
264
- callerSub,
265
- // Automatic pull: never auto-prune content the caller authored, and never
266
- // make a destructive guess about unknown-author (legacy) orphans. The
267
- // explicit `hq sync narrow` ritual opts out of the unknown-author shield.
228
+ currentPrefixSet: run.currentPrefixSet,
229
+ callerSub: run.options.callerSub,
268
230
  protectUnknownAuthors: true,
269
231
  });
270
- // Policy: the background menubar runner ("auto-recover") can take no
271
- // interactive flag, so it must never throw on a shrink — it self-heals
272
- // non-destructively (dirty kept on disk + un-tracked, clean quarantined).
273
- // A foreground `hq sync` ("block", the default) keeps the protective gate
274
- // but renders FOLLOWABLE advice. `autoRecover` implies force (proceed) and
275
- // bypasses the bulk-prune cap (quarantine is non-destructive, so a large
276
- // recovery move is safe). DEV-1768.
277
- const scopeShrinkPolicy = options.scopeShrinkPolicy ?? "block";
232
+ const scopeShrinkPolicy = run.options.scopeShrinkPolicy ?? "block";
278
233
  const autoRecover = scopeShrinkPolicy === "auto-recover";
279
234
  const adviceContext = autoRecover ? "runner" : "cli";
280
- const effectiveForce = options.forceScopeShrink === true || autoRecover;
235
+ const effectiveForce = run.options.forceScopeShrink === true || autoRecover;
236
+ return {
237
+ lastRecord,
238
+ shrinkPlan,
239
+ autoRecover,
240
+ adviceContext,
241
+ effectiveForce,
242
+ };
243
+ }
244
+ function executeScopeShrink(run, scopePlan) {
245
+ const { lastRecord, shrinkPlan, adviceContext, effectiveForce } = scopePlan;
281
246
  if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
282
- throw new ScopeShrinkBlockedError(ctx.uid, lastRecord?.syncMode ?? "unknown", syncMode, shrinkPlan.dirty, shrinkPlan.clean, adviceContext);
247
+ throw new ScopeShrinkBlockedError(run.ctx.uid, lastRecord?.syncMode ?? "unknown", run.syncMode, shrinkPlan.dirty, shrinkPlan.clean, adviceContext);
283
248
  }
284
- // Bulk guard: refuse to auto-move more than the safety cap of CLEAN files in
285
- // a single foreground sync. A deliberate large narrow goes through
286
- // `hq sync narrow --apply` (its own confirmation); `--force-scope-shrink` (or
287
- // raising HQ_SYNC_MAX_AUTO_PRUNE) overrides. Cap of 0 = unlimited. Skipped
288
- // under auto-recover — quarantine is non-destructive so a big recovery is
289
- // safe, and the runner has no way to act on a thrown cap. The engine moves
290
- // nothing when it throws here.
291
249
  const autoPruneCap = resolveAutoPruneCap();
292
250
  if (!effectiveForce &&
293
251
  autoPruneCap > 0 &&
294
252
  shrinkPlan.clean.length > autoPruneCap) {
295
- throw new ScopeShrinkLargePruneError(ctx.uid, syncMode, shrinkPlan.clean.length, autoPruneCap, adviceContext);
253
+ throw new ScopeShrinkLargePruneError(run.ctx.uid, run.syncMode, shrinkPlan.clean.length, autoPruneCap, adviceContext);
296
254
  }
297
- // Clean orphans are QUARANTINED (moved into `.hq/scope-quarantine/<slug>/`,
298
- // recoverable), never silently deleted — a background sync purging local
299
- // files unannounced was DEV-1768 fix #3. The quarantine root lives under the
300
- // real HQ root's `.hq/` (outside `companyRoot` and never pushed), so moved
301
- // files don't round-trip back through S3.
302
- const scopeQuarantineRoot = path.join(hqRoot, ".hq", "scope-quarantine", journalSlug);
255
+ const scopeQuarantineRoot = path.join(run.hqRoot, ".hq", "scope-quarantine", run.journalSlug);
303
256
  const shrinkResult = applyScopeShrink({
304
- journal,
257
+ journal: run.journal,
305
258
  plan: shrinkPlan,
306
- hqRoot: companyRoot,
259
+ hqRoot: run.companyRoot,
307
260
  forceScopeShrink: effectiveForce,
308
261
  reason: "scope_shrink",
309
262
  cleanDisposition: "quarantine",
310
263
  quarantineRoot: scopeQuarantineRoot,
311
264
  });
312
- // Surface each affected orphan explicitly (named path) so the prune is never
313
- // silent. Quarantined clean files render as `deleted: true` (removed from the
314
- // working tree, recoverable in quarantine); dirty files KEPT on disk render
315
- // as a non-deletion notice so the operator knows they were un-tracked, not
316
- // removed. The Rust menubar parser already handles `deleted: true`.
317
265
  for (const relPath of shrinkResult.quarantinedPaths) {
318
- emit({
266
+ run.emit({
319
267
  type: "progress",
320
268
  path: relPath,
321
269
  bytes: 0,
@@ -324,7 +272,7 @@ async function syncWithOperationLockHeld(options) {
324
272
  });
325
273
  }
326
274
  for (const relPath of shrinkResult.removedPaths) {
327
- emit({
275
+ run.emit({
328
276
  type: "progress",
329
277
  path: relPath,
330
278
  bytes: 0,
@@ -333,498 +281,314 @@ async function syncWithOperationLockHeld(options) {
333
281
  });
334
282
  }
335
283
  for (const relPath of shrinkResult.dirtyKeptPaths) {
336
- emit({
284
+ run.emit({
337
285
  type: "progress",
338
286
  path: relPath,
339
287
  bytes: 0,
340
288
  message: "scope-narrowed: locally-modified file KEPT on disk, un-tracked from sync (outside scope)",
341
289
  });
342
290
  }
343
- // "Removed from the working tree" = deleted OR quarantined; both vacate the
344
- // file's original path. Reported as `scopeOrphansRemoved` for back-compat.
345
- const scopeOrphansRemoved = shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined;
346
- // Stage 2: execute the plan. Per-item branching mirrors the pre-refactor
347
- // inline loop; the only structural change is that classification has
348
- // already happened (so `localHash` is reused instead of re-hashing).
349
- //
350
- // 5.36.0: download items go through a bounded-concurrent pool
351
- // (`TRANSFER_CONCURRENCY`, default 16, tunable via
352
- // `HQ_SYNC_TRANSFER_CONCURRENCY`) for 4-8x speedup on transfer-heavy
353
- // syncs. Conflict items stay serial — they may prompt the operator
354
- // and the abort path must short-circuit before any further work. We
355
- // partition the plan items into "conflict (serial)" and "download
356
- // (parallel)" buckets and run the serial pass first; the parallel pass
357
- // only runs if no conflict aborted.
358
- //
359
- // Per-file `progress` events fire at the moment each individual download
360
- // settles (inside the pool wrapper), NOT in plan-walk order. The cross-
361
- // file interleave is acceptable: the menubar stream parser already
362
- // handles per-company interleave, and the same shape applies within a
363
- // single company's pool. Per-file event-count correctness is preserved
364
- // (one progress per download, one error per failure).
365
- const TRANSFER_CONCURRENCY = (() => {
366
- const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
367
- if (raw === undefined || raw === "")
368
- return 16;
369
- const parsed = Number.parseInt(raw, 10);
370
- return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
371
- })();
372
- // First pass: serial walk for non-download outcomes (skips + conflicts).
373
- // Conflicts may set `aborted = true` and short-circuit the whole pull;
374
- // we detect that and skip the parallel pass. Download items are
375
- // collected into `downloadItems[]` for the pool pass below.
291
+ return {
292
+ shrinkPlan,
293
+ shrinkResult,
294
+ scopeOrphansRemoved: shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined,
295
+ };
296
+ }
297
+ async function refreshRunContextIfExpiring(run) {
298
+ if (isExpiringSoon(run.ctx.expiresAt)) {
299
+ run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
300
+ }
301
+ }
302
+ async function executeConflictExecutor(run, plan, scopeRun, counters) {
376
303
  const downloadItems = [];
377
- let aborted = false;
378
- let abortResult = null;
379
304
  for (const item of plan.items) {
380
- if (aborted)
381
- break;
382
305
  if (item.action === "skip-ignored" ||
383
306
  item.action === "skip-personal-mode" ||
384
307
  item.action === "skip-unchanged" ||
385
308
  item.action === "skip-local-only") {
386
- filesSkipped++;
309
+ counters.filesSkipped++;
387
310
  continue;
388
311
  }
389
312
  if (item.action === "skip-excluded-policy") {
390
- // Policy-excluded items count separately from `filesSkipped` so the
391
- // pull result mirrors the push side's `filesExcludedByPolicy`
392
- // counter — `filesSkipped` stays a measure of "unchanged on this
393
- // run", not a catch-all for everything we didn't download.
394
313
  continue;
395
314
  }
396
315
  if (item.action === "skip-out-of-scope") {
397
- // Outside the effective `syncMode` scope (US-005). Counted on its own
398
- // axis so `filesSkipped` keeps meaning "unchanged on this run" — these
399
- // are "deliberately not downloaded because of your sync scope".
400
- filesOutOfScope++;
316
+ counters.filesOutOfScope++;
401
317
  continue;
402
318
  }
403
319
  if (item.action === "tombstone-delete") {
404
- // Authoritative FILE_TOMBSTONE delete (delete-resync): the remote object
405
- // is present but a tombstone marks the key intentionally deleted and it is
406
- // not a newer re-create. Delete any local copy and drop the journal entry
407
- // so it stays gone — the mirror of the journal-vs-LIST tombstone executor
408
- // below, but WITHOUT the HEAD-verify (the remote object is present by
409
- // definition; the FILE_TOMBSTONE is the deletion authority). The planner
410
- // already routed any divergent local copy to `conflict`, so a local file
411
- // reaching here matches the deleted baseline and is safe to remove.
412
- const tombstoneKey = item.remoteFile.key;
413
- // Same Windows-backslash landmine guard as the journal-tombstone executor:
414
- // a malformed key must never reach fs.unlinkSync (path.join collapses the
415
- // backslashes onto a REAL POSIX file). Traversal keys are likewise
416
- // refused before any local filesystem or journal mutation.
417
- const tombstonePath = resolveContainedVaultPath(companyRoot, tombstoneKey);
418
- if (tombstonePath === null)
419
- continue;
420
- try {
421
- const lstat = fs.lstatSync(tombstonePath);
422
- if (tombstoneTargetDiverged(journal, tombstoneKey, tombstonePath, lstat)) {
423
- continue;
424
- }
425
- if (lstat.isSymbolicLink() || lstat.isFile()) {
426
- fs.unlinkSync(tombstonePath);
427
- }
428
- // A directory at the key: don't recursively rm-rf the operator's dir;
429
- // just drop the journal entry (safe-by-default, same as the other path).
430
- }
431
- catch (err) {
432
- const code = err && typeof err === "object" && "code" in err
433
- ? err.code
434
- : undefined;
435
- // ENOENT → local already absent (the common case: a fresh machine that
436
- // never held the file, or a prior pull already removed it) → drop the
437
- // journal entry and converge. Other errors (EACCES/EPERM/…) leave the
438
- // file in place; surface and KEEP the journal entry so the next sync
439
- // retries rather than forgetting the delete.
440
- if (code !== "ENOENT") {
441
- emit({
442
- type: "error",
443
- path: tombstoneKey,
444
- message: `tombstone-suppress unlink failed: ${err instanceof Error ? err.message : String(err)}`,
445
- });
446
- continue;
447
- }
448
- }
449
- removeEntry(journal, tombstoneKey);
450
- filesTombstoned++;
451
- emit({ type: "progress", path: tombstoneKey, bytes: 0 });
320
+ executeFileTombstoneDelete(run, item, counters);
452
321
  continue;
453
322
  }
454
323
  if (item.action === "download") {
455
324
  downloadItems.push(item);
456
325
  continue;
457
326
  }
458
- const { remoteFile, localPath } = item;
459
- // Auto-refresh context if credentials expiring (kept in execute phase
460
- // because Stage 1 is fast — no need to refresh just to classify).
461
- if (isExpiringSoon(ctx.expiresAt)) {
462
- ctx = await refreshEntityContext(companyRef, vaultConfig);
327
+ const abortResult = await executeConflictItem(run, plan, scopeRun, counters, downloadItems, item);
328
+ if (abortResult) {
329
+ return { downloadItems, abortResult };
463
330
  }
464
- if (item.action === "conflict") {
465
- // ── Convergence guard ────────────────────────────────────────────
466
- // The planner flags a conflict purely from journal-relative deltas:
467
- // local hash != journal hash AND remote etag != journal etag. It can
468
- // NEVER compare local bytes against remote bytes directly — the remote
469
- // LIST only carries {key, size, etag, lastModified}, and ListObjectsV2
470
- // returns no content hash. So when the journal baseline goes stale,
471
- // both deltas fire even though local and remote are byte-for-byte
472
- // identical. Stale-baseline triggers seen in the wild: a shared-journal
473
- // cross-root collision (personal vault + companies/personal sharing one
474
- // journal), an mtime-rounding fast-path miss (journal stamps 1700..351,
475
- // the FS returns 1700..350.96, the `===` fast-path misses and re-hashes
476
- // — harmless on its own but combines with the etag delta), KMS/multipart
477
- // etag churn on a no-op re-upload, a second machine advancing S3 + its
478
- // own journal, or a manual revert. Materializing such a false positive
479
- // as a conflict litters a useless byte-identical `.conflict-*` mirror
480
- // and, under `--on-conflict abort`, halts the WHOLE sync over zero real
481
- // divergence. (One live run produced 140 byte-identical mirrors this way.)
482
- //
483
- // We have no remote content hash up front, so prove convergence by
484
- // fetching the remote bytes once — to the very path the conflict mirror
485
- // would occupy and hashing them. Identical bytes are not a conflict:
486
- // re-stamp the journal baseline (so neither side looks changed next run)
487
- // and skip. Genuine divergence reuses the already-fetched bytes as the
488
- // inspection mirror, so the common keep/skip path costs no extra I/O.
489
- const detectedAt = new Date().toISOString();
490
- const machineId = readShortMachineId(hqRoot);
491
- const originalRelative = path.relative(hqRoot, localPath);
492
- const conflictRelative = buildConflictPath(originalRelative, detectedAt, machineId);
493
- const conflictAbs = path.join(hqRoot, conflictRelative);
494
- const conflictKey = toPosixKey(path.relative(companyRoot, conflictAbs));
495
- if (!isDownloadWritePathStillContained(companyRoot, conflictKey, conflictAbs)) {
496
- filesSkipped++;
497
- emit({
498
- type: "error",
499
- path: remoteFile.key,
500
- message: "conflict mirror skipped: local parent escaped the sync root",
501
- });
502
- continue;
503
- }
504
- let remoteFetched = false;
505
- let converged = false;
506
- try {
507
- const downloaded = await downloadFile(ctx, remoteFile.key, conflictAbs);
508
- remoteFetched = true;
509
- // Hash the fetched remote exactly the way the planner hashed local
510
- // (symlink-aware) so the two hashes are directly comparable. A
511
- // symlink record round-trips to a symlink on disk; hashing its
512
- // target string matches `hashSymlinkTarget(localPath)`.
513
- const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
514
- ? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
515
- : (downloaded.contentHash ?? hashFile(conflictAbs));
516
- converged = remoteHash === item.localHash;
517
- }
518
- catch (probeErr) {
519
- // Couldn't fetch or hash the remote — fail safe by falling through to
520
- // the conventional conflict path (converged stays false). No mirror
521
- // is on disk in this case.
522
- emit({
523
- type: "error",
524
- path: remoteFile.key,
525
- message: `conflict convergence probe failed: ${probeErr instanceof Error ? probeErr.message : String(probeErr)}`,
526
- });
527
- }
528
- if (converged) {
529
- // False positive: remote == local. Drop the byte-identical mirror and
530
- // re-stamp the baseline (current localHash + current remoteEtag) so
531
- // the next sync sees "no change on either side". Counts as a skip.
532
- if (remoteFetched) {
533
- try {
534
- fs.rmSync(conflictAbs, { force: true });
535
- }
536
- catch {
537
- /* best-effort cleanup; a stray identical mirror is harmless */
538
- }
539
- }
540
- updateEntry(journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
541
- emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
542
- filesSkipped++;
543
- continue;
544
- }
545
- // ── Genuine divergence ───────────────────────────────────────────
546
- conflicts++;
547
- conflictPaths.push(remoteFile.key);
548
- const resolution = await resolveConflict({
549
- path: remoteFile.key,
550
- localHash: item.localHash,
551
- remoteModified: remoteFile.lastModified,
552
- // Use the lstat-mtime captured by the planner — statSync
553
- // here would follow a dangling symlink and throw ENOENT,
554
- // aborting the pull before resolveConflict could prompt.
555
- localModified: item.localMtime,
556
- direction: "pull",
557
- }, onConflict);
558
- emit({
559
- type: "conflict",
560
- path: remoteFile.key,
561
- direction: "pull",
562
- resolution,
563
- });
564
- // The remote bytes were already fetched to `conflictAbs` by the
565
- // convergence probe. For "keep"/"skip" they become the
566
- // `<original>.conflict-<ts>-<machine>.<ext>` inspection mirror — just
567
- // index it (no second download). For "abort" (user gave up) and
568
- // "overwrite" (cloud bytes are about to replace local) the mirror is
569
- // redundant, so discard it. Best-effort: failure here only emits an
570
- // error, doesn't break the sync.
571
- if (resolution !== "abort" && resolution !== "overwrite") {
572
- if (remoteFetched) {
573
- try {
574
- appendConflictEntry(hqRoot, {
575
- id: buildConflictId(originalRelative, detectedAt),
576
- originalPath: originalRelative,
577
- conflictPath: conflictRelative,
578
- detectedAt,
579
- side: "pull",
580
- machineId,
581
- localHash: item.localHash,
582
- remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
583
- });
584
- }
585
- catch (mirrorErr) {
586
- emit({
587
- type: "error",
588
- path: remoteFile.key,
589
- message: `conflict mirror index write failed: ${mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)}`,
590
- });
591
- }
592
- }
593
- // If the probe download failed (!remoteFetched) there is no mirror on
594
- // disk; the probe already emitted the error. The conflict is still
595
- // surfaced and journal-stamped below so it doesn't re-fire silently.
596
- }
597
- else if (remoteFetched) {
598
- try {
599
- fs.rmSync(conflictAbs, { force: true });
600
- }
601
- catch {
602
- /* best-effort; a leftover mirror is cosmetic, not corrupting */
603
- }
604
- }
605
- if (resolution === "abort") {
606
- emit({ type: "new-files", files: [] });
607
- writeJournal(journalSlug, journal);
608
- aborted = true;
609
- abortResult = {
610
- filesDownloaded,
611
- bytesDownloaded,
612
- filesSkipped,
613
- conflicts,
614
- conflictPaths,
615
- aborted: true,
616
- newFiles: plan.newFiles,
617
- newFilesCount: plan.newFilesCount,
618
- filesExcludedByPolicy: plan.filesExcludedByPolicy,
619
- // Abort short-circuits before the tombstone loop runs; report
620
- // 0 so the field shape stays stable for consumers that
621
- // destructure it.
622
- filesTombstoned: 0,
623
- // Scope-shrink ran before execution, so its counts are real even on
624
- // a conflict abort. `filesOutOfScope` reflects how far the serial
625
- // pass got before the abort; that's acceptable for an abort result.
626
- filesOutOfScope,
627
- scopeOrphansRemoved,
628
- scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
629
- };
630
- break;
631
- }
632
- if (resolution === "keep" || resolution === "skip") {
633
- filesSkipped++;
634
- // Stamp the journal with the new baseline so the same conflict
635
- // doesn't re-fire on every subsequent sync. After "keep", local
636
- // wins — the user has accepted that the cloud version we just
637
- // mirrored is what cloud is at this etag, and they don't want
638
- // it. Recording (current localHash + current remoteEtag) tells
639
- // the next sync "no change on either side" until something new
640
- // diverges. Without this, both `localChanged` and `remoteChanged`
641
- // stay true forever and the conflict is sticky.
642
- // Stamp from planner-captured size (symlink-aware), NOT
643
- // statSync — which would follow a dangling symlink and
644
- // throw ENOENT, get swallowed, and leave the journal
645
- // stale so this conflict would re-fire on every sync
646
- // forever. localSize is sourced from the same lstat that
647
- // computed localMtime + localHash above.
648
- updateEntry(journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
649
- continue;
650
- }
651
- // "overwrite" falls through to download — re-route through the pool
652
- // so it benefits from parallelism too. Synthesize a download item
653
- // pointing at the same remoteFile/localPath; isNew=false because
654
- // there was a conflict-eligible local file present.
655
- downloadItems.push({
656
- action: "download",
657
- remoteFile,
658
- localPath,
659
- isNew: false,
331
+ }
332
+ return { downloadItems, abortResult: null };
333
+ }
334
+ function executeFileTombstoneDelete(run, item, counters) {
335
+ const tombstoneKey = item.remoteFile.key;
336
+ const tombstonePath = resolveContainedVaultPath(run.companyRoot, tombstoneKey);
337
+ if (tombstonePath === null)
338
+ return;
339
+ try {
340
+ const lstat = fs.lstatSync(tombstonePath);
341
+ if (tombstoneTargetDiverged(run.journal, tombstoneKey, tombstonePath, lstat)) {
342
+ return;
343
+ }
344
+ if (lstat.isSymbolicLink() || lstat.isFile()) {
345
+ fs.unlinkSync(tombstonePath);
346
+ }
347
+ }
348
+ catch (err) {
349
+ const code = err && typeof err === "object" && "code" in err
350
+ ? err.code
351
+ : undefined;
352
+ if (code !== "ENOENT") {
353
+ run.emit({
354
+ type: "error",
355
+ path: tombstoneKey,
356
+ message: `tombstone-suppress unlink failed: ${err instanceof Error ? err.message : String(err)}`,
660
357
  });
661
- continue;
358
+ return;
662
359
  }
663
360
  }
664
- // Early-return on conflict abort BEFORE running the parallel download
665
- // pool — the abort intent is "stop the pull now", not "stop new work but
666
- // finish what's in flight". Since the pool hasn't started yet, this is
667
- // a clean drain (zero items in flight) by construction.
668
- if (aborted && abortResult) {
669
- return abortResult;
361
+ removeEntry(run.journal, tombstoneKey);
362
+ counters.filesTombstoned++;
363
+ run.emit({ type: "progress", path: tombstoneKey, bytes: 0 });
364
+ }
365
+ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems, item) {
366
+ const { remoteFile, localPath } = item;
367
+ await refreshRunContextIfExpiring(run);
368
+ const detectedAt = new Date().toISOString();
369
+ const machineId = readShortMachineId(run.hqRoot);
370
+ const originalRelative = path.relative(run.hqRoot, localPath);
371
+ const conflictRelative = buildConflictPath(originalRelative, detectedAt, machineId);
372
+ const conflictAbs = path.join(run.hqRoot, conflictRelative);
373
+ const conflictKey = toPosixKey(path.relative(run.companyRoot, conflictAbs));
374
+ if (!isDownloadWritePathStillContained(run.companyRoot, conflictKey, conflictAbs)) {
375
+ counters.filesSkipped++;
376
+ run.emit({
377
+ type: "error",
378
+ path: remoteFile.key,
379
+ message: "conflict mirror skipped: local parent escaped the sync root",
380
+ });
381
+ return null;
670
382
  }
671
- // ── Parallel download pool (5.36.0) ───────────────────────────────────
672
- // Bounded concurrency: TRANSFER_CONCURRENCY simultaneous downloads. Same
673
- // race-based shape as the HEAD-verify pool above but applied to the body
674
- // of each download. Per-item progress events fire at file-settle time
675
- // (inside the wrapper), so cross-file interleave is expected and the
676
- // menubar's stream parser already handles it.
677
- if (downloadItems.length > 0) {
678
- // Batch pre-mint GET URLs for every download in one shot (chunked server-
679
- // side) so the pool below — and the new-files HEAD enrichment that follows,
680
- // which re-reads the same keys — reuse them instead of presigning per file.
681
- // On a large initial pull this is the difference between ~ceil(N/100)
682
- // presign calls and N (which would 429 past the 100-req/hr limit). No-op
683
- // on the S3 SDK transport; best-effort (failure falls back to per-file).
684
- await primeObjectTransport(ctx, "get", downloadItems.map((d) => d.remoteFile.key));
685
- const queue = [...downloadItems];
686
- const inFlight = new Set();
687
- const downloadOne = async (downloadItem) => {
688
- const { remoteFile, localPath } = downloadItem;
689
- // Auto-refresh context if credentials expiring. Each task checks
690
- // independently — refresh is idempotent on the same context object.
691
- if (isExpiringSoon(ctx.expiresAt)) {
692
- ctx = await refreshEntityContext(companyRef, vaultConfig);
383
+ let remoteFetched = false;
384
+ let converged = false;
385
+ try {
386
+ const downloaded = await downloadFile(run.ctx, remoteFile.key, conflictAbs);
387
+ remoteFetched = true;
388
+ const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
389
+ ? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
390
+ : (downloaded.contentHash ?? hashFile(conflictAbs));
391
+ converged = remoteHash === item.localHash;
392
+ }
393
+ catch (probeErr) {
394
+ run.emit({
395
+ type: "error",
396
+ path: remoteFile.key,
397
+ message: `conflict convergence probe failed: ${probeErr instanceof Error ? probeErr.message : String(probeErr)}`,
398
+ });
399
+ }
400
+ if (converged) {
401
+ if (remoteFetched) {
402
+ try {
403
+ fs.rmSync(conflictAbs, { force: true });
693
404
  }
694
- if (!isDownloadWritePathStillContained(companyRoot, remoteFile.key, localPath)) {
695
- filesSkipped++;
696
- emit({
697
- type: "error",
698
- path: remoteFile.key,
699
- message: "download skipped: local parent escaped the sync root",
700
- });
701
- return;
405
+ catch {
406
+ /* best-effort cleanup; a stray identical mirror is harmless */
702
407
  }
408
+ }
409
+ updateEntry(run.journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
410
+ run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
411
+ counters.filesSkipped++;
412
+ return null;
413
+ }
414
+ counters.conflicts++;
415
+ counters.conflictPaths.push(remoteFile.key);
416
+ const resolution = await resolveConflict({
417
+ path: remoteFile.key,
418
+ localHash: item.localHash,
419
+ remoteModified: remoteFile.lastModified,
420
+ localModified: item.localMtime,
421
+ direction: "pull",
422
+ }, run.options.onConflict);
423
+ run.emit({
424
+ type: "conflict",
425
+ path: remoteFile.key,
426
+ direction: "pull",
427
+ resolution,
428
+ });
429
+ if (resolution !== "abort" && resolution !== "overwrite") {
430
+ if (remoteFetched) {
703
431
  try {
704
- const { metadata, contentHash, contentSize } = await downloadFile(ctx, remoteFile.key, localPath);
705
- const author = metadata?.["created-by"] ?? null;
706
- // Author sub for the scope-shrink authorship guard — same field the
707
- // upload side stamps, read straight off the GET response metadata.
708
- const createdBySub = metadata?.["created-by-sub"];
709
- // Symlink records materialize as real symlinks on disk. lstat
710
- // (does not follow) lets us detect that case so the journal stamp
711
- // mirrors what the push side would emit on the next tick:
712
- // hash = sha256(readlink target string)
713
- // size = 0
714
- // Without this check, hashFile would follow the link and stamp the
715
- // target file's contents — a value the next push would never
716
- // produce — which makes skipUnchanged perpetually re-upload every
717
- // symlink, defeating the point of the gate.
718
- const localLstat = fs.lstatSync(localPath);
719
- const isLocalSymlink = localLstat.isSymbolicLink();
720
- const hash = isLocalSymlink
721
- ? hashSymlinkTarget(fs.readlinkSync(localPath))
722
- : (contentHash ?? hashFile(localPath));
723
- const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
724
- // Capture the listing's ETag so subsequent syncs can detect remote
725
- // drift independently of mtime drift. Stamp mtimeMs from localLstat
726
- // (5.36.0) so the next push planner's lstat fast-path can skip the
727
- // SHA256 for this file without reading its bytes.
728
- //
729
- // 5.37.0 ordering invariant: downloadFile applies hq-mtime via
730
- // utimesSync AFTER its byte write but BEFORE returning, and this
731
- // lstat runs AFTER downloadFile resolves — so localLstat.mtimeMs
732
- // already reflects the source-stamped mtime, not the wall-clock
733
- // write-time. The journal therefore matches what the next push's
734
- // lstat fast-path will see, and the file is correctly skipped on
735
- // re-sync instead of being hashed every tick. Do not move this
736
- // lstat earlier; do not stamp the journal from any pre-download
737
- // mtime.
738
- updateEntry(journal, remoteFile.key, hash, size, "down", remoteFile.etag, localLstat.mtimeMs, createdBySub);
739
- // Attach message from the prior journal entry if present (set by a
740
- // previous `share` operation that included a --message).
741
- const priorEntry = getEntry(journal, remoteFile.key);
742
- const remoteJournalMessage = priorEntry?.message;
743
- emit({
744
- type: "progress",
745
- path: remoteFile.key,
746
- bytes: size,
747
- ...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
748
- ...(author ? { author } : {}),
432
+ appendConflictEntry(run.hqRoot, {
433
+ id: buildConflictId(originalRelative, detectedAt),
434
+ originalPath: originalRelative,
435
+ conflictPath: conflictRelative,
436
+ detectedAt,
437
+ side: "pull",
438
+ machineId,
439
+ localHash: item.localHash,
440
+ remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
749
441
  });
750
- filesDownloaded++;
751
- bytesDownloaded += size;
752
442
  }
753
- catch (err) {
754
- // STS session policy may deny access to some paths — this is expected
755
- // for guest members with allowedPrefixes
756
- if (isAccessDenied(err)) {
757
- filesSkipped++;
758
- }
759
- else {
760
- emit({
761
- type: "error",
762
- path: remoteFile.key,
763
- message: err instanceof Error ? err.message : String(err),
764
- });
765
- }
766
- }
767
- };
768
- // Codex P1 (5.36.x): worker promises wrapped in .catch so an
769
- // unhandled rejection inside downloadOne (e.g. refreshEntityContext
770
- // before the per-item try/catch, or lstatSync after the download
771
- // succeeded but before journal stamping) cannot escape
772
- // `Promise.race(inFlight)` and unwind the drain mid-flight. Without
773
- // this wrap, sibling downloads kept running after share()/sync()
774
- // had already failed, their files materialized on disk without
775
- // matching journal entries, and the next sync re-downloaded
776
- // everything. Errors are collected and surfaced after the pool
777
- // fully drains — see workerErrors throw below.
778
- const workerErrors = [];
779
- while (queue.length > 0 || inFlight.size > 0) {
780
- while (inFlight.size < TRANSFER_CONCURRENCY && queue.length > 0) {
781
- const downloadItem = queue.shift();
782
- const p = downloadOne(downloadItem)
783
- .catch((err) => {
784
- workerErrors.push(err instanceof Error ? err : new Error(String(err)));
785
- })
786
- .finally(() => {
787
- inFlight.delete(p);
443
+ catch (mirrorErr) {
444
+ run.emit({
445
+ type: "error",
446
+ path: remoteFile.key,
447
+ message: `conflict mirror index write failed: ${mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)}`,
788
448
  });
789
- inFlight.add(p);
790
- }
791
- if (inFlight.size > 0) {
792
- // Wait for at least one in-flight task to settle before topping up
793
- // the pool. allSettled-style semantics via Promise.race — the
794
- // .catch wrap above guarantees no worker promise can reject.
795
- await Promise.race(Array.from(inFlight));
796
449
  }
797
450
  }
798
- // Pool drained. If any worker rejected, write the journal first
799
- // (so the lstat fast-path stamps for successfully-downloaded files
800
- // persist) then throw the first error, preserving its stack.
801
- if (workerErrors.length > 0) {
802
- writeJournal(journalSlug, journal);
803
- const first = workerErrors[0];
804
- if (workerErrors.length > 1) {
805
- first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
806
- }
807
- throw first;
451
+ }
452
+ else if (remoteFetched) {
453
+ try {
454
+ fs.rmSync(conflictAbs, { force: true });
808
455
  }
456
+ catch {
457
+ /* best-effort; a leftover mirror is cosmetic, not corrupting */
458
+ }
459
+ }
460
+ if (resolution === "abort") {
461
+ run.emit({ type: "new-files", files: [] });
462
+ writeJournal(run.journalSlug, run.journal);
463
+ return {
464
+ filesDownloaded: counters.filesDownloaded,
465
+ bytesDownloaded: counters.bytesDownloaded,
466
+ filesSkipped: counters.filesSkipped,
467
+ conflicts: counters.conflicts,
468
+ conflictPaths: counters.conflictPaths,
469
+ aborted: true,
470
+ newFiles: plan.newFiles,
471
+ newFilesCount: plan.newFilesCount,
472
+ filesExcludedByPolicy: plan.filesExcludedByPolicy,
473
+ filesTombstoned: 0,
474
+ filesOutOfScope: counters.filesOutOfScope,
475
+ scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
476
+ scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
477
+ };
478
+ }
479
+ if (resolution === "keep" || resolution === "skip") {
480
+ counters.filesSkipped++;
481
+ updateEntry(run.journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
482
+ return null;
483
+ }
484
+ downloadItems.push({
485
+ action: "download",
486
+ remoteFile,
487
+ localPath,
488
+ isNew: false,
489
+ });
490
+ return null;
491
+ }
492
+ async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
493
+ if (downloadItems.length === 0)
494
+ return;
495
+ await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
496
+ const queue = [...downloadItems];
497
+ const inFlight = new Set();
498
+ const workerErrors = [];
499
+ while (queue.length > 0 || inFlight.size > 0) {
500
+ while (inFlight.size < transferConcurrency && queue.length > 0) {
501
+ const downloadItem = queue.shift();
502
+ const p = downloadOne(run, downloadItem, counters)
503
+ .catch((err) => {
504
+ workerErrors.push(err instanceof Error ? err : new Error(String(err)));
505
+ })
506
+ .finally(() => {
507
+ inFlight.delete(p);
508
+ });
509
+ inFlight.add(p);
510
+ }
511
+ if (inFlight.size > 0) {
512
+ await Promise.race(Array.from(inFlight));
513
+ }
514
+ }
515
+ if (workerErrors.length > 0) {
516
+ writeJournal(run.journalSlug, run.journal);
517
+ const first = workerErrors[0];
518
+ if (workerErrors.length > 1) {
519
+ first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
520
+ }
521
+ throw first;
522
+ }
523
+ }
524
+ async function downloadOne(run, downloadItem, counters) {
525
+ const { remoteFile, localPath } = downloadItem;
526
+ await refreshRunContextIfExpiring(run);
527
+ if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
528
+ counters.filesSkipped++;
529
+ run.emit({
530
+ type: "error",
531
+ path: remoteFile.key,
532
+ message: "download skipped: local parent escaped the sync root",
533
+ });
534
+ return;
535
+ }
536
+ try {
537
+ const { metadata, contentHash, contentSize } = await downloadFile(run.ctx, remoteFile.key, localPath);
538
+ const author = metadata?.["created-by"] ?? null;
539
+ const createdBySub = metadata?.["created-by-sub"];
540
+ const localLstat = fs.lstatSync(localPath);
541
+ const isLocalSymlink = localLstat.isSymbolicLink();
542
+ const hash = isLocalSymlink
543
+ ? hashSymlinkTarget(fs.readlinkSync(localPath))
544
+ : (contentHash ?? hashFile(localPath));
545
+ const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
546
+ updateEntry(run.journal, remoteFile.key, hash, size, "down", remoteFile.etag, localLstat.mtimeMs, createdBySub);
547
+ const priorEntry = getEntry(run.journal, remoteFile.key);
548
+ const remoteJournalMessage = priorEntry?.message;
549
+ run.emit({
550
+ type: "progress",
551
+ path: remoteFile.key,
552
+ bytes: size,
553
+ ...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
554
+ ...(author ? { author } : {}),
555
+ });
556
+ counters.filesDownloaded++;
557
+ counters.bytesDownloaded += size;
809
558
  }
810
- // ── New-files attribution (US-002) ─────────────────────────────────────
811
- // Enrich plan.newFiles with `addedBy` from S3 user metadata. HeadObject
812
- // calls are best-effort and capped at 5 concurrent to avoid hammering S3.
559
+ catch (err) {
560
+ if (isAccessDenied(err)) {
561
+ counters.filesSkipped++;
562
+ }
563
+ else {
564
+ run.emit({
565
+ type: "error",
566
+ path: remoteFile.key,
567
+ message: err instanceof Error ? err.message : String(err),
568
+ });
569
+ }
570
+ }
571
+ }
572
+ async function emitAndReportNewFiles(run, plan) {
813
573
  const enrichedNewFiles = [];
574
+ // Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
575
+ // created-by HEADs below reuse the cache instead of each minting its own
576
+ // presign. Without this, a big catch-up pull (hundreds of new files) bursts
577
+ // the presign endpoint, trips the circuit breaker, and every enrichment HEAD
578
+ // then fails. Mirrors the tombstone HEAD-verify pre-prime.
579
+ await primeObjectTransport(run.ctx, "get", plan.newFiles.map((nf) => nf.path));
814
580
  const HEAD_CONCURRENCY = 5;
815
581
  for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
816
582
  const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
817
583
  const results = await Promise.all(batch.map(async (nf) => {
818
584
  let addedBy = null;
819
585
  try {
820
- const head = await headRemoteFile(ctx, nf.path);
586
+ const head = await headRemoteFile(run.ctx, nf.path);
821
587
  if (head?.metadata?.["created-by"]) {
822
588
  addedBy = head.metadata["created-by"];
823
589
  }
824
590
  }
825
591
  catch (headErr) {
826
- // Best-effort: log to console (Sentry captures via global handler)
827
- // and fall through with addedBy = null.
828
592
  try {
829
593
  console.error(`[hq-sync] HeadObject failed for ${nf.path}: ${headErr instanceof Error ? headErr.message : String(headErr)}`);
830
594
  }
@@ -836,75 +600,49 @@ async function syncWithOperationLockHeld(options) {
836
600
  }));
837
601
  enrichedNewFiles.push(...results);
838
602
  }
839
- emit({ type: "new-files", files: enrichedNewFiles });
840
- // Report new files to the notification service so they persist as a
841
- // cross-session "new files" history in the HQ Sync app (POST
842
- // /v1/notify/file-added → per-recipient FILE_EVENT rows for THIS user, who is
843
- // the one the files are new for). Best-effort and bounded: a failure or a
844
- // hung request must never delay or break the sync — the durable signal is the
845
- // synced file itself, this is only a notification mirror.
846
- await reportNewFilesToNotify(vaultConfig, ctx.uid, ctx.slug, enrichedNewFiles);
847
- // Codex P1 (PR #24 follow-up): scope-gate tombstone candidates with
848
- // a per-object HEAD before unlinking. `listRemoteFiles` is STS-scoped
849
- // (guest sessions with `allowedPrefixes`, role downgrade, custom
850
- // sync-mode prefix sets see the AccessDenied branch in the download
851
- // loop above), so a journal entry's absence from LIST does not prove
852
- // the object was deleted; it may simply be invisible to this session.
853
- // HEAD each candidate:
854
- // - HEAD returns metadata → object exists → NOT in our LIST scope →
855
- // skip the tombstone (peer didn't delete it; we just can't see it).
856
- // - HEAD returns null (NotFound) → confirmed deleted → tombstone.
857
- // - HEAD throws AccessDenied → can't tell → defensive skip; journal
858
- // stays so next sync (with broader scope) can re-evaluate.
859
- // - HEAD throws transient → defensive skip + emit error.
860
- // Bounded concurrency mirrors the new-files attribution pass above.
861
- if (plan.tombstones.length > 0) {
862
- // Pre-mint GET URLs for the tombstone HEAD-verify probes below (headRemote
863
- // File presigns a GET), so a large delete set doesn't add N presign calls.
864
- await primeObjectTransport(ctx, "get", plan.tombstones);
865
- const HEAD_VERIFY_CONCURRENCY = 5;
866
- const verified = [];
867
- for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
868
- const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
869
- const results = await Promise.all(batch.map(async (key) => {
870
- try {
871
- const head = await headRemoteFile(ctx, key);
872
- return head === null ? key : null;
873
- }
874
- catch (err) {
875
- if (isAccessDenied(err))
876
- return null;
877
- emit({
878
- type: "error",
879
- path: key,
880
- message: `tombstone HEAD verify failed (deferring): ${err instanceof Error ? err.message : String(err)}`,
881
- });
603
+ run.emit({ type: "new-files", files: enrichedNewFiles });
604
+ await reportNewFilesToNotify(run.vaultConfig, run.ctx.uid, run.ctx.slug, enrichedNewFiles);
605
+ }
606
+ async function verifyPlannedJournalTombstones(run, plan) {
607
+ if (plan.tombstones.length === 0)
608
+ return;
609
+ await primeObjectTransport(run.ctx, "get", plan.tombstones);
610
+ const HEAD_VERIFY_CONCURRENCY = 5;
611
+ const verified = [];
612
+ for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
613
+ const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
614
+ const results = await Promise.all(batch.map(async (key) => {
615
+ try {
616
+ const head = await headRemoteFile(run.ctx, key);
617
+ return head === null ? key : null;
618
+ }
619
+ catch (err) {
620
+ if (isAccessDenied(err))
882
621
  return null;
883
- }
884
- }));
885
- for (const k of results) {
886
- if (k !== null)
887
- verified.push(k);
622
+ run.emit({
623
+ type: "error",
624
+ path: key,
625
+ message: `tombstone HEAD verify failed (deferring): ${err instanceof Error ? err.message : String(err)}`,
626
+ });
627
+ return null;
888
628
  }
629
+ }));
630
+ for (const k of results) {
631
+ if (k !== null)
632
+ verified.push(k);
889
633
  }
890
- plan.tombstones = verified;
891
634
  }
892
- // Bug #9 — apply cross-machine delete propagation. Each tombstone is a
893
- // key the journal records as previously synced but the remote LIST no
894
- // longer contains. We delete the local file (or symlink, or empty dir
895
- // remnant) and drop the journal entry so the next sync's planner stays
896
- // converged. Failures are reported but non-fatal — the entry stays in
897
- // the journal and the next run retries.
635
+ plan.tombstones = verified;
636
+ }
637
+ function executeJournalTombstoneDeletes(run, plan, counters) {
898
638
  for (const key of plan.tombstones) {
899
- // Last line of defense: a malformed or traversal key must NEVER reach
900
- // fs.unlinkSync or journal mutation for a path outside the sync root.
901
- const localPath = resolveContainedVaultPath(companyRoot, key);
639
+ const localPath = resolveContainedVaultPath(run.companyRoot, key);
902
640
  if (localPath === null)
903
641
  continue;
904
642
  let removedSomething = false;
905
643
  try {
906
644
  const lstat = fs.lstatSync(localPath);
907
- if (tombstoneTargetDiverged(journal, key, localPath, lstat)) {
645
+ if (tombstoneTargetDiverged(run.journal, key, localPath, lstat)) {
908
646
  continue;
909
647
  }
910
648
  if (lstat.isSymbolicLink() || lstat.isFile()) {
@@ -912,130 +650,74 @@ async function syncWithOperationLockHeld(options) {
912
650
  removedSomething = true;
913
651
  }
914
652
  else if (lstat.isDirectory()) {
915
- // A dir at a key likely from a (local-dir, cloud-file) historic
916
- // state. Don't recursively rm-rf the operator's dir; just drop
917
- // the journal entry so we converge with reality.
653
+ // A dir at a key is converged by dropping only the journal entry.
918
654
  }
919
655
  }
920
656
  catch (err) {
921
657
  const code = err && typeof err === "object" && "code" in err
922
658
  ? err.code
923
659
  : undefined;
924
- // ENOENT → local already gone; safe to drop the journal entry.
925
- // Other errors (EACCES/EPERM/EBUSY/etc.) leave the local file in
926
- // place — if we dropped the journal entry anyway, the pull side
927
- // would forget the peer's delete and a later push could re-upload
928
- // the still-present local file, silently undoing the peer's delete.
929
- // Surface the error and KEEP the journal entry so the next sync
930
- // retries the unlink after the operator fixes the permission.
931
660
  if (code !== "ENOENT") {
932
- emit({
661
+ run.emit({
933
662
  type: "error",
934
663
  path: key,
935
664
  message: `tombstone unlink failed: ${err instanceof Error ? err.message : String(err)}`,
936
665
  });
937
- // Skip removeEntry / filesTombstoned / progress event — the
938
- // tombstone hasn't actually been honored. Next sync retries.
939
666
  continue;
940
667
  }
941
668
  }
942
- removeEntry(journal, key);
943
- filesTombstoned++;
944
- emit({
669
+ removeEntry(run.journal, key);
670
+ counters.filesTombstoned++;
671
+ run.emit({
945
672
  type: "progress",
946
673
  path: key,
947
674
  bytes: 0,
948
675
  deleted: true,
949
- // Suffix differentiates a tombstone from a normal delete in the
950
- // tty stream — matches the push-side `defaultConsoleLogger`
951
- // tombstone surface in share.ts.
952
676
  message: removedSomething ? "tombstone (cross-machine delete)" : "tombstone (already absent locally)",
953
677
  });
954
678
  }
955
- // Record this pull's boundary (US-005) so the NEXT pull can diff its scope
956
- // against ours and detect a shrink. Append before the journal write so it
957
- // persists. `prefixSet` is stored in the same company-relative namespace as
958
- // the journal keys; `all` mode records `[""]` (covers everything).
959
- appendPullRecord(journal, {
679
+ }
680
+ function finalizePullRun(run, plan, scopeRun, counters) {
681
+ appendPullRecord(run.journal, {
960
682
  pullId: generatePullId(),
961
- companyUid: ctx.uid,
962
- startedAt,
683
+ companyUid: run.ctx.uid,
684
+ startedAt: run.startedAt,
963
685
  completedAt: new Date().toISOString(),
964
- syncMode,
965
- prefixSet: currentPrefixSet,
966
- scopeChangeDetected: shrinkPlan.scopeChangeDetected,
967
- orphansRemoved: scopeOrphansRemoved,
968
- orphansBlocked: shrinkResult.dirtyTombstoned,
686
+ syncMode: run.syncMode,
687
+ prefixSet: run.currentPrefixSet,
688
+ scopeChangeDetected: scopeRun.shrinkPlan.scopeChangeDetected,
689
+ orphansRemoved: scopeRun.scopeOrphansRemoved,
690
+ orphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
969
691
  });
970
- // Stamp lastSync on every successful run so the menubar's "Last sync · X ago"
971
- // ticks even when nothing transferred. updateEntry only fires on actual
972
- // downloads; without this, a no-op sync leaves lastSync at the time of the
973
- // last file change, which is misleading.
974
- journal.lastSync = new Date().toISOString();
975
- writeJournal(journalSlug, journal);
976
- // When the pull actually changed on-disk sources (new files, tombstoned
977
- // removals, or scope-orphan cleanups), refresh the generated skill wrappers,
978
- // personal-overlay mirrors, and workers registry. reindex is idempotent and
979
- // best-effort — it must never fail a sync, and is skipped on no-op syncs
980
- // (the common daemon case) and when the caller opts out via skipReindex.
981
- const changedOnDisk = filesDownloaded > 0 ||
982
- filesTombstoned > 0 ||
983
- scopeOrphansRemoved > 0;
984
- if (!options.skipReindex && changedOnDisk) {
692
+ run.journal.lastSync = new Date().toISOString();
693
+ writeJournal(run.journalSlug, run.journal);
694
+ const changedOnDisk = counters.filesDownloaded > 0 ||
695
+ counters.filesTombstoned > 0 ||
696
+ scopeRun.scopeOrphansRemoved > 0;
697
+ if (!run.options.skipReindex && changedOnDisk) {
985
698
  try {
986
- // skipLock: the surrounding sync run already holds this root's operation
987
- // lock; reindex re-acquiring would refuse against our own live PID.
988
- reindex({ repoRoot: hqRoot, skipLock: true });
699
+ reindex({ repoRoot: run.hqRoot, skipLock: true });
989
700
  }
990
701
  catch {
991
702
  // best-effort: a post-sync refresh failure never fails the sync
992
703
  }
993
704
  }
994
705
  return {
995
- filesDownloaded,
996
- bytesDownloaded,
997
- filesSkipped,
998
- conflicts,
999
- conflictPaths,
706
+ filesDownloaded: counters.filesDownloaded,
707
+ bytesDownloaded: counters.bytesDownloaded,
708
+ filesSkipped: counters.filesSkipped,
709
+ conflicts: counters.conflicts,
710
+ conflictPaths: counters.conflictPaths,
1000
711
  aborted: false,
1001
712
  newFiles: plan.newFiles,
1002
713
  newFilesCount: plan.newFilesCount,
1003
714
  filesExcludedByPolicy: plan.filesExcludedByPolicy,
1004
- filesTombstoned,
1005
- filesOutOfScope,
1006
- scopeOrphansRemoved,
1007
- scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
715
+ filesTombstoned: counters.filesTombstoned,
716
+ filesOutOfScope: counters.filesOutOfScope,
717
+ scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
718
+ scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
1008
719
  };
1009
720
  }
1010
- /**
1011
- * Resolve active company from .hq/config.json.
1012
- */
1013
- function resolveActiveCompany(hqRoot) {
1014
- const configPath = path.join(hqRoot, ".hq", "config.json");
1015
- if (fs.existsSync(configPath)) {
1016
- try {
1017
- const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
1018
- return config.activeCompany ?? config.companySlug;
1019
- }
1020
- catch {
1021
- // Ignore parse errors
1022
- }
1023
- }
1024
- return undefined;
1025
- }
1026
- /**
1027
- * Returns true when the remote object appears to have moved since the
1028
- * journal entry's last-recorded sync. Prefers ETag equality; falls back to
1029
- * `lastModified > syncedAt` for legacy entries written before remoteEtag
1030
- * was tracked. Conservative on tie (`<=` skews "remote unchanged").
1031
- */
1032
- function hasRemoteChanged(remote, entry) {
1033
- if (entry.remoteEtag) {
1034
- return normalizeEtag(remote.etag) !== entry.remoteEtag;
1035
- }
1036
- const syncedAt = new Date(entry.syncedAt).getTime();
1037
- return remote.lastModified.getTime() > syncedAt;
1038
- }
1039
721
  /**
1040
722
  * Decide whether a remote object present in the LIST is a GENUINE RE-CREATE
1041
723
  * written AFTER a FILE_TOMBSTONE — in which case the tombstone is stale and the
@@ -1617,15 +1299,6 @@ fileTombstones = new Map()) {
1617
1299
  tombstones,
1618
1300
  };
1619
1301
  }
1620
- /**
1621
- * Check if an error is an S3 access denied (expected for filtered guests).
1622
- */
1623
- function isAccessDenied(err) {
1624
- if (err && typeof err === "object" && "name" in err) {
1625
- return err.name === "AccessDenied" || err.name === "Forbidden";
1626
- }
1627
- return false;
1628
- }
1629
1302
  /**
1630
1303
  * Default human-readable event rendering. Preserves the exact output format
1631
1304
  * that `hq sync` emitted before SyncProgressEvent was introduced, so callers