@indigoai-us/hq-cloud 6.11.12 → 6.11.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/.github/workflows/ci.yml +17 -26
  2. package/dist/bin/sync-runner-company.d.ts +35 -0
  3. package/dist/bin/sync-runner-company.d.ts.map +1 -0
  4. package/dist/bin/sync-runner-company.js +290 -0
  5. package/dist/bin/sync-runner-company.js.map +1 -0
  6. package/dist/bin/sync-runner-events.d.ts +12 -0
  7. package/dist/bin/sync-runner-events.d.ts.map +1 -0
  8. package/dist/bin/sync-runner-events.js +12 -0
  9. package/dist/bin/sync-runner-events.js.map +1 -0
  10. package/dist/bin/sync-runner-planning.d.ts +53 -0
  11. package/dist/bin/sync-runner-planning.d.ts.map +1 -0
  12. package/dist/bin/sync-runner-planning.js +59 -0
  13. package/dist/bin/sync-runner-planning.js.map +1 -0
  14. package/dist/bin/sync-runner-rollup.d.ts +24 -0
  15. package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
  16. package/dist/bin/sync-runner-rollup.js +46 -0
  17. package/dist/bin/sync-runner-rollup.js.map +1 -0
  18. package/dist/bin/sync-runner-telemetry.d.ts +5 -0
  19. package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
  20. package/dist/bin/sync-runner-telemetry.js +5 -0
  21. package/dist/bin/sync-runner-telemetry.js.map +1 -0
  22. package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
  23. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
  24. package/dist/bin/sync-runner-watch-loop.js +372 -0
  25. package/dist/bin/sync-runner-watch-loop.js.map +1 -0
  26. package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
  27. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
  28. package/dist/bin/sync-runner-watch-routes.js +74 -0
  29. package/dist/bin/sync-runner-watch-routes.js.map +1 -0
  30. package/dist/bin/sync-runner.d.ts +3 -54
  31. package/dist/bin/sync-runner.d.ts.map +1 -1
  32. package/dist/bin/sync-runner.js +73 -1154
  33. package/dist/bin/sync-runner.js.map +1 -1
  34. package/dist/cli/reindex.d.ts.map +1 -1
  35. package/dist/cli/reindex.js +34 -17
  36. package/dist/cli/reindex.js.map +1 -1
  37. package/dist/cli/reindex.test.js +39 -5
  38. package/dist/cli/reindex.test.js.map +1 -1
  39. package/dist/cli/rescue-classify-ordering.test.js +17 -0
  40. package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
  41. package/dist/cli/rescue-core.d.ts +45 -0
  42. package/dist/cli/rescue-core.d.ts.map +1 -1
  43. package/dist/cli/rescue-core.js +197 -170
  44. package/dist/cli/rescue-core.js.map +1 -1
  45. package/dist/cli/share.d.ts.map +1 -1
  46. package/dist/cli/share.js +235 -677
  47. package/dist/cli/share.js.map +1 -1
  48. package/dist/cli/sync.d.ts.map +1 -1
  49. package/dist/cli/sync.js +412 -727
  50. package/dist/cli/sync.js.map +1 -1
  51. package/dist/cli/sync.test.js +20 -0
  52. package/dist/cli/sync.test.js.map +1 -1
  53. package/dist/daemon-worker.d.ts +2 -2
  54. package/dist/daemon-worker.js +3 -3
  55. package/dist/daemon-worker.js.map +1 -1
  56. package/dist/index.d.ts +2 -0
  57. package/dist/index.d.ts.map +1 -1
  58. package/dist/index.js +3 -0
  59. package/dist/index.js.map +1 -1
  60. package/dist/object-io.js +1 -1
  61. package/dist/object-io.js.map +1 -1
  62. package/dist/remote-pull.d.ts +2 -2
  63. package/dist/remote-pull.d.ts.map +1 -1
  64. package/dist/remote-pull.js +23 -3
  65. package/dist/remote-pull.js.map +1 -1
  66. package/dist/remote-pull.test.js +24 -2
  67. package/dist/remote-pull.test.js.map +1 -1
  68. package/dist/sync/push-receiver.d.ts +6 -0
  69. package/dist/sync/push-receiver.d.ts.map +1 -1
  70. package/dist/sync/push-receiver.js +32 -2
  71. package/dist/sync/push-receiver.js.map +1 -1
  72. package/dist/sync/push-receiver.test.js +31 -0
  73. package/dist/sync/push-receiver.test.js.map +1 -1
  74. package/dist/sync-core.d.ts +27 -0
  75. package/dist/sync-core.d.ts.map +1 -0
  76. package/dist/sync-core.js +54 -0
  77. package/dist/sync-core.js.map +1 -0
  78. package/dist/sync-progress.d.ts +37 -0
  79. package/dist/sync-progress.d.ts.map +1 -0
  80. package/dist/sync-progress.js +104 -0
  81. package/dist/sync-progress.js.map +1 -0
  82. package/dist/sync-progress.test.d.ts +2 -0
  83. package/dist/sync-progress.test.d.ts.map +1 -0
  84. package/dist/sync-progress.test.js +70 -0
  85. package/dist/sync-progress.test.js.map +1 -0
  86. package/dist/vault-client.d.ts.map +1 -1
  87. package/dist/vault-client.js +284 -36
  88. package/dist/vault-client.js.map +1 -1
  89. package/dist/vault-client.test.js +59 -0
  90. package/dist/vault-client.test.js.map +1 -1
  91. package/dist/watcher.d.ts +2 -20
  92. package/dist/watcher.d.ts.map +1 -1
  93. package/dist/watcher.js +3 -113
  94. package/dist/watcher.js.map +1 -1
  95. package/package.json +1 -1
  96. package/src/bin/sync-runner-company.ts +350 -0
  97. package/src/bin/sync-runner-events.ts +25 -0
  98. package/src/bin/sync-runner-planning.ts +121 -0
  99. package/src/bin/sync-runner-rollup.ts +72 -0
  100. package/src/bin/sync-runner-telemetry.ts +8 -0
  101. package/src/bin/sync-runner-watch-loop.ts +443 -0
  102. package/src/bin/sync-runner-watch-routes.ts +86 -0
  103. package/src/bin/sync-runner.ts +96 -1253
  104. package/src/cli/reindex.test.ts +41 -3
  105. package/src/cli/reindex.ts +35 -19
  106. package/src/cli/rescue-classify-ordering.test.ts +20 -0
  107. package/src/cli/rescue-core.ts +252 -176
  108. package/src/cli/share.ts +374 -706
  109. package/src/cli/sync.test.ts +25 -0
  110. package/src/cli/sync.ts +625 -803
  111. package/src/daemon-worker.ts +3 -3
  112. package/src/index.ts +9 -0
  113. package/src/object-io.ts +1 -1
  114. package/src/remote-pull.test.ts +30 -1
  115. package/src/remote-pull.ts +29 -4
  116. package/src/sync/push-receiver.test.ts +35 -0
  117. package/src/sync/push-receiver.ts +41 -2
  118. package/src/sync-core.ts +58 -0
  119. package/src/sync-progress.test.ts +81 -0
  120. package/src/sync-progress.ts +133 -0
  121. package/src/vault-client.test.ts +74 -0
  122. package/src/vault-client.ts +395 -43
  123. package/src/watcher.ts +6 -141
package/src/cli/sync.ts CHANGED
@@ -7,9 +7,14 @@
7
7
 
8
8
  import * as fs from "fs";
9
9
  import * as path from "path";
10
- import type { VaultServiceConfig, SyncJournal } from "../types.js";
10
+ import type {
11
+ EntityContext,
12
+ VaultServiceConfig,
13
+ SyncJournal,
14
+ } from "../types.js";
11
15
  import type { SyncMode } from "../vault-client.js";
12
16
  import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
17
+ import { createSyncProgressRecorder } from "../sync-progress.js";
13
18
  import {
14
19
  downloadFile,
15
20
  listRemoteFiles,
@@ -49,6 +54,12 @@ import {
49
54
  type ScopePrefixInput,
50
55
  } from "../prefix-coalesce.js";
51
56
  import { createIgnoreFilter } from "../ignore.js";
57
+ import {
58
+ hasRemoteChanged,
59
+ isAccessDenied,
60
+ resolveActiveCompany,
61
+ resolveTransferConcurrency,
62
+ } from "../sync-core.js";
52
63
  import { isEphemeralPath, isMalformedVaultKey } from "./share.js";
53
64
  import { resolveConflict } from "./conflict.js";
54
65
  import type { ConflictStrategy, ConflictResolution } from "./conflict.js";
@@ -489,6 +500,52 @@ export interface SyncResult {
489
500
  scopeOrphansBlocked: number;
490
501
  }
491
502
 
503
+ type SyncEventEmitter = (event: SyncProgressEvent) => void;
504
+
505
+ type PullDownloadItem = Extract<PullPlanItem, { action: "download" }>;
506
+
507
+ interface PullRunContext {
508
+ options: SyncOptions;
509
+ companyRef: string;
510
+ vaultConfig: VaultServiceConfig;
511
+ hqRoot: string;
512
+ emit: SyncEventEmitter;
513
+ ctx: EntityContext;
514
+ companyRoot: string;
515
+ shouldSync: (filePath: string, isDir?: boolean) => boolean;
516
+ journalSlug: string;
517
+ startedAt: string;
518
+ journal: SyncJournal;
519
+ remoteFiles: RemoteFile[];
520
+ syncMode: SyncMode;
521
+ currentPrefixSet: string[];
522
+ fileTombstones: ReadonlyMap<string, CompanyTombstone>;
523
+ }
524
+
525
+ interface PullCounters {
526
+ filesDownloaded: number;
527
+ bytesDownloaded: number;
528
+ filesSkipped: number;
529
+ conflicts: number;
530
+ filesTombstoned: number;
531
+ filesOutOfScope: number;
532
+ conflictPaths: string[];
533
+ }
534
+
535
+ interface ScopeShrinkPlanState {
536
+ lastRecord: ReturnType<typeof lastPullRecord>;
537
+ shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
538
+ autoRecover: boolean;
539
+ adviceContext: ScopeShrinkAdviceContext;
540
+ effectiveForce: boolean;
541
+ }
542
+
543
+ interface ScopeShrinkRun {
544
+ shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
545
+ shrinkResult: ReturnType<typeof applyScopeShrink>;
546
+ scopeOrphansRemoved: number;
547
+ }
548
+
492
549
  /**
493
550
  * Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
494
551
  * scope shrink that would delete more than this many CLEAN local files in one
@@ -616,10 +673,55 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
616
673
  async function syncWithOperationLockHeld(
617
674
  options: SyncOptions,
618
675
  ): Promise<SyncResult> {
619
- const { company, onConflict, vaultConfig, hqRoot } = options;
620
- const emit = options.onEvent ?? defaultConsoleLogger;
676
+ const run = await buildPullContext(options);
677
+ const plan = planPull(run);
678
+
679
+ emitPullPlan(run.emit, plan);
680
+
681
+ const scopePlan = planScopeShrink(run);
682
+ const scopeRun = executeScopeShrink(run, scopePlan);
683
+ const counters = createPullCounters();
684
+
685
+ const transferConcurrency = resolveTransferConcurrency();
686
+ const conflictRun = await executeConflictExecutor(
687
+ run,
688
+ plan,
689
+ scopeRun,
690
+ counters,
691
+ );
692
+ if (conflictRun.abortResult) {
693
+ return conflictRun.abortResult;
694
+ }
695
+
696
+ await executeDownloadExecutor(
697
+ run,
698
+ conflictRun.downloadItems,
699
+ transferConcurrency,
700
+ counters,
701
+ );
702
+
703
+ await emitAndReportNewFiles(run, plan);
704
+ await verifyPlannedJournalTombstones(run, plan);
705
+ executeJournalTombstoneDeletes(run, plan, counters);
706
+
707
+ return finalizePullRun(run, plan, scopeRun, counters);
708
+ }
709
+
710
+ async function buildPullContext(options: SyncOptions): Promise<PullRunContext> {
711
+ const { company, vaultConfig, hqRoot } = options;
712
+ const baseEmit = options.onEvent ?? defaultConsoleLogger;
713
+ // Mirror every progress event into the shared cross-process snapshot so the
714
+ // menubar shows live progress for THIS sync regardless of who launched it
715
+ // (auto-sync / Sync Now / CLI). Best-effort; never affects the sync.
716
+ const recordProgress = createSyncProgressRecorder({
717
+ company: company ?? null,
718
+ phase: "pull",
719
+ });
720
+ const emit: SyncEventEmitter = (event) => {
721
+ baseEmit(event);
722
+ recordProgress(event);
723
+ };
621
724
 
622
- // Resolve company
623
725
  const companyRef = company ?? resolveActiveCompany(hqRoot);
624
726
  if (!companyRef) {
625
727
  throw new Error(
@@ -628,184 +730,131 @@ async function syncWithOperationLockHeld(
628
730
  );
629
731
  }
630
732
 
631
- // Resolve entity context
632
- let ctx = await resolveEntityContext(companyRef, vaultConfig);
633
- // Every company's files land under companies/{slug}/ so fanning out multiple
634
- // companies into the same hqRoot doesn't cross-clobber files with overlapping
635
- // S3 keys (e.g. every company has a .hq/manifest.json). Remote keys stay
636
- // company-relative; the prefix lives only on disk.
637
- // In personalMode the journal slug + S3 keys are person-relative (e.g. "docs/foo.md");
638
- // the local target is `hqRoot` directly, NOT `<hqRoot>/companies/<personSlug>/`. This
639
- // keeps round-trip parity with the Rust personal first-push (Step 7) which sources
640
- // `<hqRoot>/docs/foo.md`.
733
+ const ctx = await resolveEntityContext(companyRef, vaultConfig);
641
734
  const companyRoot = options.personalMode === true
642
735
  ? hqRoot
643
736
  : path.join(hqRoot, "companies", ctx.slug);
644
737
  const shouldSync = createIgnoreFilter(hqRoot);
645
738
  const journalSlug = options.journalSlug ?? ctx.slug;
646
739
  const startedAt = new Date().toISOString();
647
- // Personal-vault callers must never start from an empty journal when only
648
- // the legacy `personal` file exists (mass re-download/etag churn). Seeding
649
- // here — inside the engine — covers every consumer (sync-runner already
650
- // seeds; hq-cli historically didn't, which split the vault's bookkeeping
651
- // across two journal files and re-flagged synced files as conflicts).
740
+
652
741
  if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG) migratePersonalVaultJournal();
653
- // Migrate v1 → v2 in place so the scope-shrink / pull-record machinery has
654
- // its fields, and GC any tombstones past the 30-day retention window before
655
- // we re-evaluate orphans (so a long-pruned path can re-download cleanly).
656
742
  const journal = migrateToV2(readJournal(journalSlug));
657
743
  gcTombstones(journal, Date.now());
658
744
 
659
- // ── Effective download scope (US-005) ─────────────────────────────────────
660
- // `all` → prefixSet `[""]`, which `isCoveredByAny` treats as "covers
661
- // everything" — so the download filter and the scope-shrink
662
- // comparison both become no-ops, preserving legacy full-bucket
663
- // behavior bit-for-bit.
664
- // `shared`/`custom` → the coalesced, company-relative prefix set the runner
665
- // resolved. An empty set means "nothing in scope" → download
666
- // nothing (the runner falls back to `all` on resolution errors, so
667
- // empty here is an intentional "nothing shared", never a failure).
668
745
  const syncMode: SyncMode = options.syncMode ?? "all";
669
- const currentPrefixSet =
746
+ const currentPrefixSet: string[] =
670
747
  syncMode === "all" ? [""] : coalescePrefixes(options.prefixSet ?? []);
671
- // Authorship guard input (scope-shrink): the caller's own Cognito sub,
672
- // injected by the entry point (the runner sources it from its decoded
673
- // idToken claims — the same sub stamped onto uploads as `created-by-sub`).
674
- // Undefined degrades safely: own-author files lose their special shield, but
675
- // the `protectUnknownAuthors` conservative path below still prevents a
676
- // routine sync from deleting anything it can't prove is foreign.
677
- const callerSub = options.callerSub;
678
-
679
- let filesDownloaded = 0;
680
- let bytesDownloaded = 0;
681
- let filesSkipped = 0;
682
- let conflicts = 0;
683
- let filesTombstoned = 0;
684
- let filesOutOfScope = 0;
685
- const conflictPaths: string[] = [];
686
748
 
687
- // List all remote files (IAM session policy filters at the AWS layer)
688
749
  const remoteFiles = await listRemoteFiles(ctx);
689
-
690
- // Fetch the company's FILE_TOMBSTONE records so the planner can suppress
691
- // resurrection of an intentionally-deleted object (delete-resync). Done in
692
- // parallel intent with the LIST above conceptually, but kept serial here for
693
- // a clean read of `ctx`; best-effort — a failed read degrades to an empty map
694
- // (no suppression), preserving the pre-fix behavior. ctx.uid is the verified
695
- // companyUid the tombstone rows are keyed under.
696
- //
697
- // SKIP for the personal vault: its `ctx.uid` is a personUid (`prs_…`), but
698
- // `GET /v1/files/tombstones?company=…` is COMPANY-scoped server-side
699
- // (findCallerWithMembership), so a personal-vault request resolves
700
- // `company=prs_…` to no membership and is correctly rejected with
701
- // `403 "No active membership for caller in company prs_…"`. That 403 is
702
- // benign for the pull (it already degrades to the empty map below), but
703
- // hq-pro captures EVERY one as a Sentry warning — the per-personal-vault
704
- // no-membership cluster (one Sentry issue per signed-in user). Personal-vault
705
- // delete-resync was never a committed feature and there is no person-scoped
706
- // tombstone path, so for the personal target we skip the fetch and use an
707
- // empty map — byte-for-byte the current degraded behavior, minus the 403 spam.
708
- // FUTURE FOLLOW-UP (not built here): if personal-vault delete-resync is
709
- // wanted, it needs a real person-scoped tombstone endpoint + client read.
710
- const tombstones =
750
+ const fileTombstones =
711
751
  options.personalMode === true
712
752
  ? new Map<string, CompanyTombstone>()
713
753
  : await fetchCompanyTombstones(vaultConfig, ctx.uid);
714
754
 
715
- // Stage 1: classify every remote file against the journal + local disk.
716
- // Hashing happens here (not in the transfer loop) so the plan event below
717
- // carries an accurate denominator before any progress events fire.
718
- const plan = computePullPlan(
719
- remoteFiles,
720
- journal,
755
+ return {
756
+ options,
757
+ companyRef,
758
+ vaultConfig,
759
+ hqRoot,
760
+ emit,
761
+ ctx,
721
762
  companyRoot,
722
763
  shouldSync,
723
- options.personalMode === true,
724
- options.includeLocalCompanies === true,
725
- options.teamSyncedSlugs ?? null,
764
+ journalSlug,
765
+ startedAt,
766
+ journal,
767
+ remoteFiles,
768
+ syncMode,
726
769
  currentPrefixSet,
727
- tombstones,
770
+ fileTombstones,
771
+ };
772
+ }
773
+
774
+ function planPull(run: PullRunContext): PullPlan {
775
+ return computePullPlan(
776
+ run.remoteFiles,
777
+ run.journal,
778
+ run.companyRoot,
779
+ run.shouldSync,
780
+ run.options.personalMode === true,
781
+ run.options.includeLocalCompanies === true,
782
+ run.options.teamSyncedSlugs ?? null,
783
+ run.currentPrefixSet,
784
+ run.fileTombstones,
728
785
  );
786
+ }
729
787
 
788
+ function emitPullPlan(emit: SyncEventEmitter, plan: PullPlan): void {
730
789
  emit({
731
790
  type: "plan",
732
791
  filesToDownload: plan.filesToDownload,
733
792
  bytesToDownload: plan.bytesToDownload,
734
- // sync() is pull-only; push counts are sourced from share()'s plan event.
735
793
  filesToUpload: 0,
736
794
  bytesToUpload: 0,
737
795
  filesToSkip: plan.filesToSkip,
738
796
  filesToConflict: plan.filesToConflict,
739
- // Authoritative FILE_TOMBSTONE suppressions (delete-resync) are the only
740
- // deletes known at plan time; the journal-vs-LIST tombstones are
741
- // HEAD-verified later and surfaced via the final filesTombstoned count.
742
797
  filesToDelete: plan.filesToTombstoneDelete,
743
798
  });
799
+ }
744
800
 
745
- // ── Scope-shrink cleanup (US-005) ─────────────────────────────────────────
746
- // If the effective scope narrowed since the last pull, files that were
747
- // pulled under the old scope but fall outside the new one are orphans. We
748
- // delete only CLEAN orphans (provably unchanged since last sync); dirty
749
- // (locally-modified) orphans are sacred. By default a dirty orphan aborts
750
- // the leg with a structured error the CLI renders; `forceScopeShrink` keeps
751
- // dirty files on disk and only tombstones their journal entries.
752
- //
753
- // `companyRoot` is passed as the module's `hqRoot` so its `path.join(root,
754
- // key)` resolves company-relative journal keys correctly (the scope-shrink
755
- // module is namespace-agnostic — root + keys + prefixSet must simply agree).
756
- //
757
- // Note: this is the durable selective-download fix for OWNERS. An owner's
758
- // STS is wide (role-bypass), so the remote LIST returns everything and the
759
- // AWS layer never narrows the pull. This client-side shrink is what makes
760
- // `hq sync mode shared` actually stick across re-syncs for an owner.
761
- const lastRecord = lastPullRecord(journal, ctx.uid);
762
- // A missing record, or a v1-migrated record with an empty prefixSet, means
763
- // "no recorded scope" → treat the last scope as full-bucket `all` (`[""]`),
764
- // per the PullRecord.prefixSet contract in types.ts.
801
+ function createPullCounters(): PullCounters {
802
+ return {
803
+ filesDownloaded: 0,
804
+ bytesDownloaded: 0,
805
+ filesSkipped: 0,
806
+ conflicts: 0,
807
+ filesTombstoned: 0,
808
+ filesOutOfScope: 0,
809
+ conflictPaths: [],
810
+ };
811
+ }
812
+
813
+ function planScopeShrink(run: PullRunContext): ScopeShrinkPlanState {
814
+ const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
765
815
  const lastPrefixSet =
766
816
  lastRecord && lastRecord.prefixSet.length > 0
767
817
  ? lastRecord.prefixSet
768
818
  : [""];
769
819
  const shrinkPlan = buildScopeShrinkPlan({
770
- journal,
771
- hqRoot: companyRoot,
820
+ journal: run.journal,
821
+ hqRoot: run.companyRoot,
772
822
  lastPrefixSet,
773
- currentPrefixSet,
774
- callerSub,
775
- // Automatic pull: never auto-prune content the caller authored, and never
776
- // make a destructive guess about unknown-author (legacy) orphans. The
777
- // explicit `hq sync narrow` ritual opts out of the unknown-author shield.
823
+ currentPrefixSet: run.currentPrefixSet,
824
+ callerSub: run.options.callerSub,
778
825
  protectUnknownAuthors: true,
779
826
  });
780
- // Policy: the background menubar runner ("auto-recover") can take no
781
- // interactive flag, so it must never throw on a shrink — it self-heals
782
- // non-destructively (dirty kept on disk + un-tracked, clean quarantined).
783
- // A foreground `hq sync` ("block", the default) keeps the protective gate
784
- // but renders FOLLOWABLE advice. `autoRecover` implies force (proceed) and
785
- // bypasses the bulk-prune cap (quarantine is non-destructive, so a large
786
- // recovery move is safe). DEV-1768.
787
- const scopeShrinkPolicy = options.scopeShrinkPolicy ?? "block";
827
+ const scopeShrinkPolicy = run.options.scopeShrinkPolicy ?? "block";
788
828
  const autoRecover = scopeShrinkPolicy === "auto-recover";
789
829
  const adviceContext: ScopeShrinkAdviceContext = autoRecover ? "runner" : "cli";
790
- const effectiveForce = options.forceScopeShrink === true || autoRecover;
830
+ const effectiveForce = run.options.forceScopeShrink === true || autoRecover;
831
+
832
+ return {
833
+ lastRecord,
834
+ shrinkPlan,
835
+ autoRecover,
836
+ adviceContext,
837
+ effectiveForce,
838
+ };
839
+ }
840
+
841
+ function executeScopeShrink(
842
+ run: PullRunContext,
843
+ scopePlan: ScopeShrinkPlanState,
844
+ ): ScopeShrinkRun {
845
+ const { lastRecord, shrinkPlan, adviceContext, effectiveForce } = scopePlan;
791
846
 
792
847
  if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
793
848
  throw new ScopeShrinkBlockedError(
794
- ctx.uid,
849
+ run.ctx.uid,
795
850
  lastRecord?.syncMode ?? "unknown",
796
- syncMode,
851
+ run.syncMode,
797
852
  shrinkPlan.dirty,
798
853
  shrinkPlan.clean,
799
854
  adviceContext,
800
855
  );
801
856
  }
802
- // Bulk guard: refuse to auto-move more than the safety cap of CLEAN files in
803
- // a single foreground sync. A deliberate large narrow goes through
804
- // `hq sync narrow --apply` (its own confirmation); `--force-scope-shrink` (or
805
- // raising HQ_SYNC_MAX_AUTO_PRUNE) overrides. Cap of 0 = unlimited. Skipped
806
- // under auto-recover — quarantine is non-destructive so a big recovery is
807
- // safe, and the runner has no way to act on a thrown cap. The engine moves
808
- // nothing when it throws here.
857
+
809
858
  const autoPruneCap = resolveAutoPruneCap();
810
859
  if (
811
860
  !effectiveForce &&
@@ -813,40 +862,32 @@ async function syncWithOperationLockHeld(
813
862
  shrinkPlan.clean.length > autoPruneCap
814
863
  ) {
815
864
  throw new ScopeShrinkLargePruneError(
816
- ctx.uid,
817
- syncMode,
865
+ run.ctx.uid,
866
+ run.syncMode,
818
867
  shrinkPlan.clean.length,
819
868
  autoPruneCap,
820
869
  adviceContext,
821
870
  );
822
871
  }
823
- // Clean orphans are QUARANTINED (moved into `.hq/scope-quarantine/<slug>/`,
824
- // recoverable), never silently deleted — a background sync purging local
825
- // files unannounced was DEV-1768 fix #3. The quarantine root lives under the
826
- // real HQ root's `.hq/` (outside `companyRoot` and never pushed), so moved
827
- // files don't round-trip back through S3.
872
+
828
873
  const scopeQuarantineRoot = path.join(
829
- hqRoot,
874
+ run.hqRoot,
830
875
  ".hq",
831
876
  "scope-quarantine",
832
- journalSlug,
877
+ run.journalSlug,
833
878
  );
834
879
  const shrinkResult = applyScopeShrink({
835
- journal,
880
+ journal: run.journal,
836
881
  plan: shrinkPlan,
837
- hqRoot: companyRoot,
882
+ hqRoot: run.companyRoot,
838
883
  forceScopeShrink: effectiveForce,
839
884
  reason: "scope_shrink",
840
885
  cleanDisposition: "quarantine",
841
886
  quarantineRoot: scopeQuarantineRoot,
842
887
  });
843
- // Surface each affected orphan explicitly (named path) so the prune is never
844
- // silent. Quarantined clean files render as `deleted: true` (removed from the
845
- // working tree, recoverable in quarantine); dirty files KEPT on disk render
846
- // as a non-deletion notice so the operator knows they were un-tracked, not
847
- // removed. The Rust menubar parser already handles `deleted: true`.
888
+
848
889
  for (const relPath of shrinkResult.quarantinedPaths) {
849
- emit({
890
+ run.emit({
850
891
  type: "progress",
851
892
  path: relPath,
852
893
  bytes: 0,
@@ -855,7 +896,7 @@ async function syncWithOperationLockHeld(
855
896
  });
856
897
  }
857
898
  for (const relPath of shrinkResult.removedPaths) {
858
- emit({
899
+ run.emit({
859
900
  type: "progress",
860
901
  path: relPath,
861
902
  bytes: 0,
@@ -864,7 +905,7 @@ async function syncWithOperationLockHeld(
864
905
  });
865
906
  }
866
907
  for (const relPath of shrinkResult.dirtyKeptPaths) {
867
- emit({
908
+ run.emit({
868
909
  type: "progress",
869
910
  path: relPath,
870
911
  bytes: 0,
@@ -872,547 +913,410 @@ async function syncWithOperationLockHeld(
872
913
  "scope-narrowed: locally-modified file KEPT on disk, un-tracked from sync (outside scope)",
873
914
  });
874
915
  }
875
- // "Removed from the working tree" = deleted OR quarantined; both vacate the
876
- // file's original path. Reported as `scopeOrphansRemoved` for back-compat.
877
- const scopeOrphansRemoved =
878
- shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined;
879
-
880
- // Stage 2: execute the plan. Per-item branching mirrors the pre-refactor
881
- // inline loop; the only structural change is that classification has
882
- // already happened (so `localHash` is reused instead of re-hashing).
883
- //
884
- // 5.36.0: download items go through a bounded-concurrent pool
885
- // (`TRANSFER_CONCURRENCY`, default 16, tunable via
886
- // `HQ_SYNC_TRANSFER_CONCURRENCY`) for 4-8x speedup on transfer-heavy
887
- // syncs. Conflict items stay serial — they may prompt the operator
888
- // and the abort path must short-circuit before any further work. We
889
- // partition the plan items into "conflict (serial)" and "download
890
- // (parallel)" buckets and run the serial pass first; the parallel pass
891
- // only runs if no conflict aborted.
892
- //
893
- // Per-file `progress` events fire at the moment each individual download
894
- // settles (inside the pool wrapper), NOT in plan-walk order. The cross-
895
- // file interleave is acceptable: the menubar stream parser already
896
- // handles per-company interleave, and the same shape applies within a
897
- // single company's pool. Per-file event-count correctness is preserved
898
- // (one progress per download, one error per failure).
899
- const TRANSFER_CONCURRENCY = (() => {
900
- const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
901
- if (raw === undefined || raw === "") return 16;
902
- const parsed = Number.parseInt(raw, 10);
903
- return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
904
- })();
905
-
906
- // First pass: serial walk for non-download outcomes (skips + conflicts).
907
- // Conflicts may set `aborted = true` and short-circuit the whole pull;
908
- // we detect that and skip the parallel pass. Download items are
909
- // collected into `downloadItems[]` for the pool pass below.
910
- const downloadItems: Array<typeof plan.items[number] & { action: "download" }> = [];
911
- let aborted = false;
912
- let abortResult: SyncResult | null = null;
916
+
917
+ return {
918
+ shrinkPlan,
919
+ shrinkResult,
920
+ scopeOrphansRemoved:
921
+ shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined,
922
+ };
923
+ }
924
+
925
+ async function refreshRunContextIfExpiring(
926
+ run: PullRunContext,
927
+ ): Promise<void> {
928
+ if (isExpiringSoon(run.ctx.expiresAt)) {
929
+ run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
930
+ }
931
+ }
932
+
933
+ async function executeConflictExecutor(
934
+ run: PullRunContext,
935
+ plan: PullPlan,
936
+ scopeRun: ScopeShrinkRun,
937
+ counters: PullCounters,
938
+ ): Promise<{ downloadItems: PullDownloadItem[]; abortResult: SyncResult | null }> {
939
+ const downloadItems: PullDownloadItem[] = [];
913
940
 
914
941
  for (const item of plan.items) {
915
- if (aborted) break;
916
942
  if (
917
943
  item.action === "skip-ignored" ||
918
944
  item.action === "skip-personal-mode" ||
919
945
  item.action === "skip-unchanged" ||
920
946
  item.action === "skip-local-only"
921
947
  ) {
922
- filesSkipped++;
948
+ counters.filesSkipped++;
923
949
  continue;
924
950
  }
925
951
  if (item.action === "skip-excluded-policy") {
926
- // Policy-excluded items count separately from `filesSkipped` so the
927
- // pull result mirrors the push side's `filesExcludedByPolicy`
928
- // counter — `filesSkipped` stays a measure of "unchanged on this
929
- // run", not a catch-all for everything we didn't download.
930
952
  continue;
931
953
  }
932
954
  if (item.action === "skip-out-of-scope") {
933
- // Outside the effective `syncMode` scope (US-005). Counted on its own
934
- // axis so `filesSkipped` keeps meaning "unchanged on this run" — these
935
- // are "deliberately not downloaded because of your sync scope".
936
- filesOutOfScope++;
955
+ counters.filesOutOfScope++;
937
956
  continue;
938
957
  }
939
-
940
958
  if (item.action === "tombstone-delete") {
941
- // Authoritative FILE_TOMBSTONE delete (delete-resync): the remote object
942
- // is present but a tombstone marks the key intentionally deleted and it is
943
- // not a newer re-create. Delete any local copy and drop the journal entry
944
- // so it stays gone — the mirror of the journal-vs-LIST tombstone executor
945
- // below, but WITHOUT the HEAD-verify (the remote object is present by
946
- // definition; the FILE_TOMBSTONE is the deletion authority). The planner
947
- // already routed any divergent local copy to `conflict`, so a local file
948
- // reaching here matches the deleted baseline and is safe to remove.
949
- const tombstoneKey = item.remoteFile.key;
950
- // Same Windows-backslash landmine guard as the journal-tombstone executor:
951
- // a malformed key must never reach fs.unlinkSync (path.join collapses the
952
- // backslashes onto a REAL POSIX file). Traversal keys are likewise
953
- // refused before any local filesystem or journal mutation.
954
- const tombstonePath = resolveContainedVaultPath(companyRoot, tombstoneKey);
955
- if (tombstonePath === null) continue;
956
- try {
957
- const lstat = fs.lstatSync(tombstonePath);
958
- if (tombstoneTargetDiverged(journal, tombstoneKey, tombstonePath, lstat)) {
959
- continue;
960
- }
961
- if (lstat.isSymbolicLink() || lstat.isFile()) {
962
- fs.unlinkSync(tombstonePath);
963
- }
964
- // A directory at the key: don't recursively rm-rf the operator's dir;
965
- // just drop the journal entry (safe-by-default, same as the other path).
966
- } catch (err: unknown) {
967
- const code =
968
- err && typeof err === "object" && "code" in err
969
- ? (err as { code?: string }).code
970
- : undefined;
971
- // ENOENT → local already absent (the common case: a fresh machine that
972
- // never held the file, or a prior pull already removed it) → drop the
973
- // journal entry and converge. Other errors (EACCES/EPERM/…) leave the
974
- // file in place; surface and KEEP the journal entry so the next sync
975
- // retries rather than forgetting the delete.
976
- if (code !== "ENOENT") {
977
- emit({
978
- type: "error",
979
- path: tombstoneKey,
980
- message: `tombstone-suppress unlink failed: ${
981
- err instanceof Error ? err.message : String(err)
982
- }`,
983
- });
984
- continue;
985
- }
986
- }
987
- removeEntry(journal, tombstoneKey);
988
- filesTombstoned++;
989
- emit({ type: "progress", path: tombstoneKey, bytes: 0 });
959
+ executeFileTombstoneDelete(run, item, counters);
990
960
  continue;
991
961
  }
992
-
993
962
  if (item.action === "download") {
994
963
  downloadItems.push(item);
995
964
  continue;
996
965
  }
997
966
 
998
- const { remoteFile, localPath } = item;
967
+ const abortResult = await executeConflictItem(
968
+ run,
969
+ plan,
970
+ scopeRun,
971
+ counters,
972
+ downloadItems,
973
+ item,
974
+ );
975
+ if (abortResult) {
976
+ return { downloadItems, abortResult };
977
+ }
978
+ }
979
+
980
+ return { downloadItems, abortResult: null };
981
+ }
999
982
 
1000
- // Auto-refresh context if credentials expiring (kept in execute phase
1001
- // because Stage 1 is fast — no need to refresh just to classify).
1002
- if (isExpiringSoon(ctx.expiresAt)) {
1003
- ctx = await refreshEntityContext(companyRef, vaultConfig);
983
+ function executeFileTombstoneDelete(
984
+ run: PullRunContext,
985
+ item: Extract<PullPlanItem, { action: "tombstone-delete" }>,
986
+ counters: PullCounters,
987
+ ): void {
988
+ const tombstoneKey = item.remoteFile.key;
989
+ const tombstonePath = resolveContainedVaultPath(run.companyRoot, tombstoneKey);
990
+ if (tombstonePath === null) return;
991
+ try {
992
+ const lstat = fs.lstatSync(tombstonePath);
993
+ if (tombstoneTargetDiverged(run.journal, tombstoneKey, tombstonePath, lstat)) {
994
+ return;
995
+ }
996
+ if (lstat.isSymbolicLink() || lstat.isFile()) {
997
+ fs.unlinkSync(tombstonePath);
1004
998
  }
999
+ } catch (err: unknown) {
1000
+ const code =
1001
+ err && typeof err === "object" && "code" in err
1002
+ ? (err as { code?: string }).code
1003
+ : undefined;
1004
+ if (code !== "ENOENT") {
1005
+ run.emit({
1006
+ type: "error",
1007
+ path: tombstoneKey,
1008
+ message: `tombstone-suppress unlink failed: ${
1009
+ err instanceof Error ? err.message : String(err)
1010
+ }`,
1011
+ });
1012
+ return;
1013
+ }
1014
+ }
1015
+ removeEntry(run.journal, tombstoneKey);
1016
+ counters.filesTombstoned++;
1017
+ run.emit({ type: "progress", path: tombstoneKey, bytes: 0 });
1018
+ }
1005
1019
 
1006
- if (item.action === "conflict") {
1007
- // ── Convergence guard ────────────────────────────────────────────
1008
- // The planner flags a conflict purely from journal-relative deltas:
1009
- // local hash != journal hash AND remote etag != journal etag. It can
1010
- // NEVER compare local bytes against remote bytes directly — the remote
1011
- // LIST only carries {key, size, etag, lastModified}, and ListObjectsV2
1012
- // returns no content hash. So when the journal baseline goes stale,
1013
- // both deltas fire even though local and remote are byte-for-byte
1014
- // identical. Stale-baseline triggers seen in the wild: a shared-journal
1015
- // cross-root collision (personal vault + companies/personal sharing one
1016
- // journal), an mtime-rounding fast-path miss (journal stamps 1700..351,
1017
- // the FS returns 1700..350.96, the `===` fast-path misses and re-hashes
1018
- // harmless on its own but combines with the etag delta), KMS/multipart
1019
- // etag churn on a no-op re-upload, a second machine advancing S3 + its
1020
- // own journal, or a manual revert. Materializing such a false positive
1021
- // as a conflict litters a useless byte-identical `.conflict-*` mirror
1022
- // and, under `--on-conflict abort`, halts the WHOLE sync over zero real
1023
- // divergence. (One live run produced 140 byte-identical mirrors this way.)
1024
- //
1025
- // We have no remote content hash up front, so prove convergence by
1026
- // fetching the remote bytes once — to the very path the conflict mirror
1027
- // would occupy — and hashing them. Identical bytes are not a conflict:
1028
- // re-stamp the journal baseline (so neither side looks changed next run)
1029
- // and skip. Genuine divergence reuses the already-fetched bytes as the
1030
- // inspection mirror, so the common keep/skip path costs no extra I/O.
1031
- const detectedAt = new Date().toISOString();
1032
- const machineId = readShortMachineId(hqRoot);
1033
- const originalRelative = path.relative(hqRoot, localPath);
1034
- const conflictRelative = buildConflictPath(
1035
- originalRelative,
1036
- detectedAt,
1037
- machineId,
1038
- );
1039
- const conflictAbs = path.join(hqRoot, conflictRelative);
1040
- const conflictKey = toPosixKey(path.relative(companyRoot, conflictAbs));
1020
+ async function executeConflictItem(
1021
+ run: PullRunContext,
1022
+ plan: PullPlan,
1023
+ scopeRun: ScopeShrinkRun,
1024
+ counters: PullCounters,
1025
+ downloadItems: PullDownloadItem[],
1026
+ item: Extract<PullPlanItem, { action: "conflict" }>,
1027
+ ): Promise<SyncResult | null> {
1028
+ const { remoteFile, localPath } = item;
1029
+
1030
+ await refreshRunContextIfExpiring(run);
1031
+
1032
+ const detectedAt = new Date().toISOString();
1033
+ const machineId = readShortMachineId(run.hqRoot);
1034
+ const originalRelative = path.relative(run.hqRoot, localPath);
1035
+ const conflictRelative = buildConflictPath(
1036
+ originalRelative,
1037
+ detectedAt,
1038
+ machineId,
1039
+ );
1040
+ const conflictAbs = path.join(run.hqRoot, conflictRelative);
1041
+ const conflictKey = toPosixKey(path.relative(run.companyRoot, conflictAbs));
1042
+
1043
+ if (!isDownloadWritePathStillContained(run.companyRoot, conflictKey, conflictAbs)) {
1044
+ counters.filesSkipped++;
1045
+ run.emit({
1046
+ type: "error",
1047
+ path: remoteFile.key,
1048
+ message: "conflict mirror skipped: local parent escaped the sync root",
1049
+ });
1050
+ return null;
1051
+ }
1041
1052
 
1042
- if (!isDownloadWritePathStillContained(companyRoot, conflictKey, conflictAbs)) {
1043
- filesSkipped++;
1044
- emit({
1045
- type: "error",
1046
- path: remoteFile.key,
1047
- message: "conflict mirror skipped: local parent escaped the sync root",
1048
- });
1049
- continue;
1050
- }
1053
+ let remoteFetched = false;
1054
+ let converged = false;
1055
+ try {
1056
+ const downloaded = await downloadFile(run.ctx, remoteFile.key, conflictAbs);
1057
+ remoteFetched = true;
1058
+ const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
1059
+ ? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
1060
+ : (downloaded.contentHash ?? hashFile(conflictAbs));
1061
+ converged = remoteHash === item.localHash;
1062
+ } catch (probeErr) {
1063
+ run.emit({
1064
+ type: "error",
1065
+ path: remoteFile.key,
1066
+ message: `conflict convergence probe failed: ${
1067
+ probeErr instanceof Error ? probeErr.message : String(probeErr)
1068
+ }`,
1069
+ });
1070
+ }
1051
1071
 
1052
- let remoteFetched = false;
1053
- let converged = false;
1072
+ if (converged) {
1073
+ if (remoteFetched) {
1054
1074
  try {
1055
- const downloaded = await downloadFile(ctx, remoteFile.key, conflictAbs);
1056
- remoteFetched = true;
1057
- // Hash the fetched remote exactly the way the planner hashed local
1058
- // (symlink-aware) so the two hashes are directly comparable. A
1059
- // symlink record round-trips to a symlink on disk; hashing its
1060
- // target string matches `hashSymlinkTarget(localPath)`.
1061
- const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
1062
- ? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
1063
- : (downloaded.contentHash ?? hashFile(conflictAbs));
1064
- converged = remoteHash === item.localHash;
1065
- } catch (probeErr) {
1066
- // Couldn't fetch or hash the remote — fail safe by falling through to
1067
- // the conventional conflict path (converged stays false). No mirror
1068
- // is on disk in this case.
1069
- emit({
1070
- type: "error",
1071
- path: remoteFile.key,
1072
- message: `conflict convergence probe failed: ${
1073
- probeErr instanceof Error ? probeErr.message : String(probeErr)
1074
- }`,
1075
- });
1075
+ fs.rmSync(conflictAbs, { force: true });
1076
+ } catch {
1077
+ /* best-effort cleanup; a stray identical mirror is harmless */
1076
1078
  }
1079
+ }
1080
+ updateEntry(
1081
+ run.journal,
1082
+ remoteFile.key,
1083
+ item.localHash,
1084
+ item.localSize,
1085
+ "down",
1086
+ remoteFile.etag,
1087
+ item.localMtime.getTime(),
1088
+ );
1089
+ run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
1090
+ counters.filesSkipped++;
1091
+ return null;
1092
+ }
1077
1093
 
1078
- if (converged) {
1079
- // False positive: remote == local. Drop the byte-identical mirror and
1080
- // re-stamp the baseline (current localHash + current remoteEtag) so
1081
- // the next sync sees "no change on either side". Counts as a skip.
1082
- if (remoteFetched) {
1083
- try {
1084
- fs.rmSync(conflictAbs, { force: true });
1085
- } catch {
1086
- /* best-effort cleanup; a stray identical mirror is harmless */
1087
- }
1088
- }
1089
- updateEntry(
1090
- journal,
1091
- remoteFile.key,
1092
- item.localHash,
1093
- item.localSize,
1094
- "down",
1095
- remoteFile.etag,
1096
- item.localMtime.getTime(),
1097
- );
1098
- emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
1099
- filesSkipped++;
1100
- continue;
1101
- }
1094
+ counters.conflicts++;
1095
+ counters.conflictPaths.push(remoteFile.key);
1096
+
1097
+ const resolution = await resolveConflict(
1098
+ {
1099
+ path: remoteFile.key,
1100
+ localHash: item.localHash,
1101
+ remoteModified: remoteFile.lastModified,
1102
+ localModified: item.localMtime,
1103
+ direction: "pull",
1104
+ },
1105
+ run.options.onConflict,
1106
+ );
1102
1107
 
1103
- // ── Genuine divergence ───────────────────────────────────────────
1104
- conflicts++;
1105
- conflictPaths.push(remoteFile.key);
1108
+ run.emit({
1109
+ type: "conflict",
1110
+ path: remoteFile.key,
1111
+ direction: "pull",
1112
+ resolution,
1113
+ });
1106
1114
 
1107
- const resolution = await resolveConflict(
1108
- {
1109
- path: remoteFile.key,
1115
+ if (resolution !== "abort" && resolution !== "overwrite") {
1116
+ if (remoteFetched) {
1117
+ try {
1118
+ appendConflictEntry(run.hqRoot, {
1119
+ id: buildConflictId(originalRelative, detectedAt),
1120
+ originalPath: originalRelative,
1121
+ conflictPath: conflictRelative,
1122
+ detectedAt,
1123
+ side: "pull",
1124
+ machineId,
1110
1125
  localHash: item.localHash,
1111
- remoteModified: remoteFile.lastModified,
1112
- // Use the lstat-mtime captured by the planner — statSync
1113
- // here would follow a dangling symlink and throw ENOENT,
1114
- // aborting the pull before resolveConflict could prompt.
1115
- localModified: item.localMtime,
1116
- direction: "pull",
1117
- },
1118
- onConflict,
1119
- );
1120
-
1121
- emit({
1122
- type: "conflict",
1123
- path: remoteFile.key,
1124
- direction: "pull",
1125
- resolution,
1126
- });
1127
-
1128
- // The remote bytes were already fetched to `conflictAbs` by the
1129
- // convergence probe. For "keep"/"skip" they become the
1130
- // `<original>.conflict-<ts>-<machine>.<ext>` inspection mirror — just
1131
- // index it (no second download). For "abort" (user gave up) and
1132
- // "overwrite" (cloud bytes are about to replace local) the mirror is
1133
- // redundant, so discard it. Best-effort: failure here only emits an
1134
- // error, doesn't break the sync.
1135
- if (resolution !== "abort" && resolution !== "overwrite") {
1136
- if (remoteFetched) {
1137
- try {
1138
- appendConflictEntry(hqRoot, {
1139
- id: buildConflictId(originalRelative, detectedAt),
1140
- originalPath: originalRelative,
1141
- conflictPath: conflictRelative,
1142
- detectedAt,
1143
- side: "pull",
1144
- machineId,
1145
- localHash: item.localHash,
1146
- remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
1147
- });
1148
- } catch (mirrorErr) {
1149
- emit({
1150
- type: "error",
1151
- path: remoteFile.key,
1152
- message: `conflict mirror index write failed: ${
1153
- mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
1154
- }`,
1155
- });
1156
- }
1157
- }
1158
- // If the probe download failed (!remoteFetched) there is no mirror on
1159
- // disk; the probe already emitted the error. The conflict is still
1160
- // surfaced and journal-stamped below so it doesn't re-fire silently.
1161
- } else if (remoteFetched) {
1162
- try {
1163
- fs.rmSync(conflictAbs, { force: true });
1164
- } catch {
1165
- /* best-effort; a leftover mirror is cosmetic, not corrupting */
1166
- }
1167
- }
1168
-
1169
- if (resolution === "abort") {
1170
- emit({ type: "new-files", files: [] });
1171
- writeJournal(journalSlug, journal);
1172
- aborted = true;
1173
- abortResult = {
1174
- filesDownloaded,
1175
- bytesDownloaded,
1176
- filesSkipped,
1177
- conflicts,
1178
- conflictPaths,
1179
- aborted: true,
1180
- newFiles: plan.newFiles,
1181
- newFilesCount: plan.newFilesCount,
1182
- filesExcludedByPolicy: plan.filesExcludedByPolicy,
1183
- // Abort short-circuits before the tombstone loop runs; report
1184
- // 0 so the field shape stays stable for consumers that
1185
- // destructure it.
1186
- filesTombstoned: 0,
1187
- // Scope-shrink ran before execution, so its counts are real even on
1188
- // a conflict abort. `filesOutOfScope` reflects how far the serial
1189
- // pass got before the abort; that's acceptable for an abort result.
1190
- filesOutOfScope,
1191
- scopeOrphansRemoved,
1192
- scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
1193
- };
1194
- break;
1195
- }
1196
- if (resolution === "keep" || resolution === "skip") {
1197
- filesSkipped++;
1198
- // Stamp the journal with the new baseline so the same conflict
1199
- // doesn't re-fire on every subsequent sync. After "keep", local
1200
- // wins — the user has accepted that the cloud version we just
1201
- // mirrored is what cloud is at this etag, and they don't want
1202
- // it. Recording (current localHash + current remoteEtag) tells
1203
- // the next sync "no change on either side" until something new
1204
- // diverges. Without this, both `localChanged` and `remoteChanged`
1205
- // stay true forever and the conflict is sticky.
1206
- // Stamp from planner-captured size (symlink-aware), NOT
1207
- // statSync — which would follow a dangling symlink and
1208
- // throw ENOENT, get swallowed, and leave the journal
1209
- // stale so this conflict would re-fire on every sync
1210
- // forever. localSize is sourced from the same lstat that
1211
- // computed localMtime + localHash above.
1212
- updateEntry(
1213
- journal,
1214
- remoteFile.key,
1215
- item.localHash,
1216
- item.localSize,
1217
- "down",
1218
- remoteFile.etag,
1219
- item.localMtime.getTime(),
1220
- );
1221
- continue;
1126
+ remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
1127
+ });
1128
+ } catch (mirrorErr) {
1129
+ run.emit({
1130
+ type: "error",
1131
+ path: remoteFile.key,
1132
+ message: `conflict mirror index write failed: ${
1133
+ mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
1134
+ }`,
1135
+ });
1222
1136
  }
1223
- // "overwrite" falls through to download — re-route through the pool
1224
- // so it benefits from parallelism too. Synthesize a download item
1225
- // pointing at the same remoteFile/localPath; isNew=false because
1226
- // there was a conflict-eligible local file present.
1227
- downloadItems.push({
1228
- action: "download",
1229
- remoteFile,
1230
- localPath,
1231
- isNew: false,
1232
- });
1233
- continue;
1137
+ }
1138
+ } else if (remoteFetched) {
1139
+ try {
1140
+ fs.rmSync(conflictAbs, { force: true });
1141
+ } catch {
1142
+ /* best-effort; a leftover mirror is cosmetic, not corrupting */
1234
1143
  }
1235
1144
  }
1236
1145
 
1237
- // Early-return on conflict abort BEFORE running the parallel download
1238
- // pool — the abort intent is "stop the pull now", not "stop new work but
1239
- // finish what's in flight". Since the pool hasn't started yet, this is
1240
- // a clean drain (zero items in flight) by construction.
1241
- if (aborted && abortResult) {
1242
- return abortResult;
1146
+ if (resolution === "abort") {
1147
+ run.emit({ type: "new-files", files: [] });
1148
+ writeJournal(run.journalSlug, run.journal);
1149
+ return {
1150
+ filesDownloaded: counters.filesDownloaded,
1151
+ bytesDownloaded: counters.bytesDownloaded,
1152
+ filesSkipped: counters.filesSkipped,
1153
+ conflicts: counters.conflicts,
1154
+ conflictPaths: counters.conflictPaths,
1155
+ aborted: true,
1156
+ newFiles: plan.newFiles,
1157
+ newFilesCount: plan.newFilesCount,
1158
+ filesExcludedByPolicy: plan.filesExcludedByPolicy,
1159
+ filesTombstoned: 0,
1160
+ filesOutOfScope: counters.filesOutOfScope,
1161
+ scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
1162
+ scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
1163
+ };
1243
1164
  }
1244
1165
 
1245
- // ── Parallel download pool (5.36.0) ───────────────────────────────────
1246
- // Bounded concurrency: TRANSFER_CONCURRENCY simultaneous downloads. Same
1247
- // race-based shape as the HEAD-verify pool above but applied to the body
1248
- // of each download. Per-item progress events fire at file-settle time
1249
- // (inside the wrapper), so cross-file interleave is expected and the
1250
- // menubar's stream parser already handles it.
1251
- if (downloadItems.length > 0) {
1252
- // Batch pre-mint GET URLs for every download in one shot (chunked server-
1253
- // side) so the pool below — and the new-files HEAD enrichment that follows,
1254
- // which re-reads the same keys — reuse them instead of presigning per file.
1255
- // On a large initial pull this is the difference between ~ceil(N/100)
1256
- // presign calls and N (which would 429 past the 100-req/hr limit). No-op
1257
- // on the S3 SDK transport; best-effort (failure falls back to per-file).
1258
- await primeObjectTransport(
1259
- ctx,
1260
- "get",
1261
- downloadItems.map((d) => d.remoteFile.key),
1166
+ if (resolution === "keep" || resolution === "skip") {
1167
+ counters.filesSkipped++;
1168
+ updateEntry(
1169
+ run.journal,
1170
+ remoteFile.key,
1171
+ item.localHash,
1172
+ item.localSize,
1173
+ "down",
1174
+ remoteFile.etag,
1175
+ item.localMtime.getTime(),
1262
1176
  );
1177
+ return null;
1178
+ }
1263
1179
 
1264
- const queue = [...downloadItems];
1265
- const inFlight: Set<Promise<unknown>> = new Set();
1180
+ downloadItems.push({
1181
+ action: "download",
1182
+ remoteFile,
1183
+ localPath,
1184
+ isNew: false,
1185
+ });
1186
+ return null;
1187
+ }
1266
1188
 
1267
- const downloadOne = async (
1268
- downloadItem: typeof downloadItems[number],
1269
- ): Promise<void> => {
1270
- const { remoteFile, localPath } = downloadItem;
1189
+ async function executeDownloadExecutor(
1190
+ run: PullRunContext,
1191
+ downloadItems: PullDownloadItem[],
1192
+ transferConcurrency: number,
1193
+ counters: PullCounters,
1194
+ ): Promise<void> {
1195
+ if (downloadItems.length === 0) return;
1271
1196
 
1272
- // Auto-refresh context if credentials expiring. Each task checks
1273
- // independently — refresh is idempotent on the same context object.
1274
- if (isExpiringSoon(ctx.expiresAt)) {
1275
- ctx = await refreshEntityContext(companyRef, vaultConfig);
1276
- }
1197
+ await primeObjectTransport(
1198
+ run.ctx,
1199
+ "get",
1200
+ downloadItems.map((d) => d.remoteFile.key),
1201
+ );
1277
1202
 
1278
- if (!isDownloadWritePathStillContained(companyRoot, remoteFile.key, localPath)) {
1279
- filesSkipped++;
1280
- emit({
1281
- type: "error",
1282
- path: remoteFile.key,
1283
- message: "download skipped: local parent escaped the sync root",
1203
+ const queue = [...downloadItems];
1204
+ const inFlight: Set<Promise<unknown>> = new Set();
1205
+ const workerErrors: Error[] = [];
1206
+
1207
+ while (queue.length > 0 || inFlight.size > 0) {
1208
+ while (inFlight.size < transferConcurrency && queue.length > 0) {
1209
+ const downloadItem = queue.shift()!;
1210
+ const p: Promise<void> = downloadOne(run, downloadItem, counters)
1211
+ .catch((err: unknown) => {
1212
+ workerErrors.push(err instanceof Error ? err : new Error(String(err)));
1213
+ })
1214
+ .finally(() => {
1215
+ inFlight.delete(p);
1284
1216
  });
1285
- return;
1286
- }
1217
+ inFlight.add(p);
1218
+ }
1219
+ if (inFlight.size > 0) {
1220
+ await Promise.race(Array.from(inFlight));
1221
+ }
1222
+ }
1287
1223
 
1288
- try {
1289
- const { metadata, contentHash, contentSize } = await downloadFile(
1290
- ctx,
1291
- remoteFile.key,
1292
- localPath,
1293
- );
1294
- const author = metadata?.["created-by"] ?? null;
1295
- // Author sub for the scope-shrink authorship guard — same field the
1296
- // upload side stamps, read straight off the GET response metadata.
1297
- const createdBySub = metadata?.["created-by-sub"];
1298
-
1299
- // Symlink records materialize as real symlinks on disk. lstat
1300
- // (does not follow) lets us detect that case so the journal stamp
1301
- // mirrors what the push side would emit on the next tick:
1302
- // hash = sha256(readlink target string)
1303
- // size = 0
1304
- // Without this check, hashFile would follow the link and stamp the
1305
- // target file's contents — a value the next push would never
1306
- // produce — which makes skipUnchanged perpetually re-upload every
1307
- // symlink, defeating the point of the gate.
1308
- const localLstat = fs.lstatSync(localPath);
1309
- const isLocalSymlink = localLstat.isSymbolicLink();
1310
- const hash = isLocalSymlink
1311
- ? hashSymlinkTarget(fs.readlinkSync(localPath))
1312
- : (contentHash ?? hashFile(localPath));
1313
- const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
1314
- // Capture the listing's ETag so subsequent syncs can detect remote
1315
- // drift independently of mtime drift. Stamp mtimeMs from localLstat
1316
- // (5.36.0) so the next push planner's lstat fast-path can skip the
1317
- // SHA256 for this file without reading its bytes.
1318
- //
1319
- // 5.37.0 ordering invariant: downloadFile applies hq-mtime via
1320
- // utimesSync AFTER its byte write but BEFORE returning, and this
1321
- // lstat runs AFTER downloadFile resolves — so localLstat.mtimeMs
1322
- // already reflects the source-stamped mtime, not the wall-clock
1323
- // write-time. The journal therefore matches what the next push's
1324
- // lstat fast-path will see, and the file is correctly skipped on
1325
- // re-sync instead of being hashed every tick. Do not move this
1326
- // lstat earlier; do not stamp the journal from any pre-download
1327
- // mtime.
1328
- updateEntry(
1329
- journal,
1330
- remoteFile.key,
1331
- hash,
1332
- size,
1333
- "down",
1334
- remoteFile.etag,
1335
- localLstat.mtimeMs,
1336
- createdBySub,
1337
- );
1224
+ if (workerErrors.length > 0) {
1225
+ writeJournal(run.journalSlug, run.journal);
1226
+ const first = workerErrors[0]!;
1227
+ if (workerErrors.length > 1) {
1228
+ first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
1229
+ }
1230
+ throw first;
1231
+ }
1232
+ }
1338
1233
 
1339
- // Attach message from the prior journal entry if present (set by a
1340
- // previous `share` operation that included a --message).
1341
- const priorEntry = getEntry(journal, remoteFile.key);
1342
- const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
1343
- emit({
1344
- type: "progress",
1345
- path: remoteFile.key,
1346
- bytes: size,
1347
- ...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
1348
- ...(author ? { author } : {}),
1349
- });
1234
+ async function downloadOne(
1235
+ run: PullRunContext,
1236
+ downloadItem: PullDownloadItem,
1237
+ counters: PullCounters,
1238
+ ): Promise<void> {
1239
+ const { remoteFile, localPath } = downloadItem;
1350
1240
 
1351
- filesDownloaded++;
1352
- bytesDownloaded += size;
1353
- } catch (err) {
1354
- // STS session policy may deny access to some paths — this is expected
1355
- // for guest members with allowedPrefixes
1356
- if (isAccessDenied(err)) {
1357
- filesSkipped++;
1358
- } else {
1359
- emit({
1360
- type: "error",
1361
- path: remoteFile.key,
1362
- message: err instanceof Error ? err.message : String(err),
1363
- });
1364
- }
1365
- }
1366
- };
1241
+ await refreshRunContextIfExpiring(run);
1367
1242
 
1368
- // Codex P1 (5.36.x): worker promises wrapped in .catch so an
1369
- // unhandled rejection inside downloadOne (e.g. refreshEntityContext
1370
- // before the per-item try/catch, or lstatSync after the download
1371
- // succeeded but before journal stamping) cannot escape
1372
- // `Promise.race(inFlight)` and unwind the drain mid-flight. Without
1373
- // this wrap, sibling downloads kept running after share()/sync()
1374
- // had already failed, their files materialized on disk without
1375
- // matching journal entries, and the next sync re-downloaded
1376
- // everything. Errors are collected and surfaced after the pool
1377
- // fully drains — see workerErrors throw below.
1378
- const workerErrors: Error[] = [];
1379
- while (queue.length > 0 || inFlight.size > 0) {
1380
- while (inFlight.size < TRANSFER_CONCURRENCY && queue.length > 0) {
1381
- const downloadItem = queue.shift()!;
1382
- const p: Promise<void> = downloadOne(downloadItem)
1383
- .catch((err: unknown) => {
1384
- workerErrors.push(err instanceof Error ? err : new Error(String(err)));
1385
- })
1386
- .finally(() => {
1387
- inFlight.delete(p);
1388
- });
1389
- inFlight.add(p);
1390
- }
1391
- if (inFlight.size > 0) {
1392
- // Wait for at least one in-flight task to settle before topping up
1393
- // the pool. allSettled-style semantics via Promise.race — the
1394
- // .catch wrap above guarantees no worker promise can reject.
1395
- await Promise.race(Array.from(inFlight));
1396
- }
1397
- }
1243
+ if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
1244
+ counters.filesSkipped++;
1245
+ run.emit({
1246
+ type: "error",
1247
+ path: remoteFile.key,
1248
+ message: "download skipped: local parent escaped the sync root",
1249
+ });
1250
+ return;
1251
+ }
1398
1252
 
1399
- // Pool drained. If any worker rejected, write the journal first
1400
- // (so the lstat fast-path stamps for successfully-downloaded files
1401
- // persist) then throw the first error, preserving its stack.
1402
- if (workerErrors.length > 0) {
1403
- writeJournal(journalSlug, journal);
1404
- const first = workerErrors[0]!;
1405
- if (workerErrors.length > 1) {
1406
- first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
1407
- }
1408
- throw first;
1253
+ try {
1254
+ const { metadata, contentHash, contentSize } = await downloadFile(
1255
+ run.ctx,
1256
+ remoteFile.key,
1257
+ localPath,
1258
+ );
1259
+ const author = metadata?.["created-by"] ?? null;
1260
+ const createdBySub = metadata?.["created-by-sub"];
1261
+
1262
+ const localLstat = fs.lstatSync(localPath);
1263
+ const isLocalSymlink = localLstat.isSymbolicLink();
1264
+ const hash = isLocalSymlink
1265
+ ? hashSymlinkTarget(fs.readlinkSync(localPath))
1266
+ : (contentHash ?? hashFile(localPath));
1267
+ const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
1268
+
1269
+ updateEntry(
1270
+ run.journal,
1271
+ remoteFile.key,
1272
+ hash,
1273
+ size,
1274
+ "down",
1275
+ remoteFile.etag,
1276
+ localLstat.mtimeMs,
1277
+ createdBySub,
1278
+ );
1279
+
1280
+ const priorEntry = getEntry(run.journal, remoteFile.key);
1281
+ const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
1282
+ run.emit({
1283
+ type: "progress",
1284
+ path: remoteFile.key,
1285
+ bytes: size,
1286
+ ...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
1287
+ ...(author ? { author } : {}),
1288
+ });
1289
+
1290
+ counters.filesDownloaded++;
1291
+ counters.bytesDownloaded += size;
1292
+ } catch (err) {
1293
+ if (isAccessDenied(err)) {
1294
+ counters.filesSkipped++;
1295
+ } else {
1296
+ run.emit({
1297
+ type: "error",
1298
+ path: remoteFile.key,
1299
+ message: err instanceof Error ? err.message : String(err),
1300
+ });
1409
1301
  }
1410
1302
  }
1303
+ }
1411
1304
 
1412
- // ── New-files attribution (US-002) ─────────────────────────────────────
1413
- // Enrich plan.newFiles with `addedBy` from S3 user metadata. HeadObject
1414
- // calls are best-effort and capped at 5 concurrent to avoid hammering S3.
1305
+ async function emitAndReportNewFiles(
1306
+ run: PullRunContext,
1307
+ plan: PullPlan,
1308
+ ): Promise<void> {
1415
1309
  const enrichedNewFiles: Array<{ path: string; bytes: number; addedBy: string | null }> = [];
1310
+ // Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
1311
+ // created-by HEADs below reuse the cache instead of each minting its own
1312
+ // presign. Without this, a big catch-up pull (hundreds of new files) bursts
1313
+ // the presign endpoint, trips the circuit breaker, and every enrichment HEAD
1314
+ // then fails. Mirrors the tombstone HEAD-verify pre-prime.
1315
+ await primeObjectTransport(
1316
+ run.ctx,
1317
+ "get",
1318
+ plan.newFiles.map((nf) => nf.path),
1319
+ );
1416
1320
  const HEAD_CONCURRENCY = 5;
1417
1321
  for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
1418
1322
  const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
@@ -1420,13 +1324,11 @@ async function syncWithOperationLockHeld(
1420
1324
  batch.map(async (nf) => {
1421
1325
  let addedBy: string | null = null;
1422
1326
  try {
1423
- const head = await headRemoteFile(ctx, nf.path);
1327
+ const head = await headRemoteFile(run.ctx, nf.path);
1424
1328
  if (head?.metadata?.["created-by"]) {
1425
1329
  addedBy = head.metadata["created-by"];
1426
1330
  }
1427
1331
  } catch (headErr) {
1428
- // Best-effort: log to console (Sentry captures via global handler)
1429
- // and fall through with addedBy = null.
1430
1332
  try {
1431
1333
  console.error(
1432
1334
  `[hq-sync] HeadObject failed for ${nf.path}: ${
@@ -1442,220 +1344,150 @@ async function syncWithOperationLockHeld(
1442
1344
  );
1443
1345
  enrichedNewFiles.push(...results);
1444
1346
  }
1445
- emit({ type: "new-files", files: enrichedNewFiles });
1446
-
1447
- // Report new files to the notification service so they persist as a
1448
- // cross-session "new files" history in the HQ Sync app (POST
1449
- // /v1/notify/file-added → per-recipient FILE_EVENT rows for THIS user, who is
1450
- // the one the files are new for). Best-effort and bounded: a failure or a
1451
- // hung request must never delay or break the sync — the durable signal is the
1452
- // synced file itself, this is only a notification mirror.
1453
- await reportNewFilesToNotify(vaultConfig, ctx.uid, ctx.slug, enrichedNewFiles);
1454
-
1455
- // Codex P1 (PR #24 follow-up): scope-gate tombstone candidates with
1456
- // a per-object HEAD before unlinking. `listRemoteFiles` is STS-scoped
1457
- // (guest sessions with `allowedPrefixes`, role downgrade, custom
1458
- // sync-mode prefix sets — see the AccessDenied branch in the download
1459
- // loop above), so a journal entry's absence from LIST does not prove
1460
- // the object was deleted; it may simply be invisible to this session.
1461
- // HEAD each candidate:
1462
- // - HEAD returns metadata → object exists → NOT in our LIST scope →
1463
- // skip the tombstone (peer didn't delete it; we just can't see it).
1464
- // - HEAD returns null (NotFound) confirmed deleted tombstone.
1465
- // - HEAD throws AccessDenied can't tell → defensive skip; journal
1466
- // stays so next sync (with broader scope) can re-evaluate.
1467
- // - HEAD throws transient → defensive skip + emit error.
1468
- // Bounded concurrency mirrors the new-files attribution pass above.
1469
- if (plan.tombstones.length > 0) {
1470
- // Pre-mint GET URLs for the tombstone HEAD-verify probes below (headRemote
1471
- // File presigns a GET), so a large delete set doesn't add N presign calls.
1472
- await primeObjectTransport(ctx, "get", plan.tombstones);
1473
-
1474
- const HEAD_VERIFY_CONCURRENCY = 5;
1475
- const verified: string[] = [];
1476
- for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
1477
- const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
1478
- const results = await Promise.all(
1479
- batch.map(async (key) => {
1480
- try {
1481
- const head = await headRemoteFile(ctx, key);
1482
- return head === null ? key : null;
1483
- } catch (err) {
1484
- if (isAccessDenied(err)) return null;
1485
- emit({
1486
- type: "error",
1487
- path: key,
1488
- message: `tombstone HEAD verify failed (deferring): ${
1489
- err instanceof Error ? err.message : String(err)
1490
- }`,
1491
- });
1492
- return null;
1493
- }
1494
- }),
1495
- );
1496
- for (const k of results) {
1497
- if (k !== null) verified.push(k);
1498
- }
1347
+ run.emit({ type: "new-files", files: enrichedNewFiles });
1348
+ await reportNewFilesToNotify(
1349
+ run.vaultConfig,
1350
+ run.ctx.uid,
1351
+ run.ctx.slug,
1352
+ enrichedNewFiles,
1353
+ );
1354
+ }
1355
+
1356
+ async function verifyPlannedJournalTombstones(
1357
+ run: PullRunContext,
1358
+ plan: PullPlan,
1359
+ ): Promise<void> {
1360
+ if (plan.tombstones.length === 0) return;
1361
+
1362
+ await primeObjectTransport(run.ctx, "get", plan.tombstones);
1363
+
1364
+ const HEAD_VERIFY_CONCURRENCY = 5;
1365
+ const verified: string[] = [];
1366
+ for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
1367
+ const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
1368
+ const results = await Promise.all(
1369
+ batch.map(async (key) => {
1370
+ try {
1371
+ const head = await headRemoteFile(run.ctx, key);
1372
+ return head === null ? key : null;
1373
+ } catch (err) {
1374
+ if (isAccessDenied(err)) return null;
1375
+ run.emit({
1376
+ type: "error",
1377
+ path: key,
1378
+ message: `tombstone HEAD verify failed (deferring): ${
1379
+ err instanceof Error ? err.message : String(err)
1380
+ }`,
1381
+ });
1382
+ return null;
1383
+ }
1384
+ }),
1385
+ );
1386
+ for (const k of results) {
1387
+ if (k !== null) verified.push(k);
1499
1388
  }
1500
- plan.tombstones = verified;
1501
1389
  }
1390
+ plan.tombstones = verified;
1391
+ }
1502
1392
 
1503
- // Bug #9 — apply cross-machine delete propagation. Each tombstone is a
1504
- // key the journal records as previously synced but the remote LIST no
1505
- // longer contains. We delete the local file (or symlink, or empty dir
1506
- // remnant) and drop the journal entry so the next sync's planner stays
1507
- // converged. Failures are reported but non-fatal — the entry stays in
1508
- // the journal and the next run retries.
1393
+ function executeJournalTombstoneDeletes(
1394
+ run: PullRunContext,
1395
+ plan: PullPlan,
1396
+ counters: PullCounters,
1397
+ ): void {
1509
1398
  for (const key of plan.tombstones) {
1510
- // Last line of defense: a malformed or traversal key must NEVER reach
1511
- // fs.unlinkSync or journal mutation for a path outside the sync root.
1512
- const localPath = resolveContainedVaultPath(companyRoot, key);
1399
+ const localPath = resolveContainedVaultPath(run.companyRoot, key);
1513
1400
  if (localPath === null) continue;
1514
1401
  let removedSomething = false;
1515
1402
  try {
1516
1403
  const lstat = fs.lstatSync(localPath);
1517
- if (tombstoneTargetDiverged(journal, key, localPath, lstat)) {
1404
+ if (tombstoneTargetDiverged(run.journal, key, localPath, lstat)) {
1518
1405
  continue;
1519
1406
  }
1520
1407
  if (lstat.isSymbolicLink() || lstat.isFile()) {
1521
1408
  fs.unlinkSync(localPath);
1522
1409
  removedSomething = true;
1523
1410
  } else if (lstat.isDirectory()) {
1524
- // A dir at a key likely from a (local-dir, cloud-file) historic
1525
- // state. Don't recursively rm-rf the operator's dir; just drop
1526
- // the journal entry so we converge with reality.
1411
+ // A dir at a key is converged by dropping only the journal entry.
1527
1412
  }
1528
1413
  } catch (err: unknown) {
1529
1414
  const code =
1530
1415
  err && typeof err === "object" && "code" in err
1531
1416
  ? (err as { code?: string }).code
1532
1417
  : undefined;
1533
- // ENOENT → local already gone; safe to drop the journal entry.
1534
- // Other errors (EACCES/EPERM/EBUSY/etc.) leave the local file in
1535
- // place — if we dropped the journal entry anyway, the pull side
1536
- // would forget the peer's delete and a later push could re-upload
1537
- // the still-present local file, silently undoing the peer's delete.
1538
- // Surface the error and KEEP the journal entry so the next sync
1539
- // retries the unlink after the operator fixes the permission.
1540
1418
  if (code !== "ENOENT") {
1541
- emit({
1419
+ run.emit({
1542
1420
  type: "error",
1543
1421
  path: key,
1544
1422
  message: `tombstone unlink failed: ${
1545
1423
  err instanceof Error ? err.message : String(err)
1546
1424
  }`,
1547
1425
  });
1548
- // Skip removeEntry / filesTombstoned / progress event — the
1549
- // tombstone hasn't actually been honored. Next sync retries.
1550
1426
  continue;
1551
1427
  }
1552
1428
  }
1553
- removeEntry(journal, key);
1554
- filesTombstoned++;
1555
- emit({
1429
+ removeEntry(run.journal, key);
1430
+ counters.filesTombstoned++;
1431
+ run.emit({
1556
1432
  type: "progress",
1557
1433
  path: key,
1558
1434
  bytes: 0,
1559
1435
  deleted: true,
1560
- // Suffix differentiates a tombstone from a normal delete in the
1561
- // tty stream — matches the push-side `defaultConsoleLogger`
1562
- // tombstone surface in share.ts.
1563
1436
  message: removedSomething ? "tombstone (cross-machine delete)" : "tombstone (already absent locally)",
1564
1437
  });
1565
1438
  }
1439
+ }
1566
1440
 
1567
- // Record this pull's boundary (US-005) so the NEXT pull can diff its scope
1568
- // against ours and detect a shrink. Append before the journal write so it
1569
- // persists. `prefixSet` is stored in the same company-relative namespace as
1570
- // the journal keys; `all` mode records `[""]` (covers everything).
1571
- appendPullRecord(journal, {
1441
+ function finalizePullRun(
1442
+ run: PullRunContext,
1443
+ plan: PullPlan,
1444
+ scopeRun: ScopeShrinkRun,
1445
+ counters: PullCounters,
1446
+ ): SyncResult {
1447
+ appendPullRecord(run.journal, {
1572
1448
  pullId: generatePullId(),
1573
- companyUid: ctx.uid,
1574
- startedAt,
1449
+ companyUid: run.ctx.uid,
1450
+ startedAt: run.startedAt,
1575
1451
  completedAt: new Date().toISOString(),
1576
- syncMode,
1577
- prefixSet: currentPrefixSet,
1578
- scopeChangeDetected: shrinkPlan.scopeChangeDetected,
1579
- orphansRemoved: scopeOrphansRemoved,
1580
- orphansBlocked: shrinkResult.dirtyTombstoned,
1452
+ syncMode: run.syncMode,
1453
+ prefixSet: run.currentPrefixSet,
1454
+ scopeChangeDetected: scopeRun.shrinkPlan.scopeChangeDetected,
1455
+ orphansRemoved: scopeRun.scopeOrphansRemoved,
1456
+ orphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
1581
1457
  });
1582
1458
 
1583
- // Stamp lastSync on every successful run so the menubar's "Last sync · X ago"
1584
- // ticks even when nothing transferred. updateEntry only fires on actual
1585
- // downloads; without this, a no-op sync leaves lastSync at the time of the
1586
- // last file change, which is misleading.
1587
- journal.lastSync = new Date().toISOString();
1588
- writeJournal(journalSlug, journal);
1589
-
1590
- // When the pull actually changed on-disk sources (new files, tombstoned
1591
- // removals, or scope-orphan cleanups), refresh the generated skill wrappers,
1592
- // personal-overlay mirrors, and workers registry. reindex is idempotent and
1593
- // best-effort — it must never fail a sync, and is skipped on no-op syncs
1594
- // (the common daemon case) and when the caller opts out via skipReindex.
1459
+ run.journal.lastSync = new Date().toISOString();
1460
+ writeJournal(run.journalSlug, run.journal);
1461
+
1595
1462
  const changedOnDisk =
1596
- filesDownloaded > 0 ||
1597
- filesTombstoned > 0 ||
1598
- scopeOrphansRemoved > 0;
1599
- if (!options.skipReindex && changedOnDisk) {
1463
+ counters.filesDownloaded > 0 ||
1464
+ counters.filesTombstoned > 0 ||
1465
+ scopeRun.scopeOrphansRemoved > 0;
1466
+ if (!run.options.skipReindex && changedOnDisk) {
1600
1467
  try {
1601
- // skipLock: the surrounding sync run already holds this root's operation
1602
- // lock; reindex re-acquiring would refuse against our own live PID.
1603
- reindex({ repoRoot: hqRoot, skipLock: true });
1468
+ reindex({ repoRoot: run.hqRoot, skipLock: true });
1604
1469
  } catch {
1605
1470
  // best-effort: a post-sync refresh failure never fails the sync
1606
1471
  }
1607
1472
  }
1608
1473
 
1609
1474
  return {
1610
- filesDownloaded,
1611
- bytesDownloaded,
1612
- filesSkipped,
1613
- conflicts,
1614
- conflictPaths,
1475
+ filesDownloaded: counters.filesDownloaded,
1476
+ bytesDownloaded: counters.bytesDownloaded,
1477
+ filesSkipped: counters.filesSkipped,
1478
+ conflicts: counters.conflicts,
1479
+ conflictPaths: counters.conflictPaths,
1615
1480
  aborted: false,
1616
1481
  newFiles: plan.newFiles,
1617
1482
  newFilesCount: plan.newFilesCount,
1618
1483
  filesExcludedByPolicy: plan.filesExcludedByPolicy,
1619
- filesTombstoned,
1620
- filesOutOfScope,
1621
- scopeOrphansRemoved,
1622
- scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
1484
+ filesTombstoned: counters.filesTombstoned,
1485
+ filesOutOfScope: counters.filesOutOfScope,
1486
+ scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
1487
+ scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
1623
1488
  };
1624
1489
  }
1625
1490
 
1626
- /**
1627
- * Resolve active company from .hq/config.json.
1628
- */
1629
- function resolveActiveCompany(hqRoot: string): string | undefined {
1630
- const configPath = path.join(hqRoot, ".hq", "config.json");
1631
- if (fs.existsSync(configPath)) {
1632
- try {
1633
- const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
1634
- return config.activeCompany ?? config.companySlug;
1635
- } catch {
1636
- // Ignore parse errors
1637
- }
1638
- }
1639
- return undefined;
1640
- }
1641
-
1642
- /**
1643
- * Returns true when the remote object appears to have moved since the
1644
- * journal entry's last-recorded sync. Prefers ETag equality; falls back to
1645
- * `lastModified > syncedAt` for legacy entries written before remoteEtag
1646
- * was tracked. Conservative on tie (`<=` skews "remote unchanged").
1647
- */
1648
- function hasRemoteChanged(
1649
- remote: { lastModified: Date; etag: string },
1650
- entry: { syncedAt: string; remoteEtag?: string },
1651
- ): boolean {
1652
- if (entry.remoteEtag) {
1653
- return normalizeEtag(remote.etag) !== entry.remoteEtag;
1654
- }
1655
- const syncedAt = new Date(entry.syncedAt).getTime();
1656
- return remote.lastModified.getTime() > syncedAt;
1657
- }
1658
-
1659
1491
  /**
1660
1492
  * Decide whether a remote object present in the LIST is a GENUINE RE-CREATE
1661
1493
  * written AFTER a FILE_TOMBSTONE — in which case the tombstone is stale and the
@@ -2355,16 +2187,6 @@ function computePullPlan(
2355
2187
  };
2356
2188
  }
2357
2189
 
2358
- /**
2359
- * Check if an error is an S3 access denied (expected for filtered guests).
2360
- */
2361
- function isAccessDenied(err: unknown): boolean {
2362
- if (err && typeof err === "object" && "name" in err) {
2363
- return err.name === "AccessDenied" || err.name === "Forbidden";
2364
- }
2365
- return false;
2366
- }
2367
-
2368
2190
  /**
2369
2191
  * Default human-readable event rendering. Preserves the exact output format
2370
2192
  * that `hq sync` emitted before SyncProgressEvent was introduced, so callers