@indigoai-us/hq-cloud 6.11.12 → 6.11.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/bin/sync-runner-company.d.ts +35 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -0
  3. package/dist/bin/sync-runner-company.js +290 -0
  4. package/dist/bin/sync-runner-company.js.map +1 -0
  5. package/dist/bin/sync-runner-events.d.ts +12 -0
  6. package/dist/bin/sync-runner-events.d.ts.map +1 -0
  7. package/dist/bin/sync-runner-events.js +12 -0
  8. package/dist/bin/sync-runner-events.js.map +1 -0
  9. package/dist/bin/sync-runner-planning.d.ts +53 -0
  10. package/dist/bin/sync-runner-planning.d.ts.map +1 -0
  11. package/dist/bin/sync-runner-planning.js +59 -0
  12. package/dist/bin/sync-runner-planning.js.map +1 -0
  13. package/dist/bin/sync-runner-rollup.d.ts +24 -0
  14. package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
  15. package/dist/bin/sync-runner-rollup.js +46 -0
  16. package/dist/bin/sync-runner-rollup.js.map +1 -0
  17. package/dist/bin/sync-runner-telemetry.d.ts +5 -0
  18. package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
  19. package/dist/bin/sync-runner-telemetry.js +5 -0
  20. package/dist/bin/sync-runner-telemetry.js.map +1 -0
  21. package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
  22. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
  23. package/dist/bin/sync-runner-watch-loop.js +372 -0
  24. package/dist/bin/sync-runner-watch-loop.js.map +1 -0
  25. package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
  26. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
  27. package/dist/bin/sync-runner-watch-routes.js +74 -0
  28. package/dist/bin/sync-runner-watch-routes.js.map +1 -0
  29. package/dist/bin/sync-runner.d.ts +3 -54
  30. package/dist/bin/sync-runner.d.ts.map +1 -1
  31. package/dist/bin/sync-runner.js +73 -1154
  32. package/dist/bin/sync-runner.js.map +1 -1
  33. package/dist/cli/reindex.d.ts.map +1 -1
  34. package/dist/cli/reindex.js +34 -17
  35. package/dist/cli/reindex.js.map +1 -1
  36. package/dist/cli/reindex.test.js +39 -5
  37. package/dist/cli/reindex.test.js.map +1 -1
  38. package/dist/cli/rescue-classify-ordering.test.js +17 -0
  39. package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
  40. package/dist/cli/rescue-core.d.ts +45 -0
  41. package/dist/cli/rescue-core.d.ts.map +1 -1
  42. package/dist/cli/rescue-core.js +197 -170
  43. package/dist/cli/rescue-core.js.map +1 -1
  44. package/dist/cli/share.d.ts.map +1 -1
  45. package/dist/cli/share.js +224 -676
  46. package/dist/cli/share.js.map +1 -1
  47. package/dist/cli/sync.d.ts.map +1 -1
  48. package/dist/cli/sync.js +399 -726
  49. package/dist/cli/sync.js.map +1 -1
  50. package/dist/cli/sync.test.js +20 -0
  51. package/dist/cli/sync.test.js.map +1 -1
  52. package/dist/daemon-worker.d.ts +2 -2
  53. package/dist/daemon-worker.js +3 -3
  54. package/dist/daemon-worker.js.map +1 -1
  55. package/dist/object-io.js +1 -1
  56. package/dist/object-io.js.map +1 -1
  57. package/dist/remote-pull.d.ts +2 -2
  58. package/dist/remote-pull.d.ts.map +1 -1
  59. package/dist/remote-pull.js +23 -3
  60. package/dist/remote-pull.js.map +1 -1
  61. package/dist/remote-pull.test.js +24 -2
  62. package/dist/remote-pull.test.js.map +1 -1
  63. package/dist/sync/push-receiver.d.ts +6 -0
  64. package/dist/sync/push-receiver.d.ts.map +1 -1
  65. package/dist/sync/push-receiver.js +32 -2
  66. package/dist/sync/push-receiver.js.map +1 -1
  67. package/dist/sync/push-receiver.test.js +31 -0
  68. package/dist/sync/push-receiver.test.js.map +1 -1
  69. package/dist/sync-core.d.ts +27 -0
  70. package/dist/sync-core.d.ts.map +1 -0
  71. package/dist/sync-core.js +54 -0
  72. package/dist/sync-core.js.map +1 -0
  73. package/dist/vault-client.d.ts.map +1 -1
  74. package/dist/vault-client.js +284 -36
  75. package/dist/vault-client.js.map +1 -1
  76. package/dist/vault-client.test.js +59 -0
  77. package/dist/vault-client.test.js.map +1 -1
  78. package/dist/watcher.d.ts +2 -20
  79. package/dist/watcher.d.ts.map +1 -1
  80. package/dist/watcher.js +3 -113
  81. package/dist/watcher.js.map +1 -1
  82. package/package.json +1 -1
  83. package/src/bin/sync-runner-company.ts +350 -0
  84. package/src/bin/sync-runner-events.ts +25 -0
  85. package/src/bin/sync-runner-planning.ts +121 -0
  86. package/src/bin/sync-runner-rollup.ts +72 -0
  87. package/src/bin/sync-runner-telemetry.ts +8 -0
  88. package/src/bin/sync-runner-watch-loop.ts +443 -0
  89. package/src/bin/sync-runner-watch-routes.ts +86 -0
  90. package/src/bin/sync-runner.ts +96 -1253
  91. package/src/cli/reindex.test.ts +41 -3
  92. package/src/cli/reindex.ts +35 -19
  93. package/src/cli/rescue-classify-ordering.test.ts +20 -0
  94. package/src/cli/rescue-core.ts +252 -176
  95. package/src/cli/share.ts +363 -705
  96. package/src/cli/sync.test.ts +25 -0
  97. package/src/cli/sync.ts +612 -802
  98. package/src/daemon-worker.ts +3 -3
  99. package/src/object-io.ts +1 -1
  100. package/src/remote-pull.test.ts +30 -1
  101. package/src/remote-pull.ts +29 -4
  102. package/src/sync/push-receiver.test.ts +35 -0
  103. package/src/sync/push-receiver.ts +41 -2
  104. package/src/sync-core.ts +58 -0
  105. package/src/vault-client.test.ts +74 -0
  106. package/src/vault-client.ts +395 -43
  107. package/src/watcher.ts +6 -141
package/src/cli/sync.ts CHANGED
@@ -7,7 +7,11 @@
7
7
 
8
8
  import * as fs from "fs";
9
9
  import * as path from "path";
10
- import type { VaultServiceConfig, SyncJournal } from "../types.js";
10
+ import type {
11
+ EntityContext,
12
+ VaultServiceConfig,
13
+ SyncJournal,
14
+ } from "../types.js";
11
15
  import type { SyncMode } from "../vault-client.js";
12
16
  import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
13
17
  import {
@@ -49,6 +53,12 @@ import {
49
53
  type ScopePrefixInput,
50
54
  } from "../prefix-coalesce.js";
51
55
  import { createIgnoreFilter } from "../ignore.js";
56
+ import {
57
+ hasRemoteChanged,
58
+ isAccessDenied,
59
+ resolveActiveCompany,
60
+ resolveTransferConcurrency,
61
+ } from "../sync-core.js";
52
62
  import { isEphemeralPath, isMalformedVaultKey } from "./share.js";
53
63
  import { resolveConflict } from "./conflict.js";
54
64
  import type { ConflictStrategy, ConflictResolution } from "./conflict.js";
@@ -489,6 +499,52 @@ export interface SyncResult {
489
499
  scopeOrphansBlocked: number;
490
500
  }
491
501
 
502
+ type SyncEventEmitter = (event: SyncProgressEvent) => void;
503
+
504
+ type PullDownloadItem = Extract<PullPlanItem, { action: "download" }>;
505
+
506
+ interface PullRunContext {
507
+ options: SyncOptions;
508
+ companyRef: string;
509
+ vaultConfig: VaultServiceConfig;
510
+ hqRoot: string;
511
+ emit: SyncEventEmitter;
512
+ ctx: EntityContext;
513
+ companyRoot: string;
514
+ shouldSync: (filePath: string, isDir?: boolean) => boolean;
515
+ journalSlug: string;
516
+ startedAt: string;
517
+ journal: SyncJournal;
518
+ remoteFiles: RemoteFile[];
519
+ syncMode: SyncMode;
520
+ currentPrefixSet: string[];
521
+ fileTombstones: ReadonlyMap<string, CompanyTombstone>;
522
+ }
523
+
524
+ interface PullCounters {
525
+ filesDownloaded: number;
526
+ bytesDownloaded: number;
527
+ filesSkipped: number;
528
+ conflicts: number;
529
+ filesTombstoned: number;
530
+ filesOutOfScope: number;
531
+ conflictPaths: string[];
532
+ }
533
+
534
+ interface ScopeShrinkPlanState {
535
+ lastRecord: ReturnType<typeof lastPullRecord>;
536
+ shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
537
+ autoRecover: boolean;
538
+ adviceContext: ScopeShrinkAdviceContext;
539
+ effectiveForce: boolean;
540
+ }
541
+
542
+ interface ScopeShrinkRun {
543
+ shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
544
+ shrinkResult: ReturnType<typeof applyScopeShrink>;
545
+ scopeOrphansRemoved: number;
546
+ }
547
+
492
548
  /**
493
549
  * Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
494
550
  * scope shrink that would delete more than this many CLEAN local files in one
@@ -616,10 +672,44 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
616
672
  async function syncWithOperationLockHeld(
617
673
  options: SyncOptions,
618
674
  ): Promise<SyncResult> {
619
- const { company, onConflict, vaultConfig, hqRoot } = options;
675
+ const run = await buildPullContext(options);
676
+ const plan = planPull(run);
677
+
678
+ emitPullPlan(run.emit, plan);
679
+
680
+ const scopePlan = planScopeShrink(run);
681
+ const scopeRun = executeScopeShrink(run, scopePlan);
682
+ const counters = createPullCounters();
683
+
684
+ const transferConcurrency = resolveTransferConcurrency();
685
+ const conflictRun = await executeConflictExecutor(
686
+ run,
687
+ plan,
688
+ scopeRun,
689
+ counters,
690
+ );
691
+ if (conflictRun.abortResult) {
692
+ return conflictRun.abortResult;
693
+ }
694
+
695
+ await executeDownloadExecutor(
696
+ run,
697
+ conflictRun.downloadItems,
698
+ transferConcurrency,
699
+ counters,
700
+ );
701
+
702
+ await emitAndReportNewFiles(run, plan);
703
+ await verifyPlannedJournalTombstones(run, plan);
704
+ executeJournalTombstoneDeletes(run, plan, counters);
705
+
706
+ return finalizePullRun(run, plan, scopeRun, counters);
707
+ }
708
+
709
+ async function buildPullContext(options: SyncOptions): Promise<PullRunContext> {
710
+ const { company, vaultConfig, hqRoot } = options;
620
711
  const emit = options.onEvent ?? defaultConsoleLogger;
621
712
 
622
- // Resolve company
623
713
  const companyRef = company ?? resolveActiveCompany(hqRoot);
624
714
  if (!companyRef) {
625
715
  throw new Error(
@@ -628,184 +718,131 @@ async function syncWithOperationLockHeld(
628
718
  );
629
719
  }
630
720
 
631
- // Resolve entity context
632
- let ctx = await resolveEntityContext(companyRef, vaultConfig);
633
- // Every company's files land under companies/{slug}/ so fanning out multiple
634
- // companies into the same hqRoot doesn't cross-clobber files with overlapping
635
- // S3 keys (e.g. every company has a .hq/manifest.json). Remote keys stay
636
- // company-relative; the prefix lives only on disk.
637
- // In personalMode the journal slug + S3 keys are person-relative (e.g. "docs/foo.md");
638
- // the local target is `hqRoot` directly, NOT `<hqRoot>/companies/<personSlug>/`. This
639
- // keeps round-trip parity with the Rust personal first-push (Step 7) which sources
640
- // `<hqRoot>/docs/foo.md`.
721
+ const ctx = await resolveEntityContext(companyRef, vaultConfig);
641
722
  const companyRoot = options.personalMode === true
642
723
  ? hqRoot
643
724
  : path.join(hqRoot, "companies", ctx.slug);
644
725
  const shouldSync = createIgnoreFilter(hqRoot);
645
726
  const journalSlug = options.journalSlug ?? ctx.slug;
646
727
  const startedAt = new Date().toISOString();
647
- // Personal-vault callers must never start from an empty journal when only
648
- // the legacy `personal` file exists (mass re-download/etag churn). Seeding
649
- // here — inside the engine — covers every consumer (sync-runner already
650
- // seeds; hq-cli historically didn't, which split the vault's bookkeeping
651
- // across two journal files and re-flagged synced files as conflicts).
728
+
652
729
  if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG) migratePersonalVaultJournal();
653
- // Migrate v1 → v2 in place so the scope-shrink / pull-record machinery has
654
- // its fields, and GC any tombstones past the 30-day retention window before
655
- // we re-evaluate orphans (so a long-pruned path can re-download cleanly).
656
730
  const journal = migrateToV2(readJournal(journalSlug));
657
731
  gcTombstones(journal, Date.now());
658
732
 
659
- // ── Effective download scope (US-005) ─────────────────────────────────────
660
- // `all` → prefixSet `[""]`, which `isCoveredByAny` treats as "covers
661
- // everything" — so the download filter and the scope-shrink
662
- // comparison both become no-ops, preserving legacy full-bucket
663
- // behavior bit-for-bit.
664
- // `shared`/`custom` → the coalesced, company-relative prefix set the runner
665
- // resolved. An empty set means "nothing in scope" → download
666
- // nothing (the runner falls back to `all` on resolution errors, so
667
- // empty here is an intentional "nothing shared", never a failure).
668
733
  const syncMode: SyncMode = options.syncMode ?? "all";
669
- const currentPrefixSet =
734
+ const currentPrefixSet: string[] =
670
735
  syncMode === "all" ? [""] : coalescePrefixes(options.prefixSet ?? []);
671
- // Authorship guard input (scope-shrink): the caller's own Cognito sub,
672
- // injected by the entry point (the runner sources it from its decoded
673
- // idToken claims — the same sub stamped onto uploads as `created-by-sub`).
674
- // Undefined degrades safely: own-author files lose their special shield, but
675
- // the `protectUnknownAuthors` conservative path below still prevents a
676
- // routine sync from deleting anything it can't prove is foreign.
677
- const callerSub = options.callerSub;
678
-
679
- let filesDownloaded = 0;
680
- let bytesDownloaded = 0;
681
- let filesSkipped = 0;
682
- let conflicts = 0;
683
- let filesTombstoned = 0;
684
- let filesOutOfScope = 0;
685
- const conflictPaths: string[] = [];
686
736
 
687
- // List all remote files (IAM session policy filters at the AWS layer)
688
737
  const remoteFiles = await listRemoteFiles(ctx);
689
-
690
- // Fetch the company's FILE_TOMBSTONE records so the planner can suppress
691
- // resurrection of an intentionally-deleted object (delete-resync). Done in
692
- // parallel intent with the LIST above conceptually, but kept serial here for
693
- // a clean read of `ctx`; best-effort — a failed read degrades to an empty map
694
- // (no suppression), preserving the pre-fix behavior. ctx.uid is the verified
695
- // companyUid the tombstone rows are keyed under.
696
- //
697
- // SKIP for the personal vault: its `ctx.uid` is a personUid (`prs_…`), but
698
- // `GET /v1/files/tombstones?company=…` is COMPANY-scoped server-side
699
- // (findCallerWithMembership), so a personal-vault request resolves
700
- // `company=prs_…` to no membership and is correctly rejected with
701
- // `403 "No active membership for caller in company prs_…"`. That 403 is
702
- // benign for the pull (it already degrades to the empty map below), but
703
- // hq-pro captures EVERY one as a Sentry warning — the per-personal-vault
704
- // no-membership cluster (one Sentry issue per signed-in user). Personal-vault
705
- // delete-resync was never a committed feature and there is no person-scoped
706
- // tombstone path, so for the personal target we skip the fetch and use an
707
- // empty map — byte-for-byte the current degraded behavior, minus the 403 spam.
708
- // FUTURE FOLLOW-UP (not built here): if personal-vault delete-resync is
709
- // wanted, it needs a real person-scoped tombstone endpoint + client read.
710
- const tombstones =
738
+ const fileTombstones =
711
739
  options.personalMode === true
712
740
  ? new Map<string, CompanyTombstone>()
713
741
  : await fetchCompanyTombstones(vaultConfig, ctx.uid);
714
742
 
715
- // Stage 1: classify every remote file against the journal + local disk.
716
- // Hashing happens here (not in the transfer loop) so the plan event below
717
- // carries an accurate denominator before any progress events fire.
718
- const plan = computePullPlan(
719
- remoteFiles,
720
- journal,
743
+ return {
744
+ options,
745
+ companyRef,
746
+ vaultConfig,
747
+ hqRoot,
748
+ emit,
749
+ ctx,
721
750
  companyRoot,
722
751
  shouldSync,
723
- options.personalMode === true,
724
- options.includeLocalCompanies === true,
725
- options.teamSyncedSlugs ?? null,
752
+ journalSlug,
753
+ startedAt,
754
+ journal,
755
+ remoteFiles,
756
+ syncMode,
726
757
  currentPrefixSet,
727
- tombstones,
758
+ fileTombstones,
759
+ };
760
+ }
761
+
762
+ function planPull(run: PullRunContext): PullPlan {
763
+ return computePullPlan(
764
+ run.remoteFiles,
765
+ run.journal,
766
+ run.companyRoot,
767
+ run.shouldSync,
768
+ run.options.personalMode === true,
769
+ run.options.includeLocalCompanies === true,
770
+ run.options.teamSyncedSlugs ?? null,
771
+ run.currentPrefixSet,
772
+ run.fileTombstones,
728
773
  );
774
+ }
729
775
 
776
+ function emitPullPlan(emit: SyncEventEmitter, plan: PullPlan): void {
730
777
  emit({
731
778
  type: "plan",
732
779
  filesToDownload: plan.filesToDownload,
733
780
  bytesToDownload: plan.bytesToDownload,
734
- // sync() is pull-only; push counts are sourced from share()'s plan event.
735
781
  filesToUpload: 0,
736
782
  bytesToUpload: 0,
737
783
  filesToSkip: plan.filesToSkip,
738
784
  filesToConflict: plan.filesToConflict,
739
- // Authoritative FILE_TOMBSTONE suppressions (delete-resync) are the only
740
- // deletes known at plan time; the journal-vs-LIST tombstones are
741
- // HEAD-verified later and surfaced via the final filesTombstoned count.
742
785
  filesToDelete: plan.filesToTombstoneDelete,
743
786
  });
787
+ }
744
788
 
745
- // ── Scope-shrink cleanup (US-005) ─────────────────────────────────────────
746
- // If the effective scope narrowed since the last pull, files that were
747
- // pulled under the old scope but fall outside the new one are orphans. We
748
- // delete only CLEAN orphans (provably unchanged since last sync); dirty
749
- // (locally-modified) orphans are sacred. By default a dirty orphan aborts
750
- // the leg with a structured error the CLI renders; `forceScopeShrink` keeps
751
- // dirty files on disk and only tombstones their journal entries.
752
- //
753
- // `companyRoot` is passed as the module's `hqRoot` so its `path.join(root,
754
- // key)` resolves company-relative journal keys correctly (the scope-shrink
755
- // module is namespace-agnostic — root + keys + prefixSet must simply agree).
756
- //
757
- // Note: this is the durable selective-download fix for OWNERS. An owner's
758
- // STS is wide (role-bypass), so the remote LIST returns everything and the
759
- // AWS layer never narrows the pull. This client-side shrink is what makes
760
- // `hq sync mode shared` actually stick across re-syncs for an owner.
761
- const lastRecord = lastPullRecord(journal, ctx.uid);
762
- // A missing record, or a v1-migrated record with an empty prefixSet, means
763
- // "no recorded scope" → treat the last scope as full-bucket `all` (`[""]`),
764
- // per the PullRecord.prefixSet contract in types.ts.
789
+ function createPullCounters(): PullCounters {
790
+ return {
791
+ filesDownloaded: 0,
792
+ bytesDownloaded: 0,
793
+ filesSkipped: 0,
794
+ conflicts: 0,
795
+ filesTombstoned: 0,
796
+ filesOutOfScope: 0,
797
+ conflictPaths: [],
798
+ };
799
+ }
800
+
801
+ function planScopeShrink(run: PullRunContext): ScopeShrinkPlanState {
802
+ const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
765
803
  const lastPrefixSet =
766
804
  lastRecord && lastRecord.prefixSet.length > 0
767
805
  ? lastRecord.prefixSet
768
806
  : [""];
769
807
  const shrinkPlan = buildScopeShrinkPlan({
770
- journal,
771
- hqRoot: companyRoot,
808
+ journal: run.journal,
809
+ hqRoot: run.companyRoot,
772
810
  lastPrefixSet,
773
- currentPrefixSet,
774
- callerSub,
775
- // Automatic pull: never auto-prune content the caller authored, and never
776
- // make a destructive guess about unknown-author (legacy) orphans. The
777
- // explicit `hq sync narrow` ritual opts out of the unknown-author shield.
811
+ currentPrefixSet: run.currentPrefixSet,
812
+ callerSub: run.options.callerSub,
778
813
  protectUnknownAuthors: true,
779
814
  });
780
- // Policy: the background menubar runner ("auto-recover") can take no
781
- // interactive flag, so it must never throw on a shrink — it self-heals
782
- // non-destructively (dirty kept on disk + un-tracked, clean quarantined).
783
- // A foreground `hq sync` ("block", the default) keeps the protective gate
784
- // but renders FOLLOWABLE advice. `autoRecover` implies force (proceed) and
785
- // bypasses the bulk-prune cap (quarantine is non-destructive, so a large
786
- // recovery move is safe). DEV-1768.
787
- const scopeShrinkPolicy = options.scopeShrinkPolicy ?? "block";
815
+ const scopeShrinkPolicy = run.options.scopeShrinkPolicy ?? "block";
788
816
  const autoRecover = scopeShrinkPolicy === "auto-recover";
789
817
  const adviceContext: ScopeShrinkAdviceContext = autoRecover ? "runner" : "cli";
790
- const effectiveForce = options.forceScopeShrink === true || autoRecover;
818
+ const effectiveForce = run.options.forceScopeShrink === true || autoRecover;
819
+
820
+ return {
821
+ lastRecord,
822
+ shrinkPlan,
823
+ autoRecover,
824
+ adviceContext,
825
+ effectiveForce,
826
+ };
827
+ }
828
+
829
+ function executeScopeShrink(
830
+ run: PullRunContext,
831
+ scopePlan: ScopeShrinkPlanState,
832
+ ): ScopeShrinkRun {
833
+ const { lastRecord, shrinkPlan, adviceContext, effectiveForce } = scopePlan;
791
834
 
792
835
  if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
793
836
  throw new ScopeShrinkBlockedError(
794
- ctx.uid,
837
+ run.ctx.uid,
795
838
  lastRecord?.syncMode ?? "unknown",
796
- syncMode,
839
+ run.syncMode,
797
840
  shrinkPlan.dirty,
798
841
  shrinkPlan.clean,
799
842
  adviceContext,
800
843
  );
801
844
  }
802
- // Bulk guard: refuse to auto-move more than the safety cap of CLEAN files in
803
- // a single foreground sync. A deliberate large narrow goes through
804
- // `hq sync narrow --apply` (its own confirmation); `--force-scope-shrink` (or
805
- // raising HQ_SYNC_MAX_AUTO_PRUNE) overrides. Cap of 0 = unlimited. Skipped
806
- // under auto-recover — quarantine is non-destructive so a big recovery is
807
- // safe, and the runner has no way to act on a thrown cap. The engine moves
808
- // nothing when it throws here.
845
+
809
846
  const autoPruneCap = resolveAutoPruneCap();
810
847
  if (
811
848
  !effectiveForce &&
@@ -813,40 +850,32 @@ async function syncWithOperationLockHeld(
813
850
  shrinkPlan.clean.length > autoPruneCap
814
851
  ) {
815
852
  throw new ScopeShrinkLargePruneError(
816
- ctx.uid,
817
- syncMode,
853
+ run.ctx.uid,
854
+ run.syncMode,
818
855
  shrinkPlan.clean.length,
819
856
  autoPruneCap,
820
857
  adviceContext,
821
858
  );
822
859
  }
823
- // Clean orphans are QUARANTINED (moved into `.hq/scope-quarantine/<slug>/`,
824
- // recoverable), never silently deleted — a background sync purging local
825
- // files unannounced was DEV-1768 fix #3. The quarantine root lives under the
826
- // real HQ root's `.hq/` (outside `companyRoot` and never pushed), so moved
827
- // files don't round-trip back through S3.
860
+
828
861
  const scopeQuarantineRoot = path.join(
829
- hqRoot,
862
+ run.hqRoot,
830
863
  ".hq",
831
864
  "scope-quarantine",
832
- journalSlug,
865
+ run.journalSlug,
833
866
  );
834
867
  const shrinkResult = applyScopeShrink({
835
- journal,
868
+ journal: run.journal,
836
869
  plan: shrinkPlan,
837
- hqRoot: companyRoot,
870
+ hqRoot: run.companyRoot,
838
871
  forceScopeShrink: effectiveForce,
839
872
  reason: "scope_shrink",
840
873
  cleanDisposition: "quarantine",
841
874
  quarantineRoot: scopeQuarantineRoot,
842
875
  });
843
- // Surface each affected orphan explicitly (named path) so the prune is never
844
- // silent. Quarantined clean files render as `deleted: true` (removed from the
845
- // working tree, recoverable in quarantine); dirty files KEPT on disk render
846
- // as a non-deletion notice so the operator knows they were un-tracked, not
847
- // removed. The Rust menubar parser already handles `deleted: true`.
876
+
848
877
  for (const relPath of shrinkResult.quarantinedPaths) {
849
- emit({
878
+ run.emit({
850
879
  type: "progress",
851
880
  path: relPath,
852
881
  bytes: 0,
@@ -855,7 +884,7 @@ async function syncWithOperationLockHeld(
855
884
  });
856
885
  }
857
886
  for (const relPath of shrinkResult.removedPaths) {
858
- emit({
887
+ run.emit({
859
888
  type: "progress",
860
889
  path: relPath,
861
890
  bytes: 0,
@@ -864,7 +893,7 @@ async function syncWithOperationLockHeld(
864
893
  });
865
894
  }
866
895
  for (const relPath of shrinkResult.dirtyKeptPaths) {
867
- emit({
896
+ run.emit({
868
897
  type: "progress",
869
898
  path: relPath,
870
899
  bytes: 0,
@@ -872,547 +901,410 @@ async function syncWithOperationLockHeld(
872
901
  "scope-narrowed: locally-modified file KEPT on disk, un-tracked from sync (outside scope)",
873
902
  });
874
903
  }
875
- // "Removed from the working tree" = deleted OR quarantined; both vacate the
876
- // file's original path. Reported as `scopeOrphansRemoved` for back-compat.
877
- const scopeOrphansRemoved =
878
- shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined;
879
-
880
- // Stage 2: execute the plan. Per-item branching mirrors the pre-refactor
881
- // inline loop; the only structural change is that classification has
882
- // already happened (so `localHash` is reused instead of re-hashing).
883
- //
884
- // 5.36.0: download items go through a bounded-concurrent pool
885
- // (`TRANSFER_CONCURRENCY`, default 16, tunable via
886
- // `HQ_SYNC_TRANSFER_CONCURRENCY`) for 4-8x speedup on transfer-heavy
887
- // syncs. Conflict items stay serial — they may prompt the operator
888
- // and the abort path must short-circuit before any further work. We
889
- // partition the plan items into "conflict (serial)" and "download
890
- // (parallel)" buckets and run the serial pass first; the parallel pass
891
- // only runs if no conflict aborted.
892
- //
893
- // Per-file `progress` events fire at the moment each individual download
894
- // settles (inside the pool wrapper), NOT in plan-walk order. The cross-
895
- // file interleave is acceptable: the menubar stream parser already
896
- // handles per-company interleave, and the same shape applies within a
897
- // single company's pool. Per-file event-count correctness is preserved
898
- // (one progress per download, one error per failure).
899
- const TRANSFER_CONCURRENCY = (() => {
900
- const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
901
- if (raw === undefined || raw === "") return 16;
902
- const parsed = Number.parseInt(raw, 10);
903
- return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
904
- })();
905
-
906
- // First pass: serial walk for non-download outcomes (skips + conflicts).
907
- // Conflicts may set `aborted = true` and short-circuit the whole pull;
908
- // we detect that and skip the parallel pass. Download items are
909
- // collected into `downloadItems[]` for the pool pass below.
910
- const downloadItems: Array<typeof plan.items[number] & { action: "download" }> = [];
911
- let aborted = false;
912
- let abortResult: SyncResult | null = null;
904
+
905
+ return {
906
+ shrinkPlan,
907
+ shrinkResult,
908
+ scopeOrphansRemoved:
909
+ shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined,
910
+ };
911
+ }
912
+
913
+ async function refreshRunContextIfExpiring(
914
+ run: PullRunContext,
915
+ ): Promise<void> {
916
+ if (isExpiringSoon(run.ctx.expiresAt)) {
917
+ run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
918
+ }
919
+ }
920
+
921
+ async function executeConflictExecutor(
922
+ run: PullRunContext,
923
+ plan: PullPlan,
924
+ scopeRun: ScopeShrinkRun,
925
+ counters: PullCounters,
926
+ ): Promise<{ downloadItems: PullDownloadItem[]; abortResult: SyncResult | null }> {
927
+ const downloadItems: PullDownloadItem[] = [];
913
928
 
914
929
  for (const item of plan.items) {
915
- if (aborted) break;
916
930
  if (
917
931
  item.action === "skip-ignored" ||
918
932
  item.action === "skip-personal-mode" ||
919
933
  item.action === "skip-unchanged" ||
920
934
  item.action === "skip-local-only"
921
935
  ) {
922
- filesSkipped++;
936
+ counters.filesSkipped++;
923
937
  continue;
924
938
  }
925
939
  if (item.action === "skip-excluded-policy") {
926
- // Policy-excluded items count separately from `filesSkipped` so the
927
- // pull result mirrors the push side's `filesExcludedByPolicy`
928
- // counter — `filesSkipped` stays a measure of "unchanged on this
929
- // run", not a catch-all for everything we didn't download.
930
940
  continue;
931
941
  }
932
942
  if (item.action === "skip-out-of-scope") {
933
- // Outside the effective `syncMode` scope (US-005). Counted on its own
934
- // axis so `filesSkipped` keeps meaning "unchanged on this run" — these
935
- // are "deliberately not downloaded because of your sync scope".
936
- filesOutOfScope++;
943
+ counters.filesOutOfScope++;
937
944
  continue;
938
945
  }
939
-
940
946
  if (item.action === "tombstone-delete") {
941
- // Authoritative FILE_TOMBSTONE delete (delete-resync): the remote object
942
- // is present but a tombstone marks the key intentionally deleted and it is
943
- // not a newer re-create. Delete any local copy and drop the journal entry
944
- // so it stays gone — the mirror of the journal-vs-LIST tombstone executor
945
- // below, but WITHOUT the HEAD-verify (the remote object is present by
946
- // definition; the FILE_TOMBSTONE is the deletion authority). The planner
947
- // already routed any divergent local copy to `conflict`, so a local file
948
- // reaching here matches the deleted baseline and is safe to remove.
949
- const tombstoneKey = item.remoteFile.key;
950
- // Same Windows-backslash landmine guard as the journal-tombstone executor:
951
- // a malformed key must never reach fs.unlinkSync (path.join collapses the
952
- // backslashes onto a REAL POSIX file). Traversal keys are likewise
953
- // refused before any local filesystem or journal mutation.
954
- const tombstonePath = resolveContainedVaultPath(companyRoot, tombstoneKey);
955
- if (tombstonePath === null) continue;
956
- try {
957
- const lstat = fs.lstatSync(tombstonePath);
958
- if (tombstoneTargetDiverged(journal, tombstoneKey, tombstonePath, lstat)) {
959
- continue;
960
- }
961
- if (lstat.isSymbolicLink() || lstat.isFile()) {
962
- fs.unlinkSync(tombstonePath);
963
- }
964
- // A directory at the key: don't recursively rm-rf the operator's dir;
965
- // just drop the journal entry (safe-by-default, same as the other path).
966
- } catch (err: unknown) {
967
- const code =
968
- err && typeof err === "object" && "code" in err
969
- ? (err as { code?: string }).code
970
- : undefined;
971
- // ENOENT → local already absent (the common case: a fresh machine that
972
- // never held the file, or a prior pull already removed it) → drop the
973
- // journal entry and converge. Other errors (EACCES/EPERM/…) leave the
974
- // file in place; surface and KEEP the journal entry so the next sync
975
- // retries rather than forgetting the delete.
976
- if (code !== "ENOENT") {
977
- emit({
978
- type: "error",
979
- path: tombstoneKey,
980
- message: `tombstone-suppress unlink failed: ${
981
- err instanceof Error ? err.message : String(err)
982
- }`,
983
- });
984
- continue;
985
- }
986
- }
987
- removeEntry(journal, tombstoneKey);
988
- filesTombstoned++;
989
- emit({ type: "progress", path: tombstoneKey, bytes: 0 });
947
+ executeFileTombstoneDelete(run, item, counters);
990
948
  continue;
991
949
  }
992
-
993
950
  if (item.action === "download") {
994
951
  downloadItems.push(item);
995
952
  continue;
996
953
  }
997
954
 
998
- const { remoteFile, localPath } = item;
955
+ const abortResult = await executeConflictItem(
956
+ run,
957
+ plan,
958
+ scopeRun,
959
+ counters,
960
+ downloadItems,
961
+ item,
962
+ );
963
+ if (abortResult) {
964
+ return { downloadItems, abortResult };
965
+ }
966
+ }
967
+
968
+ return { downloadItems, abortResult: null };
969
+ }
999
970
 
1000
- // Auto-refresh context if credentials expiring (kept in execute phase
1001
- // because Stage 1 is fast — no need to refresh just to classify).
1002
- if (isExpiringSoon(ctx.expiresAt)) {
1003
- ctx = await refreshEntityContext(companyRef, vaultConfig);
971
+ function executeFileTombstoneDelete(
972
+ run: PullRunContext,
973
+ item: Extract<PullPlanItem, { action: "tombstone-delete" }>,
974
+ counters: PullCounters,
975
+ ): void {
976
+ const tombstoneKey = item.remoteFile.key;
977
+ const tombstonePath = resolveContainedVaultPath(run.companyRoot, tombstoneKey);
978
+ if (tombstonePath === null) return;
979
+ try {
980
+ const lstat = fs.lstatSync(tombstonePath);
981
+ if (tombstoneTargetDiverged(run.journal, tombstoneKey, tombstonePath, lstat)) {
982
+ return;
983
+ }
984
+ if (lstat.isSymbolicLink() || lstat.isFile()) {
985
+ fs.unlinkSync(tombstonePath);
1004
986
  }
987
+ } catch (err: unknown) {
988
+ const code =
989
+ err && typeof err === "object" && "code" in err
990
+ ? (err as { code?: string }).code
991
+ : undefined;
992
+ if (code !== "ENOENT") {
993
+ run.emit({
994
+ type: "error",
995
+ path: tombstoneKey,
996
+ message: `tombstone-suppress unlink failed: ${
997
+ err instanceof Error ? err.message : String(err)
998
+ }`,
999
+ });
1000
+ return;
1001
+ }
1002
+ }
1003
+ removeEntry(run.journal, tombstoneKey);
1004
+ counters.filesTombstoned++;
1005
+ run.emit({ type: "progress", path: tombstoneKey, bytes: 0 });
1006
+ }
1005
1007
 
1006
- if (item.action === "conflict") {
1007
- // ── Convergence guard ────────────────────────────────────────────
1008
- // The planner flags a conflict purely from journal-relative deltas:
1009
- // local hash != journal hash AND remote etag != journal etag. It can
1010
- // NEVER compare local bytes against remote bytes directly — the remote
1011
- // LIST only carries {key, size, etag, lastModified}, and ListObjectsV2
1012
- // returns no content hash. So when the journal baseline goes stale,
1013
- // both deltas fire even though local and remote are byte-for-byte
1014
- // identical. Stale-baseline triggers seen in the wild: a shared-journal
1015
- // cross-root collision (personal vault + companies/personal sharing one
1016
- // journal), an mtime-rounding fast-path miss (journal stamps 1700..351,
1017
- // the FS returns 1700..350.96, the `===` fast-path misses and re-hashes
1018
- // harmless on its own but combines with the etag delta), KMS/multipart
1019
- // etag churn on a no-op re-upload, a second machine advancing S3 + its
1020
- // own journal, or a manual revert. Materializing such a false positive
1021
- // as a conflict litters a useless byte-identical `.conflict-*` mirror
1022
- // and, under `--on-conflict abort`, halts the WHOLE sync over zero real
1023
- // divergence. (One live run produced 140 byte-identical mirrors this way.)
1024
- //
1025
- // We have no remote content hash up front, so prove convergence by
1026
- // fetching the remote bytes once — to the very path the conflict mirror
1027
- // would occupy — and hashing them. Identical bytes are not a conflict:
1028
- // re-stamp the journal baseline (so neither side looks changed next run)
1029
- // and skip. Genuine divergence reuses the already-fetched bytes as the
1030
- // inspection mirror, so the common keep/skip path costs no extra I/O.
1031
- const detectedAt = new Date().toISOString();
1032
- const machineId = readShortMachineId(hqRoot);
1033
- const originalRelative = path.relative(hqRoot, localPath);
1034
- const conflictRelative = buildConflictPath(
1035
- originalRelative,
1036
- detectedAt,
1037
- machineId,
1038
- );
1039
- const conflictAbs = path.join(hqRoot, conflictRelative);
1040
- const conflictKey = toPosixKey(path.relative(companyRoot, conflictAbs));
1008
+ async function executeConflictItem(
1009
+ run: PullRunContext,
1010
+ plan: PullPlan,
1011
+ scopeRun: ScopeShrinkRun,
1012
+ counters: PullCounters,
1013
+ downloadItems: PullDownloadItem[],
1014
+ item: Extract<PullPlanItem, { action: "conflict" }>,
1015
+ ): Promise<SyncResult | null> {
1016
+ const { remoteFile, localPath } = item;
1017
+
1018
+ await refreshRunContextIfExpiring(run);
1019
+
1020
+ const detectedAt = new Date().toISOString();
1021
+ const machineId = readShortMachineId(run.hqRoot);
1022
+ const originalRelative = path.relative(run.hqRoot, localPath);
1023
+ const conflictRelative = buildConflictPath(
1024
+ originalRelative,
1025
+ detectedAt,
1026
+ machineId,
1027
+ );
1028
+ const conflictAbs = path.join(run.hqRoot, conflictRelative);
1029
+ const conflictKey = toPosixKey(path.relative(run.companyRoot, conflictAbs));
1030
+
1031
+ if (!isDownloadWritePathStillContained(run.companyRoot, conflictKey, conflictAbs)) {
1032
+ counters.filesSkipped++;
1033
+ run.emit({
1034
+ type: "error",
1035
+ path: remoteFile.key,
1036
+ message: "conflict mirror skipped: local parent escaped the sync root",
1037
+ });
1038
+ return null;
1039
+ }
1041
1040
 
1042
- if (!isDownloadWritePathStillContained(companyRoot, conflictKey, conflictAbs)) {
1043
- filesSkipped++;
1044
- emit({
1045
- type: "error",
1046
- path: remoteFile.key,
1047
- message: "conflict mirror skipped: local parent escaped the sync root",
1048
- });
1049
- continue;
1050
- }
1041
+ let remoteFetched = false;
1042
+ let converged = false;
1043
+ try {
1044
+ const downloaded = await downloadFile(run.ctx, remoteFile.key, conflictAbs);
1045
+ remoteFetched = true;
1046
+ const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
1047
+ ? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
1048
+ : (downloaded.contentHash ?? hashFile(conflictAbs));
1049
+ converged = remoteHash === item.localHash;
1050
+ } catch (probeErr) {
1051
+ run.emit({
1052
+ type: "error",
1053
+ path: remoteFile.key,
1054
+ message: `conflict convergence probe failed: ${
1055
+ probeErr instanceof Error ? probeErr.message : String(probeErr)
1056
+ }`,
1057
+ });
1058
+ }
1051
1059
 
1052
- let remoteFetched = false;
1053
- let converged = false;
1060
+ if (converged) {
1061
+ if (remoteFetched) {
1054
1062
  try {
1055
- const downloaded = await downloadFile(ctx, remoteFile.key, conflictAbs);
1056
- remoteFetched = true;
1057
- // Hash the fetched remote exactly the way the planner hashed local
1058
- // (symlink-aware) so the two hashes are directly comparable. A
1059
- // symlink record round-trips to a symlink on disk; hashing its
1060
- // target string matches `hashSymlinkTarget(localPath)`.
1061
- const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
1062
- ? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
1063
- : (downloaded.contentHash ?? hashFile(conflictAbs));
1064
- converged = remoteHash === item.localHash;
1065
- } catch (probeErr) {
1066
- // Couldn't fetch or hash the remote — fail safe by falling through to
1067
- // the conventional conflict path (converged stays false). No mirror
1068
- // is on disk in this case.
1069
- emit({
1070
- type: "error",
1071
- path: remoteFile.key,
1072
- message: `conflict convergence probe failed: ${
1073
- probeErr instanceof Error ? probeErr.message : String(probeErr)
1074
- }`,
1075
- });
1063
+ fs.rmSync(conflictAbs, { force: true });
1064
+ } catch {
1065
+ /* best-effort cleanup; a stray identical mirror is harmless */
1076
1066
  }
1067
+ }
1068
+ updateEntry(
1069
+ run.journal,
1070
+ remoteFile.key,
1071
+ item.localHash,
1072
+ item.localSize,
1073
+ "down",
1074
+ remoteFile.etag,
1075
+ item.localMtime.getTime(),
1076
+ );
1077
+ run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
1078
+ counters.filesSkipped++;
1079
+ return null;
1080
+ }
1077
1081
 
1078
- if (converged) {
1079
- // False positive: remote == local. Drop the byte-identical mirror and
1080
- // re-stamp the baseline (current localHash + current remoteEtag) so
1081
- // the next sync sees "no change on either side". Counts as a skip.
1082
- if (remoteFetched) {
1083
- try {
1084
- fs.rmSync(conflictAbs, { force: true });
1085
- } catch {
1086
- /* best-effort cleanup; a stray identical mirror is harmless */
1087
- }
1088
- }
1089
- updateEntry(
1090
- journal,
1091
- remoteFile.key,
1092
- item.localHash,
1093
- item.localSize,
1094
- "down",
1095
- remoteFile.etag,
1096
- item.localMtime.getTime(),
1097
- );
1098
- emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
1099
- filesSkipped++;
1100
- continue;
1101
- }
1082
+ counters.conflicts++;
1083
+ counters.conflictPaths.push(remoteFile.key);
1084
+
1085
+ const resolution = await resolveConflict(
1086
+ {
1087
+ path: remoteFile.key,
1088
+ localHash: item.localHash,
1089
+ remoteModified: remoteFile.lastModified,
1090
+ localModified: item.localMtime,
1091
+ direction: "pull",
1092
+ },
1093
+ run.options.onConflict,
1094
+ );
1102
1095
 
1103
- // ── Genuine divergence ───────────────────────────────────────────
1104
- conflicts++;
1105
- conflictPaths.push(remoteFile.key);
1096
+ run.emit({
1097
+ type: "conflict",
1098
+ path: remoteFile.key,
1099
+ direction: "pull",
1100
+ resolution,
1101
+ });
1106
1102
 
1107
- const resolution = await resolveConflict(
1108
- {
1109
- path: remoteFile.key,
1103
+ if (resolution !== "abort" && resolution !== "overwrite") {
1104
+ if (remoteFetched) {
1105
+ try {
1106
+ appendConflictEntry(run.hqRoot, {
1107
+ id: buildConflictId(originalRelative, detectedAt),
1108
+ originalPath: originalRelative,
1109
+ conflictPath: conflictRelative,
1110
+ detectedAt,
1111
+ side: "pull",
1112
+ machineId,
1110
1113
  localHash: item.localHash,
1111
- remoteModified: remoteFile.lastModified,
1112
- // Use the lstat-mtime captured by the planner — statSync
1113
- // here would follow a dangling symlink and throw ENOENT,
1114
- // aborting the pull before resolveConflict could prompt.
1115
- localModified: item.localMtime,
1116
- direction: "pull",
1117
- },
1118
- onConflict,
1119
- );
1120
-
1121
- emit({
1122
- type: "conflict",
1123
- path: remoteFile.key,
1124
- direction: "pull",
1125
- resolution,
1126
- });
1127
-
1128
- // The remote bytes were already fetched to `conflictAbs` by the
1129
- // convergence probe. For "keep"/"skip" they become the
1130
- // `<original>.conflict-<ts>-<machine>.<ext>` inspection mirror — just
1131
- // index it (no second download). For "abort" (user gave up) and
1132
- // "overwrite" (cloud bytes are about to replace local) the mirror is
1133
- // redundant, so discard it. Best-effort: failure here only emits an
1134
- // error, doesn't break the sync.
1135
- if (resolution !== "abort" && resolution !== "overwrite") {
1136
- if (remoteFetched) {
1137
- try {
1138
- appendConflictEntry(hqRoot, {
1139
- id: buildConflictId(originalRelative, detectedAt),
1140
- originalPath: originalRelative,
1141
- conflictPath: conflictRelative,
1142
- detectedAt,
1143
- side: "pull",
1144
- machineId,
1145
- localHash: item.localHash,
1146
- remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
1147
- });
1148
- } catch (mirrorErr) {
1149
- emit({
1150
- type: "error",
1151
- path: remoteFile.key,
1152
- message: `conflict mirror index write failed: ${
1153
- mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
1154
- }`,
1155
- });
1156
- }
1157
- }
1158
- // If the probe download failed (!remoteFetched) there is no mirror on
1159
- // disk; the probe already emitted the error. The conflict is still
1160
- // surfaced and journal-stamped below so it doesn't re-fire silently.
1161
- } else if (remoteFetched) {
1162
- try {
1163
- fs.rmSync(conflictAbs, { force: true });
1164
- } catch {
1165
- /* best-effort; a leftover mirror is cosmetic, not corrupting */
1166
- }
1167
- }
1168
-
1169
- if (resolution === "abort") {
1170
- emit({ type: "new-files", files: [] });
1171
- writeJournal(journalSlug, journal);
1172
- aborted = true;
1173
- abortResult = {
1174
- filesDownloaded,
1175
- bytesDownloaded,
1176
- filesSkipped,
1177
- conflicts,
1178
- conflictPaths,
1179
- aborted: true,
1180
- newFiles: plan.newFiles,
1181
- newFilesCount: plan.newFilesCount,
1182
- filesExcludedByPolicy: plan.filesExcludedByPolicy,
1183
- // Abort short-circuits before the tombstone loop runs; report
1184
- // 0 so the field shape stays stable for consumers that
1185
- // destructure it.
1186
- filesTombstoned: 0,
1187
- // Scope-shrink ran before execution, so its counts are real even on
1188
- // a conflict abort. `filesOutOfScope` reflects how far the serial
1189
- // pass got before the abort; that's acceptable for an abort result.
1190
- filesOutOfScope,
1191
- scopeOrphansRemoved,
1192
- scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
1193
- };
1194
- break;
1195
- }
1196
- if (resolution === "keep" || resolution === "skip") {
1197
- filesSkipped++;
1198
- // Stamp the journal with the new baseline so the same conflict
1199
- // doesn't re-fire on every subsequent sync. After "keep", local
1200
- // wins — the user has accepted that the cloud version we just
1201
- // mirrored is what cloud is at this etag, and they don't want
1202
- // it. Recording (current localHash + current remoteEtag) tells
1203
- // the next sync "no change on either side" until something new
1204
- // diverges. Without this, both `localChanged` and `remoteChanged`
1205
- // stay true forever and the conflict is sticky.
1206
- // Stamp from planner-captured size (symlink-aware), NOT
1207
- // statSync — which would follow a dangling symlink and
1208
- // throw ENOENT, get swallowed, and leave the journal
1209
- // stale so this conflict would re-fire on every sync
1210
- // forever. localSize is sourced from the same lstat that
1211
- // computed localMtime + localHash above.
1212
- updateEntry(
1213
- journal,
1214
- remoteFile.key,
1215
- item.localHash,
1216
- item.localSize,
1217
- "down",
1218
- remoteFile.etag,
1219
- item.localMtime.getTime(),
1220
- );
1221
- continue;
1114
+ remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
1115
+ });
1116
+ } catch (mirrorErr) {
1117
+ run.emit({
1118
+ type: "error",
1119
+ path: remoteFile.key,
1120
+ message: `conflict mirror index write failed: ${
1121
+ mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
1122
+ }`,
1123
+ });
1222
1124
  }
1223
- // "overwrite" falls through to download — re-route through the pool
1224
- // so it benefits from parallelism too. Synthesize a download item
1225
- // pointing at the same remoteFile/localPath; isNew=false because
1226
- // there was a conflict-eligible local file present.
1227
- downloadItems.push({
1228
- action: "download",
1229
- remoteFile,
1230
- localPath,
1231
- isNew: false,
1232
- });
1233
- continue;
1125
+ }
1126
+ } else if (remoteFetched) {
1127
+ try {
1128
+ fs.rmSync(conflictAbs, { force: true });
1129
+ } catch {
1130
+ /* best-effort; a leftover mirror is cosmetic, not corrupting */
1234
1131
  }
1235
1132
  }
1236
1133
 
1237
- // Early-return on conflict abort BEFORE running the parallel download
1238
- // pool — the abort intent is "stop the pull now", not "stop new work but
1239
- // finish what's in flight". Since the pool hasn't started yet, this is
1240
- // a clean drain (zero items in flight) by construction.
1241
- if (aborted && abortResult) {
1242
- return abortResult;
1134
+ if (resolution === "abort") {
1135
+ run.emit({ type: "new-files", files: [] });
1136
+ writeJournal(run.journalSlug, run.journal);
1137
+ return {
1138
+ filesDownloaded: counters.filesDownloaded,
1139
+ bytesDownloaded: counters.bytesDownloaded,
1140
+ filesSkipped: counters.filesSkipped,
1141
+ conflicts: counters.conflicts,
1142
+ conflictPaths: counters.conflictPaths,
1143
+ aborted: true,
1144
+ newFiles: plan.newFiles,
1145
+ newFilesCount: plan.newFilesCount,
1146
+ filesExcludedByPolicy: plan.filesExcludedByPolicy,
1147
+ filesTombstoned: 0,
1148
+ filesOutOfScope: counters.filesOutOfScope,
1149
+ scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
1150
+ scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
1151
+ };
1243
1152
  }
1244
1153
 
1245
- // ── Parallel download pool (5.36.0) ───────────────────────────────────
1246
- // Bounded concurrency: TRANSFER_CONCURRENCY simultaneous downloads. Same
1247
- // race-based shape as the HEAD-verify pool above but applied to the body
1248
- // of each download. Per-item progress events fire at file-settle time
1249
- // (inside the wrapper), so cross-file interleave is expected and the
1250
- // menubar's stream parser already handles it.
1251
- if (downloadItems.length > 0) {
1252
- // Batch pre-mint GET URLs for every download in one shot (chunked server-
1253
- // side) so the pool below — and the new-files HEAD enrichment that follows,
1254
- // which re-reads the same keys — reuse them instead of presigning per file.
1255
- // On a large initial pull this is the difference between ~ceil(N/100)
1256
- // presign calls and N (which would 429 past the 100-req/hr limit). No-op
1257
- // on the S3 SDK transport; best-effort (failure falls back to per-file).
1258
- await primeObjectTransport(
1259
- ctx,
1260
- "get",
1261
- downloadItems.map((d) => d.remoteFile.key),
1154
+ if (resolution === "keep" || resolution === "skip") {
1155
+ counters.filesSkipped++;
1156
+ updateEntry(
1157
+ run.journal,
1158
+ remoteFile.key,
1159
+ item.localHash,
1160
+ item.localSize,
1161
+ "down",
1162
+ remoteFile.etag,
1163
+ item.localMtime.getTime(),
1262
1164
  );
1165
+ return null;
1166
+ }
1263
1167
 
1264
- const queue = [...downloadItems];
1265
- const inFlight: Set<Promise<unknown>> = new Set();
1168
+ downloadItems.push({
1169
+ action: "download",
1170
+ remoteFile,
1171
+ localPath,
1172
+ isNew: false,
1173
+ });
1174
+ return null;
1175
+ }
1266
1176
 
1267
- const downloadOne = async (
1268
- downloadItem: typeof downloadItems[number],
1269
- ): Promise<void> => {
1270
- const { remoteFile, localPath } = downloadItem;
1177
+ async function executeDownloadExecutor(
1178
+ run: PullRunContext,
1179
+ downloadItems: PullDownloadItem[],
1180
+ transferConcurrency: number,
1181
+ counters: PullCounters,
1182
+ ): Promise<void> {
1183
+ if (downloadItems.length === 0) return;
1271
1184
 
1272
- // Auto-refresh context if credentials expiring. Each task checks
1273
- // independently — refresh is idempotent on the same context object.
1274
- if (isExpiringSoon(ctx.expiresAt)) {
1275
- ctx = await refreshEntityContext(companyRef, vaultConfig);
1276
- }
1185
+ await primeObjectTransport(
1186
+ run.ctx,
1187
+ "get",
1188
+ downloadItems.map((d) => d.remoteFile.key),
1189
+ );
1277
1190
 
1278
- if (!isDownloadWritePathStillContained(companyRoot, remoteFile.key, localPath)) {
1279
- filesSkipped++;
1280
- emit({
1281
- type: "error",
1282
- path: remoteFile.key,
1283
- message: "download skipped: local parent escaped the sync root",
1191
+ const queue = [...downloadItems];
1192
+ const inFlight: Set<Promise<unknown>> = new Set();
1193
+ const workerErrors: Error[] = [];
1194
+
1195
+ while (queue.length > 0 || inFlight.size > 0) {
1196
+ while (inFlight.size < transferConcurrency && queue.length > 0) {
1197
+ const downloadItem = queue.shift()!;
1198
+ const p: Promise<void> = downloadOne(run, downloadItem, counters)
1199
+ .catch((err: unknown) => {
1200
+ workerErrors.push(err instanceof Error ? err : new Error(String(err)));
1201
+ })
1202
+ .finally(() => {
1203
+ inFlight.delete(p);
1284
1204
  });
1285
- return;
1286
- }
1205
+ inFlight.add(p);
1206
+ }
1207
+ if (inFlight.size > 0) {
1208
+ await Promise.race(Array.from(inFlight));
1209
+ }
1210
+ }
1287
1211
 
1288
- try {
1289
- const { metadata, contentHash, contentSize } = await downloadFile(
1290
- ctx,
1291
- remoteFile.key,
1292
- localPath,
1293
- );
1294
- const author = metadata?.["created-by"] ?? null;
1295
- // Author sub for the scope-shrink authorship guard — same field the
1296
- // upload side stamps, read straight off the GET response metadata.
1297
- const createdBySub = metadata?.["created-by-sub"];
1298
-
1299
- // Symlink records materialize as real symlinks on disk. lstat
1300
- // (does not follow) lets us detect that case so the journal stamp
1301
- // mirrors what the push side would emit on the next tick:
1302
- // hash = sha256(readlink target string)
1303
- // size = 0
1304
- // Without this check, hashFile would follow the link and stamp the
1305
- // target file's contents — a value the next push would never
1306
- // produce — which makes skipUnchanged perpetually re-upload every
1307
- // symlink, defeating the point of the gate.
1308
- const localLstat = fs.lstatSync(localPath);
1309
- const isLocalSymlink = localLstat.isSymbolicLink();
1310
- const hash = isLocalSymlink
1311
- ? hashSymlinkTarget(fs.readlinkSync(localPath))
1312
- : (contentHash ?? hashFile(localPath));
1313
- const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
1314
- // Capture the listing's ETag so subsequent syncs can detect remote
1315
- // drift independently of mtime drift. Stamp mtimeMs from localLstat
1316
- // (5.36.0) so the next push planner's lstat fast-path can skip the
1317
- // SHA256 for this file without reading its bytes.
1318
- //
1319
- // 5.37.0 ordering invariant: downloadFile applies hq-mtime via
1320
- // utimesSync AFTER its byte write but BEFORE returning, and this
1321
- // lstat runs AFTER downloadFile resolves — so localLstat.mtimeMs
1322
- // already reflects the source-stamped mtime, not the wall-clock
1323
- // write-time. The journal therefore matches what the next push's
1324
- // lstat fast-path will see, and the file is correctly skipped on
1325
- // re-sync instead of being hashed every tick. Do not move this
1326
- // lstat earlier; do not stamp the journal from any pre-download
1327
- // mtime.
1328
- updateEntry(
1329
- journal,
1330
- remoteFile.key,
1331
- hash,
1332
- size,
1333
- "down",
1334
- remoteFile.etag,
1335
- localLstat.mtimeMs,
1336
- createdBySub,
1337
- );
1212
+ if (workerErrors.length > 0) {
1213
+ writeJournal(run.journalSlug, run.journal);
1214
+ const first = workerErrors[0]!;
1215
+ if (workerErrors.length > 1) {
1216
+ first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
1217
+ }
1218
+ throw first;
1219
+ }
1220
+ }
1338
1221
 
1339
- // Attach message from the prior journal entry if present (set by a
1340
- // previous `share` operation that included a --message).
1341
- const priorEntry = getEntry(journal, remoteFile.key);
1342
- const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
1343
- emit({
1344
- type: "progress",
1345
- path: remoteFile.key,
1346
- bytes: size,
1347
- ...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
1348
- ...(author ? { author } : {}),
1349
- });
1222
+ async function downloadOne(
1223
+ run: PullRunContext,
1224
+ downloadItem: PullDownloadItem,
1225
+ counters: PullCounters,
1226
+ ): Promise<void> {
1227
+ const { remoteFile, localPath } = downloadItem;
1350
1228
 
1351
- filesDownloaded++;
1352
- bytesDownloaded += size;
1353
- } catch (err) {
1354
- // STS session policy may deny access to some paths — this is expected
1355
- // for guest members with allowedPrefixes
1356
- if (isAccessDenied(err)) {
1357
- filesSkipped++;
1358
- } else {
1359
- emit({
1360
- type: "error",
1361
- path: remoteFile.key,
1362
- message: err instanceof Error ? err.message : String(err),
1363
- });
1364
- }
1365
- }
1366
- };
1229
+ await refreshRunContextIfExpiring(run);
1367
1230
 
1368
- // Codex P1 (5.36.x): worker promises wrapped in .catch so an
1369
- // unhandled rejection inside downloadOne (e.g. refreshEntityContext
1370
- // before the per-item try/catch, or lstatSync after the download
1371
- // succeeded but before journal stamping) cannot escape
1372
- // `Promise.race(inFlight)` and unwind the drain mid-flight. Without
1373
- // this wrap, sibling downloads kept running after share()/sync()
1374
- // had already failed, their files materialized on disk without
1375
- // matching journal entries, and the next sync re-downloaded
1376
- // everything. Errors are collected and surfaced after the pool
1377
- // fully drains — see workerErrors throw below.
1378
- const workerErrors: Error[] = [];
1379
- while (queue.length > 0 || inFlight.size > 0) {
1380
- while (inFlight.size < TRANSFER_CONCURRENCY && queue.length > 0) {
1381
- const downloadItem = queue.shift()!;
1382
- const p: Promise<void> = downloadOne(downloadItem)
1383
- .catch((err: unknown) => {
1384
- workerErrors.push(err instanceof Error ? err : new Error(String(err)));
1385
- })
1386
- .finally(() => {
1387
- inFlight.delete(p);
1388
- });
1389
- inFlight.add(p);
1390
- }
1391
- if (inFlight.size > 0) {
1392
- // Wait for at least one in-flight task to settle before topping up
1393
- // the pool. allSettled-style semantics via Promise.race — the
1394
- // .catch wrap above guarantees no worker promise can reject.
1395
- await Promise.race(Array.from(inFlight));
1396
- }
1397
- }
1231
+ if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
1232
+ counters.filesSkipped++;
1233
+ run.emit({
1234
+ type: "error",
1235
+ path: remoteFile.key,
1236
+ message: "download skipped: local parent escaped the sync root",
1237
+ });
1238
+ return;
1239
+ }
1398
1240
 
1399
- // Pool drained. If any worker rejected, write the journal first
1400
- // (so the lstat fast-path stamps for successfully-downloaded files
1401
- // persist) then throw the first error, preserving its stack.
1402
- if (workerErrors.length > 0) {
1403
- writeJournal(journalSlug, journal);
1404
- const first = workerErrors[0]!;
1405
- if (workerErrors.length > 1) {
1406
- first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
1407
- }
1408
- throw first;
1241
+ try {
1242
+ const { metadata, contentHash, contentSize } = await downloadFile(
1243
+ run.ctx,
1244
+ remoteFile.key,
1245
+ localPath,
1246
+ );
1247
+ const author = metadata?.["created-by"] ?? null;
1248
+ const createdBySub = metadata?.["created-by-sub"];
1249
+
1250
+ const localLstat = fs.lstatSync(localPath);
1251
+ const isLocalSymlink = localLstat.isSymbolicLink();
1252
+ const hash = isLocalSymlink
1253
+ ? hashSymlinkTarget(fs.readlinkSync(localPath))
1254
+ : (contentHash ?? hashFile(localPath));
1255
+ const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
1256
+
1257
+ updateEntry(
1258
+ run.journal,
1259
+ remoteFile.key,
1260
+ hash,
1261
+ size,
1262
+ "down",
1263
+ remoteFile.etag,
1264
+ localLstat.mtimeMs,
1265
+ createdBySub,
1266
+ );
1267
+
1268
+ const priorEntry = getEntry(run.journal, remoteFile.key);
1269
+ const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
1270
+ run.emit({
1271
+ type: "progress",
1272
+ path: remoteFile.key,
1273
+ bytes: size,
1274
+ ...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
1275
+ ...(author ? { author } : {}),
1276
+ });
1277
+
1278
+ counters.filesDownloaded++;
1279
+ counters.bytesDownloaded += size;
1280
+ } catch (err) {
1281
+ if (isAccessDenied(err)) {
1282
+ counters.filesSkipped++;
1283
+ } else {
1284
+ run.emit({
1285
+ type: "error",
1286
+ path: remoteFile.key,
1287
+ message: err instanceof Error ? err.message : String(err),
1288
+ });
1409
1289
  }
1410
1290
  }
1291
+ }
1411
1292
 
1412
- // ── New-files attribution (US-002) ─────────────────────────────────────
1413
- // Enrich plan.newFiles with `addedBy` from S3 user metadata. HeadObject
1414
- // calls are best-effort and capped at 5 concurrent to avoid hammering S3.
1293
+ async function emitAndReportNewFiles(
1294
+ run: PullRunContext,
1295
+ plan: PullPlan,
1296
+ ): Promise<void> {
1415
1297
  const enrichedNewFiles: Array<{ path: string; bytes: number; addedBy: string | null }> = [];
1298
+ // Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
1299
+ // created-by HEADs below reuse the cache instead of each minting its own
1300
+ // presign. Without this, a big catch-up pull (hundreds of new files) bursts
1301
+ // the presign endpoint, trips the circuit breaker, and every enrichment HEAD
1302
+ // then fails. Mirrors the tombstone HEAD-verify pre-prime.
1303
+ await primeObjectTransport(
1304
+ run.ctx,
1305
+ "get",
1306
+ plan.newFiles.map((nf) => nf.path),
1307
+ );
1416
1308
  const HEAD_CONCURRENCY = 5;
1417
1309
  for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
1418
1310
  const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
@@ -1420,13 +1312,11 @@ async function syncWithOperationLockHeld(
1420
1312
  batch.map(async (nf) => {
1421
1313
  let addedBy: string | null = null;
1422
1314
  try {
1423
- const head = await headRemoteFile(ctx, nf.path);
1315
+ const head = await headRemoteFile(run.ctx, nf.path);
1424
1316
  if (head?.metadata?.["created-by"]) {
1425
1317
  addedBy = head.metadata["created-by"];
1426
1318
  }
1427
1319
  } catch (headErr) {
1428
- // Best-effort: log to console (Sentry captures via global handler)
1429
- // and fall through with addedBy = null.
1430
1320
  try {
1431
1321
  console.error(
1432
1322
  `[hq-sync] HeadObject failed for ${nf.path}: ${
@@ -1442,220 +1332,150 @@ async function syncWithOperationLockHeld(
1442
1332
  );
1443
1333
  enrichedNewFiles.push(...results);
1444
1334
  }
1445
- emit({ type: "new-files", files: enrichedNewFiles });
1446
-
1447
- // Report new files to the notification service so they persist as a
1448
- // cross-session "new files" history in the HQ Sync app (POST
1449
- // /v1/notify/file-added → per-recipient FILE_EVENT rows for THIS user, who is
1450
- // the one the files are new for). Best-effort and bounded: a failure or a
1451
- // hung request must never delay or break the sync — the durable signal is the
1452
- // synced file itself, this is only a notification mirror.
1453
- await reportNewFilesToNotify(vaultConfig, ctx.uid, ctx.slug, enrichedNewFiles);
1454
-
1455
- // Codex P1 (PR #24 follow-up): scope-gate tombstone candidates with
1456
- // a per-object HEAD before unlinking. `listRemoteFiles` is STS-scoped
1457
- // (guest sessions with `allowedPrefixes`, role downgrade, custom
1458
- // sync-mode prefix sets — see the AccessDenied branch in the download
1459
- // loop above), so a journal entry's absence from LIST does not prove
1460
- // the object was deleted; it may simply be invisible to this session.
1461
- // HEAD each candidate:
1462
- // - HEAD returns metadata → object exists → NOT in our LIST scope →
1463
- // skip the tombstone (peer didn't delete it; we just can't see it).
1464
- // - HEAD returns null (NotFound) confirmed deleted tombstone.
1465
- // - HEAD throws AccessDenied can't tell → defensive skip; journal
1466
- // stays so next sync (with broader scope) can re-evaluate.
1467
- // - HEAD throws transient → defensive skip + emit error.
1468
- // Bounded concurrency mirrors the new-files attribution pass above.
1469
- if (plan.tombstones.length > 0) {
1470
- // Pre-mint GET URLs for the tombstone HEAD-verify probes below (headRemote
1471
- // File presigns a GET), so a large delete set doesn't add N presign calls.
1472
- await primeObjectTransport(ctx, "get", plan.tombstones);
1473
-
1474
- const HEAD_VERIFY_CONCURRENCY = 5;
1475
- const verified: string[] = [];
1476
- for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
1477
- const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
1478
- const results = await Promise.all(
1479
- batch.map(async (key) => {
1480
- try {
1481
- const head = await headRemoteFile(ctx, key);
1482
- return head === null ? key : null;
1483
- } catch (err) {
1484
- if (isAccessDenied(err)) return null;
1485
- emit({
1486
- type: "error",
1487
- path: key,
1488
- message: `tombstone HEAD verify failed (deferring): ${
1489
- err instanceof Error ? err.message : String(err)
1490
- }`,
1491
- });
1492
- return null;
1493
- }
1494
- }),
1495
- );
1496
- for (const k of results) {
1497
- if (k !== null) verified.push(k);
1498
- }
1335
+ run.emit({ type: "new-files", files: enrichedNewFiles });
1336
+ await reportNewFilesToNotify(
1337
+ run.vaultConfig,
1338
+ run.ctx.uid,
1339
+ run.ctx.slug,
1340
+ enrichedNewFiles,
1341
+ );
1342
+ }
1343
+
1344
+ async function verifyPlannedJournalTombstones(
1345
+ run: PullRunContext,
1346
+ plan: PullPlan,
1347
+ ): Promise<void> {
1348
+ if (plan.tombstones.length === 0) return;
1349
+
1350
+ await primeObjectTransport(run.ctx, "get", plan.tombstones);
1351
+
1352
+ const HEAD_VERIFY_CONCURRENCY = 5;
1353
+ const verified: string[] = [];
1354
+ for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
1355
+ const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
1356
+ const results = await Promise.all(
1357
+ batch.map(async (key) => {
1358
+ try {
1359
+ const head = await headRemoteFile(run.ctx, key);
1360
+ return head === null ? key : null;
1361
+ } catch (err) {
1362
+ if (isAccessDenied(err)) return null;
1363
+ run.emit({
1364
+ type: "error",
1365
+ path: key,
1366
+ message: `tombstone HEAD verify failed (deferring): ${
1367
+ err instanceof Error ? err.message : String(err)
1368
+ }`,
1369
+ });
1370
+ return null;
1371
+ }
1372
+ }),
1373
+ );
1374
+ for (const k of results) {
1375
+ if (k !== null) verified.push(k);
1499
1376
  }
1500
- plan.tombstones = verified;
1501
1377
  }
1378
+ plan.tombstones = verified;
1379
+ }
1502
1380
 
1503
- // Bug #9 — apply cross-machine delete propagation. Each tombstone is a
1504
- // key the journal records as previously synced but the remote LIST no
1505
- // longer contains. We delete the local file (or symlink, or empty dir
1506
- // remnant) and drop the journal entry so the next sync's planner stays
1507
- // converged. Failures are reported but non-fatal — the entry stays in
1508
- // the journal and the next run retries.
1381
+ function executeJournalTombstoneDeletes(
1382
+ run: PullRunContext,
1383
+ plan: PullPlan,
1384
+ counters: PullCounters,
1385
+ ): void {
1509
1386
  for (const key of plan.tombstones) {
1510
- // Last line of defense: a malformed or traversal key must NEVER reach
1511
- // fs.unlinkSync or journal mutation for a path outside the sync root.
1512
- const localPath = resolveContainedVaultPath(companyRoot, key);
1387
+ const localPath = resolveContainedVaultPath(run.companyRoot, key);
1513
1388
  if (localPath === null) continue;
1514
1389
  let removedSomething = false;
1515
1390
  try {
1516
1391
  const lstat = fs.lstatSync(localPath);
1517
- if (tombstoneTargetDiverged(journal, key, localPath, lstat)) {
1392
+ if (tombstoneTargetDiverged(run.journal, key, localPath, lstat)) {
1518
1393
  continue;
1519
1394
  }
1520
1395
  if (lstat.isSymbolicLink() || lstat.isFile()) {
1521
1396
  fs.unlinkSync(localPath);
1522
1397
  removedSomething = true;
1523
1398
  } else if (lstat.isDirectory()) {
1524
- // A dir at a key likely from a (local-dir, cloud-file) historic
1525
- // state. Don't recursively rm-rf the operator's dir; just drop
1526
- // the journal entry so we converge with reality.
1399
+ // A dir at a key is converged by dropping only the journal entry.
1527
1400
  }
1528
1401
  } catch (err: unknown) {
1529
1402
  const code =
1530
1403
  err && typeof err === "object" && "code" in err
1531
1404
  ? (err as { code?: string }).code
1532
1405
  : undefined;
1533
- // ENOENT → local already gone; safe to drop the journal entry.
1534
- // Other errors (EACCES/EPERM/EBUSY/etc.) leave the local file in
1535
- // place — if we dropped the journal entry anyway, the pull side
1536
- // would forget the peer's delete and a later push could re-upload
1537
- // the still-present local file, silently undoing the peer's delete.
1538
- // Surface the error and KEEP the journal entry so the next sync
1539
- // retries the unlink after the operator fixes the permission.
1540
1406
  if (code !== "ENOENT") {
1541
- emit({
1407
+ run.emit({
1542
1408
  type: "error",
1543
1409
  path: key,
1544
1410
  message: `tombstone unlink failed: ${
1545
1411
  err instanceof Error ? err.message : String(err)
1546
1412
  }`,
1547
1413
  });
1548
- // Skip removeEntry / filesTombstoned / progress event — the
1549
- // tombstone hasn't actually been honored. Next sync retries.
1550
1414
  continue;
1551
1415
  }
1552
1416
  }
1553
- removeEntry(journal, key);
1554
- filesTombstoned++;
1555
- emit({
1417
+ removeEntry(run.journal, key);
1418
+ counters.filesTombstoned++;
1419
+ run.emit({
1556
1420
  type: "progress",
1557
1421
  path: key,
1558
1422
  bytes: 0,
1559
1423
  deleted: true,
1560
- // Suffix differentiates a tombstone from a normal delete in the
1561
- // tty stream — matches the push-side `defaultConsoleLogger`
1562
- // tombstone surface in share.ts.
1563
1424
  message: removedSomething ? "tombstone (cross-machine delete)" : "tombstone (already absent locally)",
1564
1425
  });
1565
1426
  }
1427
+ }
1566
1428
 
1567
- // Record this pull's boundary (US-005) so the NEXT pull can diff its scope
1568
- // against ours and detect a shrink. Append before the journal write so it
1569
- // persists. `prefixSet` is stored in the same company-relative namespace as
1570
- // the journal keys; `all` mode records `[""]` (covers everything).
1571
- appendPullRecord(journal, {
1429
+ function finalizePullRun(
1430
+ run: PullRunContext,
1431
+ plan: PullPlan,
1432
+ scopeRun: ScopeShrinkRun,
1433
+ counters: PullCounters,
1434
+ ): SyncResult {
1435
+ appendPullRecord(run.journal, {
1572
1436
  pullId: generatePullId(),
1573
- companyUid: ctx.uid,
1574
- startedAt,
1437
+ companyUid: run.ctx.uid,
1438
+ startedAt: run.startedAt,
1575
1439
  completedAt: new Date().toISOString(),
1576
- syncMode,
1577
- prefixSet: currentPrefixSet,
1578
- scopeChangeDetected: shrinkPlan.scopeChangeDetected,
1579
- orphansRemoved: scopeOrphansRemoved,
1580
- orphansBlocked: shrinkResult.dirtyTombstoned,
1440
+ syncMode: run.syncMode,
1441
+ prefixSet: run.currentPrefixSet,
1442
+ scopeChangeDetected: scopeRun.shrinkPlan.scopeChangeDetected,
1443
+ orphansRemoved: scopeRun.scopeOrphansRemoved,
1444
+ orphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
1581
1445
  });
1582
1446
 
1583
- // Stamp lastSync on every successful run so the menubar's "Last sync · X ago"
1584
- // ticks even when nothing transferred. updateEntry only fires on actual
1585
- // downloads; without this, a no-op sync leaves lastSync at the time of the
1586
- // last file change, which is misleading.
1587
- journal.lastSync = new Date().toISOString();
1588
- writeJournal(journalSlug, journal);
1589
-
1590
- // When the pull actually changed on-disk sources (new files, tombstoned
1591
- // removals, or scope-orphan cleanups), refresh the generated skill wrappers,
1592
- // personal-overlay mirrors, and workers registry. reindex is idempotent and
1593
- // best-effort — it must never fail a sync, and is skipped on no-op syncs
1594
- // (the common daemon case) and when the caller opts out via skipReindex.
1447
+ run.journal.lastSync = new Date().toISOString();
1448
+ writeJournal(run.journalSlug, run.journal);
1449
+
1595
1450
  const changedOnDisk =
1596
- filesDownloaded > 0 ||
1597
- filesTombstoned > 0 ||
1598
- scopeOrphansRemoved > 0;
1599
- if (!options.skipReindex && changedOnDisk) {
1451
+ counters.filesDownloaded > 0 ||
1452
+ counters.filesTombstoned > 0 ||
1453
+ scopeRun.scopeOrphansRemoved > 0;
1454
+ if (!run.options.skipReindex && changedOnDisk) {
1600
1455
  try {
1601
- // skipLock: the surrounding sync run already holds this root's operation
1602
- // lock; reindex re-acquiring would refuse against our own live PID.
1603
- reindex({ repoRoot: hqRoot, skipLock: true });
1456
+ reindex({ repoRoot: run.hqRoot, skipLock: true });
1604
1457
  } catch {
1605
1458
  // best-effort: a post-sync refresh failure never fails the sync
1606
1459
  }
1607
1460
  }
1608
1461
 
1609
1462
  return {
1610
- filesDownloaded,
1611
- bytesDownloaded,
1612
- filesSkipped,
1613
- conflicts,
1614
- conflictPaths,
1463
+ filesDownloaded: counters.filesDownloaded,
1464
+ bytesDownloaded: counters.bytesDownloaded,
1465
+ filesSkipped: counters.filesSkipped,
1466
+ conflicts: counters.conflicts,
1467
+ conflictPaths: counters.conflictPaths,
1615
1468
  aborted: false,
1616
1469
  newFiles: plan.newFiles,
1617
1470
  newFilesCount: plan.newFilesCount,
1618
1471
  filesExcludedByPolicy: plan.filesExcludedByPolicy,
1619
- filesTombstoned,
1620
- filesOutOfScope,
1621
- scopeOrphansRemoved,
1622
- scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
1472
+ filesTombstoned: counters.filesTombstoned,
1473
+ filesOutOfScope: counters.filesOutOfScope,
1474
+ scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
1475
+ scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
1623
1476
  };
1624
1477
  }
1625
1478
 
1626
- /**
1627
- * Resolve active company from .hq/config.json.
1628
- */
1629
- function resolveActiveCompany(hqRoot: string): string | undefined {
1630
- const configPath = path.join(hqRoot, ".hq", "config.json");
1631
- if (fs.existsSync(configPath)) {
1632
- try {
1633
- const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
1634
- return config.activeCompany ?? config.companySlug;
1635
- } catch {
1636
- // Ignore parse errors
1637
- }
1638
- }
1639
- return undefined;
1640
- }
1641
-
1642
- /**
1643
- * Returns true when the remote object appears to have moved since the
1644
- * journal entry's last-recorded sync. Prefers ETag equality; falls back to
1645
- * `lastModified > syncedAt` for legacy entries written before remoteEtag
1646
- * was tracked. Conservative on tie (`<=` skews "remote unchanged").
1647
- */
1648
- function hasRemoteChanged(
1649
- remote: { lastModified: Date; etag: string },
1650
- entry: { syncedAt: string; remoteEtag?: string },
1651
- ): boolean {
1652
- if (entry.remoteEtag) {
1653
- return normalizeEtag(remote.etag) !== entry.remoteEtag;
1654
- }
1655
- const syncedAt = new Date(entry.syncedAt).getTime();
1656
- return remote.lastModified.getTime() > syncedAt;
1657
- }
1658
-
1659
1479
  /**
1660
1480
  * Decide whether a remote object present in the LIST is a GENUINE RE-CREATE
1661
1481
  * written AFTER a FILE_TOMBSTONE — in which case the tombstone is stale and the
@@ -2355,16 +2175,6 @@ function computePullPlan(
2355
2175
  };
2356
2176
  }
2357
2177
 
2358
- /**
2359
- * Check if an error is an S3 access denied (expected for filtered guests).
2360
- */
2361
- function isAccessDenied(err: unknown): boolean {
2362
- if (err && typeof err === "object" && "name" in err) {
2363
- return err.name === "AccessDenied" || err.name === "Forbidden";
2364
- }
2365
- return false;
2366
- }
2367
-
2368
2178
  /**
2369
2179
  * Default human-readable event rendering. Preserves the exact output format
2370
2180
  * that `hq sync` emitted before SyncProgressEvent was introduced, so callers