@indigoai-us/hq-cloud 6.11.11 → 6.11.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/dist/bin/sync-runner-company.d.ts +35 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -0
  3. package/dist/bin/sync-runner-company.js +290 -0
  4. package/dist/bin/sync-runner-company.js.map +1 -0
  5. package/dist/bin/sync-runner-events.d.ts +12 -0
  6. package/dist/bin/sync-runner-events.d.ts.map +1 -0
  7. package/dist/bin/sync-runner-events.js +12 -0
  8. package/dist/bin/sync-runner-events.js.map +1 -0
  9. package/dist/bin/sync-runner-planning.d.ts +53 -0
  10. package/dist/bin/sync-runner-planning.d.ts.map +1 -0
  11. package/dist/bin/sync-runner-planning.js +59 -0
  12. package/dist/bin/sync-runner-planning.js.map +1 -0
  13. package/dist/bin/sync-runner-rollup.d.ts +24 -0
  14. package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
  15. package/dist/bin/sync-runner-rollup.js +46 -0
  16. package/dist/bin/sync-runner-rollup.js.map +1 -0
  17. package/dist/bin/sync-runner-telemetry.d.ts +5 -0
  18. package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
  19. package/dist/bin/sync-runner-telemetry.js +5 -0
  20. package/dist/bin/sync-runner-telemetry.js.map +1 -0
  21. package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
  22. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
  23. package/dist/bin/sync-runner-watch-loop.js +372 -0
  24. package/dist/bin/sync-runner-watch-loop.js.map +1 -0
  25. package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
  26. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
  27. package/dist/bin/sync-runner-watch-routes.js +74 -0
  28. package/dist/bin/sync-runner-watch-routes.js.map +1 -0
  29. package/dist/bin/sync-runner.d.ts +5 -54
  30. package/dist/bin/sync-runner.d.ts.map +1 -1
  31. package/dist/bin/sync-runner.js +76 -978
  32. package/dist/bin/sync-runner.js.map +1 -1
  33. package/dist/bin/sync-runner.test.js +265 -11
  34. package/dist/bin/sync-runner.test.js.map +1 -1
  35. package/dist/cli/reindex.d.ts.map +1 -1
  36. package/dist/cli/reindex.js +34 -17
  37. package/dist/cli/reindex.js.map +1 -1
  38. package/dist/cli/reindex.test.js +39 -5
  39. package/dist/cli/reindex.test.js.map +1 -1
  40. package/dist/cli/rescue-classify-ordering.test.js +75 -0
  41. package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
  42. package/dist/cli/rescue-core.d.ts +45 -0
  43. package/dist/cli/rescue-core.d.ts.map +1 -1
  44. package/dist/cli/rescue-core.js +320 -170
  45. package/dist/cli/rescue-core.js.map +1 -1
  46. package/dist/cli/share.d.ts +2 -1
  47. package/dist/cli/share.d.ts.map +1 -1
  48. package/dist/cli/share.js +276 -660
  49. package/dist/cli/share.js.map +1 -1
  50. package/dist/cli/share.test.js +30 -0
  51. package/dist/cli/share.test.js.map +1 -1
  52. package/dist/cli/sync.d.ts +28 -1
  53. package/dist/cli/sync.d.ts.map +1 -1
  54. package/dist/cli/sync.js +541 -748
  55. package/dist/cli/sync.js.map +1 -1
  56. package/dist/cli/sync.test.js +382 -1
  57. package/dist/cli/sync.test.js.map +1 -1
  58. package/dist/cognito-auth.d.ts.map +1 -1
  59. package/dist/cognito-auth.js +55 -10
  60. package/dist/cognito-auth.js.map +1 -1
  61. package/dist/cognito-auth.test.js +61 -0
  62. package/dist/cognito-auth.test.js.map +1 -1
  63. package/dist/daemon-worker.d.ts +2 -2
  64. package/dist/daemon-worker.js +3 -3
  65. package/dist/daemon-worker.js.map +1 -1
  66. package/dist/index.d.ts +2 -1
  67. package/dist/index.d.ts.map +1 -1
  68. package/dist/index.js +1 -1
  69. package/dist/index.js.map +1 -1
  70. package/dist/journal.d.ts.map +1 -1
  71. package/dist/journal.js +93 -6
  72. package/dist/journal.js.map +1 -1
  73. package/dist/journal.test.js +59 -0
  74. package/dist/journal.test.js.map +1 -1
  75. package/dist/machine-auth.test.js +60 -2
  76. package/dist/machine-auth.test.js.map +1 -1
  77. package/dist/object-io.d.ts +37 -1
  78. package/dist/object-io.d.ts.map +1 -1
  79. package/dist/object-io.js +149 -30
  80. package/dist/object-io.js.map +1 -1
  81. package/dist/object-io.test.js +121 -0
  82. package/dist/object-io.test.js.map +1 -1
  83. package/dist/operation-lock.d.ts +8 -8
  84. package/dist/operation-lock.d.ts.map +1 -1
  85. package/dist/operation-lock.js +99 -32
  86. package/dist/operation-lock.js.map +1 -1
  87. package/dist/operation-lock.test.js +51 -4
  88. package/dist/operation-lock.test.js.map +1 -1
  89. package/dist/personal-vault.d.ts.map +1 -1
  90. package/dist/personal-vault.js +8 -2
  91. package/dist/personal-vault.js.map +1 -1
  92. package/dist/personal-vault.test.js +34 -0
  93. package/dist/personal-vault.test.js.map +1 -1
  94. package/dist/prefix-coalesce.d.ts +20 -9
  95. package/dist/prefix-coalesce.d.ts.map +1 -1
  96. package/dist/prefix-coalesce.js +124 -28
  97. package/dist/prefix-coalesce.js.map +1 -1
  98. package/dist/prefix-coalesce.test.js +57 -2
  99. package/dist/prefix-coalesce.test.js.map +1 -1
  100. package/dist/remote-pull.d.ts +8 -3
  101. package/dist/remote-pull.d.ts.map +1 -1
  102. package/dist/remote-pull.js +85 -16
  103. package/dist/remote-pull.js.map +1 -1
  104. package/dist/remote-pull.test.js +213 -2
  105. package/dist/remote-pull.test.js.map +1 -1
  106. package/dist/s3.d.ts +2 -0
  107. package/dist/s3.d.ts.map +1 -1
  108. package/dist/s3.js +197 -116
  109. package/dist/s3.js.map +1 -1
  110. package/dist/s3.test.js +109 -0
  111. package/dist/s3.test.js.map +1 -1
  112. package/dist/scope-shrink.d.ts +3 -2
  113. package/dist/scope-shrink.d.ts.map +1 -1
  114. package/dist/scope-shrink.js +1 -1
  115. package/dist/scope-shrink.js.map +1 -1
  116. package/dist/skill-telemetry.d.ts +1 -1
  117. package/dist/skill-telemetry.d.ts.map +1 -1
  118. package/dist/skill-telemetry.js +69 -9
  119. package/dist/skill-telemetry.js.map +1 -1
  120. package/dist/skill-telemetry.test.js +86 -0
  121. package/dist/skill-telemetry.test.js.map +1 -1
  122. package/dist/sync/event-sync.d.ts +6 -0
  123. package/dist/sync/event-sync.d.ts.map +1 -1
  124. package/dist/sync/event-sync.js +34 -1
  125. package/dist/sync/event-sync.js.map +1 -1
  126. package/dist/sync/event-sync.test.js +73 -0
  127. package/dist/sync/event-sync.test.js.map +1 -1
  128. package/dist/sync/metrics.d.ts +17 -1
  129. package/dist/sync/metrics.d.ts.map +1 -1
  130. package/dist/sync/metrics.js +32 -1
  131. package/dist/sync/metrics.js.map +1 -1
  132. package/dist/sync/metrics.test.js +74 -1
  133. package/dist/sync/metrics.test.js.map +1 -1
  134. package/dist/sync/pull-scope.d.ts.map +1 -1
  135. package/dist/sync/pull-scope.js +15 -7
  136. package/dist/sync/pull-scope.js.map +1 -1
  137. package/dist/sync/push-receiver.d.ts +12 -5
  138. package/dist/sync/push-receiver.d.ts.map +1 -1
  139. package/dist/sync/push-receiver.js +45 -17
  140. package/dist/sync/push-receiver.js.map +1 -1
  141. package/dist/sync/push-receiver.test.js +67 -1
  142. package/dist/sync/push-receiver.test.js.map +1 -1
  143. package/dist/sync-core.d.ts +27 -0
  144. package/dist/sync-core.d.ts.map +1 -0
  145. package/dist/sync-core.js +54 -0
  146. package/dist/sync-core.js.map +1 -0
  147. package/dist/telemetry.d.ts +1 -1
  148. package/dist/telemetry.d.ts.map +1 -1
  149. package/dist/telemetry.js +59 -6
  150. package/dist/telemetry.js.map +1 -1
  151. package/dist/telemetry.test.js +74 -0
  152. package/dist/telemetry.test.js.map +1 -1
  153. package/dist/types.d.ts +8 -0
  154. package/dist/types.d.ts.map +1 -1
  155. package/dist/vault-client.d.ts.map +1 -1
  156. package/dist/vault-client.js +284 -36
  157. package/dist/vault-client.js.map +1 -1
  158. package/dist/vault-client.test.js +59 -0
  159. package/dist/vault-client.test.js.map +1 -1
  160. package/dist/watcher.d.ts +38 -20
  161. package/dist/watcher.d.ts.map +1 -1
  162. package/dist/watcher.js +155 -143
  163. package/dist/watcher.js.map +1 -1
  164. package/dist/watcher.test.js +103 -0
  165. package/dist/watcher.test.js.map +1 -1
  166. package/package.json +1 -1
  167. package/src/bin/sync-runner-company.ts +350 -0
  168. package/src/bin/sync-runner-events.ts +25 -0
  169. package/src/bin/sync-runner-planning.ts +121 -0
  170. package/src/bin/sync-runner-rollup.ts +72 -0
  171. package/src/bin/sync-runner-telemetry.ts +8 -0
  172. package/src/bin/sync-runner-watch-loop.ts +443 -0
  173. package/src/bin/sync-runner-watch-routes.ts +86 -0
  174. package/src/bin/sync-runner.test.ts +298 -11
  175. package/src/bin/sync-runner.ts +99 -1054
  176. package/src/cli/reindex.test.ts +41 -3
  177. package/src/cli/reindex.ts +35 -19
  178. package/src/cli/rescue-classify-ordering.test.ts +81 -0
  179. package/src/cli/rescue-core.ts +400 -165
  180. package/src/cli/share.test.ts +38 -0
  181. package/src/cli/share.ts +420 -693
  182. package/src/cli/sync.test.ts +460 -1
  183. package/src/cli/sync.ts +788 -825
  184. package/src/cognito-auth.test.ts +77 -0
  185. package/src/cognito-auth.ts +73 -11
  186. package/src/daemon-worker.ts +3 -3
  187. package/src/index.ts +8 -0
  188. package/src/journal.test.ts +72 -0
  189. package/src/journal.ts +95 -8
  190. package/src/machine-auth.test.ts +64 -2
  191. package/src/object-io.test.ts +142 -0
  192. package/src/object-io.ts +183 -31
  193. package/src/operation-lock.test.ts +63 -4
  194. package/src/operation-lock.ts +99 -31
  195. package/src/personal-vault.test.ts +42 -0
  196. package/src/personal-vault.ts +8 -2
  197. package/src/prefix-coalesce.test.ts +71 -1
  198. package/src/prefix-coalesce.ts +155 -30
  199. package/src/remote-pull.test.ts +235 -1
  200. package/src/remote-pull.ts +106 -18
  201. package/src/s3.test.ts +126 -0
  202. package/src/s3.ts +237 -122
  203. package/src/scope-shrink.ts +6 -3
  204. package/src/skill-telemetry.test.ts +109 -0
  205. package/src/skill-telemetry.ts +82 -14
  206. package/src/sync/event-sync.test.ts +75 -0
  207. package/src/sync/event-sync.ts +54 -1
  208. package/src/sync/metrics.test.ts +81 -0
  209. package/src/sync/metrics.ts +59 -4
  210. package/src/sync/pull-scope.ts +23 -7
  211. package/src/sync/push-receiver.test.ts +73 -1
  212. package/src/sync/push-receiver.ts +56 -20
  213. package/src/sync-core.ts +58 -0
  214. package/src/telemetry.test.ts +85 -0
  215. package/src/telemetry.ts +69 -6
  216. package/src/types.ts +8 -0
  217. package/src/vault-client.test.ts +74 -0
  218. package/src/vault-client.ts +395 -43
  219. package/src/watcher.test.ts +117 -0
  220. package/src/watcher.ts +215 -174
@@ -71,13 +71,15 @@ import { collectAndSendTelemetry } from "../telemetry.js";
71
71
  import { collectAndSendSkillTelemetry } from "../skill-telemetry.js";
72
72
  import { reindexAfterSync } from "../qmd-reindex.js";
73
73
  import { pruneConflictIndex } from "../lib/conflict-index.js";
74
- import { withOperationLock, OperationLockedError, OPERATION_LOCKED_EXIT, } from "../operation-lock.js";
75
- import { describeError } from "../lib/describe-error.js";
76
74
  import { getOrCreateMachineId } from "../lib/machine-id.js";
77
- import { TreeWatcher, WatchPushDriver, systemClock, } from "../watcher.js";
78
- import { NoopPushReceiver, } from "../sync/push-receiver.js";
79
- import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
80
- import { PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
75
+ import { migratePersonalVaultJournal } from "../journal.js";
76
+ import { createRunnerEmitter } from "./sync-runner-events.js";
77
+ import { buildFanoutPlan, emitFanoutPlan, resolveMembershipsForRun, } from "./sync-runner-planning.js";
78
+ import { executeCompanyFanout } from "./sync-runner-company.js";
79
+ import { rollupAllComplete } from "./sync-runner-rollup.js";
80
+ import { emitTelemetry } from "./sync-runner-telemetry.js";
81
+ import { runOneShotWithOperationLock, runWatchLoop, } from "./sync-runner-watch-loop.js";
82
+ export { buildTargetedPullArgv, buildTargetedPushArgv, routeChangeToTarget, } from "./sync-runner-watch-routes.js";
81
83
  // ---------------------------------------------------------------------------
82
84
  // Defaults — mirror `hq-cli/src/utils/cognito-session.ts`. Inlined (not
83
85
  // imported) to avoid a circular dep between hq-cli and hq-cloud. If these
@@ -528,14 +530,7 @@ export async function runRunner(argv, deps = {}) {
528
530
  // events. The menubar's `HQ_CLOUD_VERSION` pin gates which runner
529
531
  // they spawn, so old menubars stay on the previous runner version
530
532
  // even after this one is published.
531
- const ERROR_TYPES = new Set([
532
- "error",
533
- "auth-error",
534
- ]);
535
- const emit = (event) => {
536
- const stream = ERROR_TYPES.has(event.type) ? stderr : stdout;
537
- stream.write(`${JSON.stringify(event)}\n`);
538
- };
533
+ const emit = createRunnerEmitter({ stdout, stderr });
539
534
  // ---- argv -------------------------------------------------------------
540
535
  const parsed = parseArgs(argv);
541
536
  if ("error" in parsed) {
@@ -619,37 +614,21 @@ export async function runRunner(argv, deps = {}) {
619
614
  // ---- resolve targets --------------------------------------------------
620
615
  let memberships;
621
616
  try {
622
- if (parsed.personal) {
623
- // Personal-vault-only mode: skip listMyMemberships entirely (and
624
- // therefore the claim-dance). The fanout plan is built solely from
625
- // the person-entity lookup below — no cloud-company targets, no
626
- // /membership/me round-trip. setup-needed is deferred to the
627
- // person-entity check (firing only when the personal entity is
628
- // also absent, not when memberships are empty by design).
629
- memberships = [];
630
- }
631
- else if (parsed.companies) {
632
- // Before giving up on memberships, run the claim-dance: new users signed
633
- // in via the tray may have email-keyed invites waiting for them. Without
634
- // this, an invited user would see "setup-needed" on every tray click.
635
- if (claims) {
636
- await runClaimDance(client, claims, stderr);
637
- }
638
- memberships = await listMembershipsWithRetry(client);
639
- if (memberships.length === 0) {
640
- // Truly empty — still a valid state (no memberships = nothing to
641
- // sync). The tray will show a friendly "create your first company"
642
- // CTA rather than an alarm banner.
643
- emit({ type: "setup-needed" });
644
- return 0;
645
- }
646
- }
647
- else {
648
- // Single-company mode: fabricate a minimal membership so the fanout
649
- // loop below treats it uniformly. We don't need to hit
650
- // /membership/me — the caller already told us which company.
651
- memberships = [{ companyUid: parsed.company }];
617
+ const resolution = await resolveMembershipsForRun({
618
+ personal: parsed.personal,
619
+ companies: parsed.companies,
620
+ company: parsed.company,
621
+ client,
622
+ claims,
623
+ stderr,
624
+ runClaimDance,
625
+ listMemberships: listMembershipsWithRetry,
626
+ });
627
+ if (resolution.status === "setup-needed") {
628
+ emit({ type: "setup-needed" });
629
+ return 0;
652
630
  }
631
+ memberships = resolution.memberships;
653
632
  }
654
633
  catch (err) {
655
634
  if (err instanceof VaultAuthError) {
@@ -669,64 +648,20 @@ export async function runRunner(argv, deps = {}) {
669
648
  });
670
649
  return 1;
671
650
  }
672
- // ---- resolve slugs for the fanout plan --------------------------------
673
- // The menubar wants "Syncing indigo" in its UI, not the raw cmp_* ULID.
674
- // If the entity fetch fails for some row (entity deleted, scoping issue),
675
- // degrade to using the UID as the slug rather than aborting the run.
676
- const plan = [];
677
- for (const m of memberships) {
678
- let slug = m.companyUid;
679
- let name;
680
- try {
681
- const info = await client.entity.get(m.companyUid);
682
- slug = info.slug || m.companyUid;
683
- name = info.name;
684
- }
685
- catch {
686
- // Best-effort — keep UID as the display identifier.
687
- }
688
- plan.push({ uid: m.companyUid, slug, ...(name ? { name } : {}) });
689
- }
690
- if ((parsed.companies || parsed.personal) &&
691
- !resolveSkipPersonal(parsed.skipPersonal)) {
692
- // Personal-target fanout slot. Skipped entirely when --skip-personal
693
- // (or HQ_SYNC_SKIP_PERSONAL=1) is set — see resolveSkipPersonal doc for
694
- // the rationale (menubar opt-out for users who only want company sync).
695
- // When skipped, the fanout-plan event below carries only company
696
- // memberships and no "personal" slug; downstream consumers (menubar
697
- // workspaces row, status surfaces) should already tolerate that
698
- // shape since pre-5.25 fanout often had it (a user with no person
699
- // entity yet, or before the canonical-person-entity machinery landed).
700
- //
701
- // `--personal` mode reaches this block with an empty `plan` and
702
- // empty `memberships`; only the personal target gets added below.
703
- const persons = await client.entity.listByType("person");
704
- const pick = pickCanonicalPersonEntity(persons);
705
- if (pick?.bucketName) {
706
- plan.push({
707
- slug: "personal",
708
- uid: pick.uid,
709
- bucketName: pick.bucketName,
710
- personalMode: true,
711
- // Reserved sentinel slug — decoupled from the `companies/personal`
712
- // company slug to avoid sharing `sync-journal.personal.json`. The
713
- // display `slug` stays "personal" (menubar UI); only the journal
714
- // shard changes. See PERSONAL_VAULT_JOURNAL_SLUG for the full
715
- // root-cause writeup.
716
- journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
717
- });
718
- }
719
- else if (parsed.personal) {
720
- // --personal mode with no canonical personal entity → setup-needed.
721
- // (In --companies mode this state is silent — companies still sync
722
- // and the missing personal target just shows an empty workspaces row.
723
- // In --personal mode it's the ONLY signal, so it gets surfaced as
724
- // setup-needed with the same shape as the empty-memberships case.)
725
- emit({ type: "setup-needed" });
726
- return 0;
727
- }
651
+ const targetPlan = await buildFanoutPlan({
652
+ memberships,
653
+ companies: parsed.companies,
654
+ personal: parsed.personal,
655
+ skipPersonal: parsed.skipPersonal,
656
+ client,
657
+ resolveSkipPersonal,
658
+ });
659
+ if (targetPlan.status === "setup-needed") {
660
+ emit({ type: "setup-needed" });
661
+ return 0;
728
662
  }
729
- emit({ type: "fanout-plan", companies: plan });
663
+ const plan = targetPlan.plan;
664
+ emitFanoutPlan(emit, plan);
730
665
  // One-time seed of the reserved personal-vault journal from the legacy
731
666
  // "personal" journal, so switching the vault slot off the colliding slug
732
667
  // does NOT trigger a mass re-upload of the HQ overlay on first run. No-op
@@ -736,495 +671,39 @@ export async function runRunner(argv, deps = {}) {
736
671
  // ---- fanout -----------------------------------------------------------
737
672
  const syncFn = deps.sync ?? defaultSync;
738
673
  const shareFn = deps.share ?? defaultShare;
739
- const doPush = parsed.direction === "push" || parsed.direction === "both";
740
- const doPull = parsed.direction === "pull" || parsed.direction === "both";
741
- const errors = [];
742
- const allConflicts = [];
743
- const stateByCompany = new Map();
744
- for (const target of plan) {
745
- const companyLabel = target.slug;
746
- const state = {
747
- company: companyLabel,
748
- // Default to "errored" so a throw before any complete-or-clean-abort
749
- // path (the original bug) leaves the entry flagged as not-clean. The
750
- // success/clean-abort paths overwrite this before the loop body exits.
751
- status: "errored",
752
- filesDownloaded: 0,
753
- bytesDownloaded: 0,
754
- filesUploaded: 0,
755
- bytesUploaded: 0,
756
- };
757
- stateByCompany.set(companyLabel, state);
758
- // Which phase is currently emitting `progress` events. Mutable closure so
759
- // tagAndEmit (defined once below) reads the latest value when each event
760
- // fires. "pull" is the default for back-compat with pull-only runs.
761
- let activePhase = doPush && !doPull ? "push" : "pull";
762
- // Per-company event tagger — shared by push and pull phases so progress
763
- // rows land on the right company regardless of which phase emitted them.
764
- // Also updates `state` for `progress` events so the rollup has accurate
765
- // partial counts even if the sync function throws before returning.
766
- const tagAndEmit = (event) => {
767
- if (event.type === "plan") {
768
- emit({
769
- type: "plan",
770
- company: companyLabel,
771
- filesToDownload: event.filesToDownload,
772
- bytesToDownload: event.bytesToDownload,
773
- filesToUpload: event.filesToUpload,
774
- bytesToUpload: event.bytesToUpload,
775
- filesToSkip: event.filesToSkip,
776
- filesToConflict: event.filesToConflict,
777
- filesToDelete: event.filesToDelete,
778
- });
779
- }
780
- else if (event.type === "progress") {
781
- if (activePhase === "push") {
782
- state.filesUploaded += 1;
783
- state.bytesUploaded += event.bytes;
784
- }
785
- else {
786
- state.filesDownloaded += 1;
787
- state.bytesDownloaded += event.bytes;
788
- }
789
- emit({
790
- type: "progress",
791
- company: companyLabel,
792
- path: event.path,
793
- bytes: event.bytes,
794
- // Stamp the transfer direction from the in-flight phase so the
795
- // menubar can label each file uploaded vs downloaded. The inner
796
- // share()/sync() emitters don't know the phase — only this tagger,
797
- // which mirrors the up/down counter bump above.
798
- direction: activePhase === "push" ? "up" : "down",
799
- ...(event.message ? { message: event.message } : {}),
800
- ...(event.deleted ? { deleted: event.deleted } : {}),
801
- });
802
- }
803
- else if (event.type === "conflict") {
804
- emit({
805
- type: "conflict",
806
- company: companyLabel,
807
- path: event.path,
808
- direction: event.direction,
809
- resolution: event.resolution,
810
- });
811
- }
812
- else if (event.type === "error") {
813
- emit({
814
- type: "error",
815
- company: companyLabel,
816
- path: event.path,
817
- message: event.message,
818
- });
819
- }
820
- else if (event.type === "new-files") {
821
- emit({
822
- type: "new-files",
823
- company: companyLabel,
824
- files: event.files,
825
- });
826
- }
827
- else if (event.type === "scope-excluded") {
828
- // Push-side ACL scope exclusions — surface the named paths tagged to
829
- // this company so the menubar/CLI can show "N skipped, outside your
830
- // access" instead of the file silently never uploading.
831
- emit({
832
- type: "scope-excluded",
833
- company: companyLabel,
834
- count: event.count,
835
- samplePaths: event.samplePaths,
836
- });
837
- }
838
- };
839
- try {
840
- let pushResult = {
841
- filesUploaded: 0,
842
- bytesUploaded: 0,
843
- filesSkipped: 0,
844
- filesDeleted: 0,
845
- filesTombstoned: 0,
846
- filesRefusedStale: 0,
847
- filesRefusedStalePaths: [],
848
- filesSuppressedByTombstone: 0,
849
- filesExcludedByPolicy: 0,
850
- filesExcludedByScope: 0,
851
- conflictPaths: [],
852
- aborted: false,
853
- };
854
- let pullResult = {
855
- filesDownloaded: 0,
856
- bytesDownloaded: 0,
857
- filesSkipped: 0,
858
- conflicts: 0,
859
- conflictPaths: [],
860
- aborted: false,
861
- newFiles: [],
862
- newFilesCount: 0,
863
- filesExcludedByPolicy: 0,
864
- filesTombstoned: 0,
865
- filesOutOfScope: 0,
866
- scopeOrphansRemoved: 0,
867
- scopeOrphansBlocked: 0,
868
- };
869
- // Push first so a subsequent pull doesn't overwrite files we were about
870
- // to broadcast. Company targets walk `companies/{slug}/`; the personal
871
- // target walks every top-level entry under hqRoot minus the exclusion
872
- // list (see PERSONAL_VAULT_EXCLUDED_TOP_LEVEL). `skipUnchanged: true`
873
- // keeps both cases efficient on re-runs.
874
- // Shared between push and pull for the personal slot. Hoisted out of
875
- // the if-blocks below so doPush + doPull see the same set:
876
- //
877
- // `HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL=1` opts INTO the
878
- // cloud:false → personal-bucket fallback. Default OFF keeps the
879
- // personal vault identical to the pre-5.20 shape until operators
880
- // explicitly enable it.
881
- //
882
- // `teamSyncedSlugs` is the slug set the operator currently has
883
- // active team-bucket Memberships for, derived from the live plan.
884
- // Used by:
885
- // • push: filter `companies/` subdir enumeration so a team-synced
886
- // company never rides the personal slot; build
887
- // `decommissionPrefixes` so share() sweeps orphan keys for
888
- // any slug that's now team-backed.
889
- // • pull: filter `listRemoteFiles` so pre-decommission orphan
890
- // keys at `companies/{team-synced-slug}/...` don't get
891
- // re-downloaded into the disk paths the team-bucket pull
892
- // now manages.
893
- const includeLocalCompanies = process.env.HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL === "1";
894
- const teamSyncedSlugs = new Set(plan
895
- .filter((p) => p.personalMode !== true)
896
- .map((p) => p.slug));
897
- // Resolve the membership's effective ACL scope ONCE so BOTH the push and
898
- // pull legs respect the granted prefixes. The vault vends a child
899
- // credential scoped to these prefixes; without filtering the PUSH plan to
900
- // them, share() would HEAD/PUT keys outside the grant and the server's
901
- // correct 403 (SCOPE_EXCEEDS_PARENT) would abort the WHOLE company with a
902
- // generic error + exit 2. Personal-vault legs have no membership
903
- // sync-config — they stay full-scope ("all"). Degrades to "all" on any
904
- // error (a transient failure must never silently filter/prune the tree).
905
- // Hoisted above the push block (it used to be resolved only for pull) so
906
- // push gets the same scope; the pull leg below reuses this value.
907
- const scope = target.personalMode === true
908
- ? { syncMode: "all" }
909
- : await resolvePullScope(client, target.uid, target.slug, parsed.hqRoot);
910
- if (doPush) {
911
- activePhase = "push";
912
- // For the personal slot we hand share() both (a) the top-level
913
- // hqRoot entries that are part of the personal vault and (b) any
914
- // `companies/{slug}/` subdirs the user has opted into the personal
915
- // bucket via `cloud: false` in `company.yaml`. Slugs that already
916
- // own a team bucket (anything else in `plan`) are excluded so a
917
- // company is never double-written.
918
- const pushPaths = target.personalMode === true
919
- ? computePersonalVaultPaths(parsed.hqRoot, {
920
- includeLocalCompanies,
921
- teamSyncedSlugs,
922
- })
923
- : [path.join(parsed.hqRoot, "companies", target.slug)];
924
- // For the personal slot, hand share() a list of `companies/{slug}/`
925
- // prefixes for every team-synced slug. If the personal-bucket
926
- // journal still holds entries under any of those prefixes — i.e.
927
- // the company used to ride the personal-bucket fallback and was
928
- // since promoted to its own team bucket — share() will sweep
929
- // those orphans via DeleteObject + journal removal. Unconditional
930
- // (not gated on `HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL`) so cleanup
931
- // still runs after an operator turns the feature off: the on-ramp
932
- // is opt-in but the off-ramp is always safe to traverse.
933
- const decommissionPrefixes = target.personalMode === true
934
- ? Array.from(teamSyncedSlugs).map((slug) => `companies/${slug}`)
935
- : undefined;
936
- pushResult = await shareFn({
937
- paths: pushPaths,
938
- company: target.uid,
939
- vaultConfig,
940
- hqRoot: parsed.hqRoot,
941
- onConflict: parsed.onConflict,
942
- skipUnchanged: true,
943
- // Local deletes propagate to S3 as soft deletes (versioning is on
944
- // — DeleteObject writes a delete-marker, prior versions remain
945
- // recoverable). Without this, a deleted file resurfaces on the
946
- // next pull because the remote object is still listable.
947
- //
948
- // Policy default in 5.24 is `owned-only` (pre-5.24 behavior;
949
- // preserved for the soak window). `HQ_SYNC_DELETE_POLICY` env
950
- // can opt INTO the safer `currency-gated` (per-file HEAD + ETag
951
- // verification) or the unsafe `all` (emergency reconcile only).
952
- // Default flips to `currency-gated` in 5.25 after at least one
953
- // machine has soaked the new path. Both personal and company
954
- // targets use the same resolver — same engine, same flip.
955
- propagateDeletes: true,
956
- propagateDeletePolicy: resolveDeletePolicy(),
957
- onEvent: tagAndEmit,
958
- ...(uploadAuthor ? { author: uploadAuthor } : {}),
959
- // Mirror the pull-side seam: only spread these for the personal
960
- // slot so company-target args stay identical to the pre-Slice-2
961
- // shape (the "no personalMode/journalSlug keys" regression test
962
- // in sync-runner.test.ts pins that contract).
963
- ...(target.personalMode !== undefined ? { personalMode: target.personalMode } : {}),
964
- ...(target.journalSlug !== undefined ? { journalSlug: target.journalSlug } : {}),
965
- ...(decommissionPrefixes && decommissionPrefixes.length > 0
966
- ? { decommissionPrefixes }
967
- : {}),
968
- // US-005 symmetry: scope the PUSH plan to the membership's granted
969
- // ACL prefixes so out-of-scope keys are skipped (and surfaced via a
970
- // `scope-excluded` event) instead of drawing the server's 403 and
971
- // aborting the company. `undefined` for `syncMode: "all"` (owner /
972
- // personal) → no scope filter, identical to the pre-fix shape so the
973
- // "company-target args" contract test stays green.
974
- ...(scope.prefixSet !== undefined ? { prefixSet: scope.prefixSet } : {}),
975
- });
976
- }
977
- // Pull runs unless the push phase aborted on conflict — aborted means
978
- // the user has local edits + remote drift; blindly pulling would erase
979
- // whichever side `--on-conflict abort` just protected.
980
- if (doPull && !pushResult.aborted) {
981
- activePhase = "pull";
982
- // US-005: the pull only materializes in-scope keys (and prunes clean
983
- // orphans when scope shrank). Reuse the `scope` resolved once above so
984
- // push and pull apply the SAME granted prefixes and we avoid a second
985
- // `listMyExplicitGrants` round-trip per company.
986
- const pullScope = scope;
987
- pullResult = await syncFn({
988
- company: target.uid,
989
- vaultConfig,
990
- hqRoot: parsed.hqRoot,
991
- onConflict: parsed.onConflict,
992
- syncMode: pullScope.syncMode,
993
- // The menubar runner can take no interactive flag, so a scope shrink
994
- // must NEVER throw here (the old `ScopeShrinkBlockedError` → exit 2
995
- // was the permanent wedge in DEV-1768). Self-heal non-destructively:
996
- // dirty out-of-scope files stay on disk + un-tracked, clean ones are
997
- // quarantined (recoverable). This also clears an already-wedged
998
- // journal — seeded by a buggy `all`-mode CLI pull — on the next sync.
999
- scopeShrinkPolicy: "auto-recover",
1000
- // Scope-shrink authorship guard: pass the caller's own sub (the very
1001
- // sub stamped onto uploads as `created-by-sub`) so a scope shrink
1002
- // never prunes content this owner authored. Owners hold their whole
1003
- // vault by role-bypass, so without this a `shared`/`custom` pull
1004
- // would treat their own un-granted work as a foreign orphan.
1005
- ...(uploadAuthor?.userSub !== undefined
1006
- ? { callerSub: uploadAuthor.userSub }
1007
- : {}),
1008
- ...(pullScope.prefixSet !== undefined
1009
- ? { prefixSet: pullScope.prefixSet }
1010
- : {}),
1011
- ...(target.personalMode !== undefined ? { personalMode: target.personalMode } : {}),
1012
- ...(target.journalSlug !== undefined ? { journalSlug: target.journalSlug } : {}),
1013
- // Symmetric to the push side: for the personal slot, tell sync()
1014
- // how to interpret `companies/{slug}/...` keys in the personal
1015
- // bucket. With `includeLocalCompanies: true`, keys for slugs NOT
1016
- // in `teamSyncedSlugs` are downloaded (legitimate cloud:false
1017
- // opt-in content from another machine); keys for slugs IN that
1018
- // set are dropped as orphans (pre-decommission residue from a
1019
- // promoted company). With `includeLocalCompanies: false` the
1020
- // legacy pre-5.20 behavior holds: all `companies/...` keys are
1021
- // dropped. Only spread for the personal slot so company-target
1022
- // args stay identical to pre-Slice-2 shape (test C pin).
1023
- ...(target.personalMode === true
1024
- ? {
1025
- includeLocalCompanies,
1026
- teamSyncedSlugs,
1027
- }
1028
- : {}),
1029
- onEvent: tagAndEmit,
1030
- });
1031
- }
1032
- // Concat push + pull conflict paths into a single per-company list,
1033
- // then dedupe — a key that legitimately conflicts on both halves of
1034
- // a bidirectional run (e.g. `.hq/install-manifest.json` round-trip
1035
- // before Fix #1) appears twice in the concat but represents a single
1036
- // logical conflict for the operator. Bug #3 in the 5.33.0 deep-test:
1037
- // every round produced `conflictPaths: [X, X]` and `conflicts: 2`,
1038
- // double-counting the conflict in every metric downstream. The push
1039
- // and pull halves each emit a `conflict` event in their own direction
1040
- // (preserved on the event stream for tracing); only the merged
1041
- // result list is collapsed. Stable first-seen order is preserved so
1042
- // consumers can rely on the pull entry coming before its push twin.
1043
- const seenConflictPaths = new Set();
1044
- const mergedConflictPaths = [];
1045
- for (const p of [...pullResult.conflictPaths, ...pushResult.conflictPaths]) {
1046
- if (seenConflictPaths.has(p))
1047
- continue;
1048
- seenConflictPaths.add(p);
1049
- mergedConflictPaths.push(p);
1050
- }
1051
- const aborted = pullResult.aborted || pushResult.aborted;
1052
- // Overwrite the progress-derived counts with the authoritative numbers
1053
- // from the sync/share return values. The `progress` stream over-counts
1054
- // when the inner walker emits a progress row for a file it then skips
1055
- // due to a journal hit — a clean return value is the source of truth.
1056
- // For the throw case below this overwrite never runs, so `state` keeps
1057
- // its progress-derived counts (which is exactly what we want there).
1058
- state.filesDownloaded = pullResult.filesDownloaded;
1059
- state.bytesDownloaded = pullResult.bytesDownloaded;
1060
- state.filesUploaded = pushResult.filesUploaded;
1061
- state.bytesUploaded = pushResult.bytesUploaded;
1062
- state.status = aborted ? "aborted" : "complete";
1063
- emit({
1064
- type: "complete",
1065
- company: companyLabel,
1066
- filesDownloaded: pullResult.filesDownloaded,
1067
- bytesDownloaded: pullResult.bytesDownloaded,
1068
- filesUploaded: pushResult.filesUploaded,
1069
- bytesUploaded: pushResult.bytesUploaded,
1070
- filesSkipped: pullResult.filesSkipped + pushResult.filesSkipped,
1071
- // Push-side counters surfaced on `complete` so the menubar's
1072
- // `SyncCompleteEvent` (which carries them as Option<u32> for
1073
- // back-compat with pre-5.25 engines) can render the new totals.
1074
- // Always emitted as numbers (0 when no push leg ran) so Rust's
1075
- // serde decodes them as `Some(0)` rather than `None` — distinct
1076
- // from the legacy-engine `None` and useful when the UI wants to
1077
- // distinguish "engine ran, nothing tombstoned" from "engine
1078
- // didn't report".
1079
- // Tombstones now flow on both legs:
1080
- // - push side: `ShareResult.filesTombstoned` (remote was already
1081
- // 404 at HEAD time, journal entry dropped).
1082
- // - pull side: `SyncResult.filesTombstoned` (Bug #9 — journal-
1083
- // known key missing from remote LIST, applied as local delete).
1084
- // Sum them so the menubar's `SyncCompleteEvent` reflects the total
1085
- // delete-propagation activity for that company across the run.
1086
- filesTombstoned: pushResult.filesTombstoned + pullResult.filesTombstoned,
1087
- filesRefusedStale: pushResult.filesRefusedStale,
1088
- // Bonus diagnostic: surface the paths so operators can triage the
1089
- // recurring `filesRefusedStale: N` signal — the count alone was
1090
- // untriageable per the 5.33.0 deep-test report's "205 issue".
1091
- // Pre-capped at 50 by share() itself.
1092
- filesRefusedStalePaths: pushResult.filesRefusedStalePaths,
1093
- // Pull side now reports an `filesExcludedByPolicy` too (Bug #2 —
1094
- // ephemeral conflict-mirror refusals in the pull walker). Sum
1095
- // both legs so the `complete` event reports total excluded across
1096
- // the full bidirectional pass; pre-fix the pull half silently
1097
- // pushed legacy `.conflict-*` litter into clean trees with the
1098
- // same counter showing 0.
1099
- filesExcludedByPolicy: pushResult.filesExcludedByPolicy + pullResult.filesExcludedByPolicy,
1100
- // Sourced from the merged path list so push-side conflicts are
1101
- // counted too — `ShareResult` doesn't expose a numeric counter,
1102
- // and using `pullResult.conflicts` alone silently dropped any
1103
- // push conflict from the count while leaving its path in
1104
- // `conflictPaths`.
1105
- conflicts: mergedConflictPaths.length,
1106
- conflictPaths: mergedConflictPaths,
1107
- // Either phase aborting marks the company aborted — the UI treats
1108
- // `aborted: true` as "sync didn't complete cleanly for this company".
1109
- aborted,
1110
- newFiles: pullResult.newFiles,
1111
- newFilesCount: pullResult.newFilesCount,
1112
- // Scope-aware download counters (US-005). Pull-only — the push leg
1113
- // has no scope concept — so they pass through from `pullResult`.
1114
- filesOutOfScope: pullResult.filesOutOfScope,
1115
- scopeOrphansRemoved: pullResult.scopeOrphansRemoved,
1116
- scopeOrphansBlocked: pullResult.scopeOrphansBlocked,
1117
- });
1118
- for (const p of pullResult.conflictPaths) {
1119
- allConflicts.push({ company: companyLabel, path: p, direction: "pull" });
1120
- }
1121
- for (const p of pushResult.conflictPaths) {
1122
- allConflicts.push({ company: companyLabel, path: p, direction: "push" });
1123
- }
1124
- }
1125
- catch (err) {
1126
- // describeError walks the cause chain so AWS SDK v3's "UnknownError"
1127
- // wrapper surfaces the underlying Node networking error (ENOTFOUND,
1128
- // ECONNRESET, …) instead of an unactionable bare "UnknownError".
1129
- const message = describeError(err);
1130
- errors.push({ company: companyLabel, message });
1131
- // `state.status` was seeded as "errored" at loop entry — the throw
1132
- // path leaves it there, and `state.files{Down,Up}loaded` reflects the
1133
- // partial counts captured from `progress` events before the throw.
1134
- // Emit a `complete` event with `aborted: true` and those partial
1135
- // counts so consumers walking the `complete` event stream see every
1136
- // company in the fanout uniformly. This is the fix for the misleading
1137
- // rollup — see file header `Exit code: 2` doc.
1138
- emit({
1139
- type: "complete",
1140
- company: companyLabel,
1141
- filesDownloaded: state.filesDownloaded,
1142
- bytesDownloaded: state.bytesDownloaded,
1143
- filesUploaded: state.filesUploaded,
1144
- bytesUploaded: state.bytesUploaded,
1145
- filesSkipped: 0,
1146
- // Mid-flight throw: we have no clean ShareResult to read these
1147
- // from. Report 0 so the event shape stays stable; the partial
1148
- // counts above already reflect what actually moved before the
1149
- // throw.
1150
- filesTombstoned: 0,
1151
- filesRefusedStale: 0,
1152
- filesRefusedStalePaths: [],
1153
- filesExcludedByPolicy: 0,
1154
- conflicts: 0,
1155
- conflictPaths: [],
1156
- aborted: true,
1157
- newFiles: [],
1158
- newFilesCount: 0,
1159
- // Mid-flight throw: no clean scope counts to report. 0 keeps the
1160
- // event shape stable (US-005).
1161
- filesOutOfScope: 0,
1162
- scopeOrphansRemoved: 0,
1163
- scopeOrphansBlocked: 0,
1164
- });
1165
- emit({
1166
- type: "error",
1167
- company: companyLabel,
1168
- path: "(company)",
1169
- message,
1170
- });
1171
- // Continue — one company's failure shouldn't abort the whole fanout.
1172
- }
1173
- }
1174
- // Walk every per-company entry — the map holds one row per planned company,
1175
- // including ones that aborted via thrown exception. This is the fix for the
1176
- // bug where `all-complete` reported `filesDownloaded: 0` for an aborted
1177
- // personal-sync that had already emitted thousands of `progress` events:
1178
- // the rollup used to only sum companies that emitted a clean `complete`,
1179
- // which silently dropped partials when the sync function threw.
1180
- let totalDownloaded = 0;
1181
- let totalDownloadedBytes = 0;
1182
- let totalUploaded = 0;
1183
- let totalUploadedBytes = 0;
1184
- let partial = false;
1185
- const companies = [];
1186
- for (const target of plan) {
1187
- const s = stateByCompany.get(target.slug);
1188
- if (!s)
1189
- continue; // unreachable — every plan entry seeds the map
1190
- totalDownloaded += s.filesDownloaded;
1191
- totalDownloadedBytes += s.bytesDownloaded;
1192
- totalUploaded += s.filesUploaded;
1193
- totalUploadedBytes += s.bytesUploaded;
1194
- if (s.status !== "complete")
1195
- partial = true;
1196
- companies.push({
1197
- company: s.company,
1198
- status: s.status,
1199
- filesDownloaded: s.filesDownloaded,
1200
- bytesDownloaded: s.bytesDownloaded,
1201
- filesUploaded: s.filesUploaded,
1202
- bytesUploaded: s.bytesUploaded,
1203
- });
1204
- }
674
+ const fanout = await executeCompanyFanout({
675
+ plan,
676
+ direction: parsed.direction,
677
+ hqRoot: parsed.hqRoot,
678
+ onConflict: parsed.onConflict,
679
+ client,
680
+ vaultConfig,
681
+ ...(uploadAuthor ? { uploadAuthor } : {}),
682
+ ...(deps.operationLockAlreadyHeld ? { operationLockAlreadyHeld: true } : {}),
683
+ syncFn,
684
+ shareFn,
685
+ resolveDeletePolicy,
686
+ emit,
687
+ });
688
+ const { errors, allConflicts } = fanout;
689
+ const rollup = rollupAllComplete(plan, fanout.stateByCompany);
1205
690
  // Fire telemetry collector before the all-complete emit so the cursor at
1206
691
  // `~/.hq/telemetry-cursor.json` is consistent with what the menubar sees.
1207
- // Awaited fully — fire-and-forget would lose in-flight POSTs at process
1208
- // exit, and the previous 10s race was wrong: a first-run user with
1209
- // backlog (e.g. 60K Claude session events) takes well over 10s legitimately,
1210
- // and the race silently dropped the entire batch when it fired. Errors
1211
- // are swallowed inside `defaultCollectTelemetry`; per-request timeouts
1212
- // come from `VaultClient`'s retry loop (3 attempts × exponential backoff),
1213
- // which naturally bounds the outer wait.
1214
- const telemetryFn = deps.collectTelemetry ??
1215
- (() => defaultCollectTelemetry(client, deps.createVaultClient !== undefined, parsed.hqRoot));
1216
- await telemetryFn().catch(() => undefined);
692
+ await emitTelemetry({
693
+ collectTelemetry: deps.collectTelemetry,
694
+ defaultCollectTelemetry: () => defaultCollectTelemetry(client, deps.createVaultClient !== undefined, parsed.hqRoot),
695
+ });
1217
696
  emit({
1218
697
  type: "all-complete",
1219
698
  companiesAttempted: plan.length,
1220
- filesDownloaded: totalDownloaded,
1221
- bytesDownloaded: totalDownloadedBytes,
1222
- filesUploaded: totalUploaded,
1223
- bytesUploaded: totalUploadedBytes,
699
+ filesDownloaded: rollup.totalDownloaded,
700
+ bytesDownloaded: rollup.totalDownloadedBytes,
701
+ filesUploaded: rollup.totalUploaded,
702
+ bytesUploaded: rollup.totalUploadedBytes,
1224
703
  conflictPaths: allConflicts,
1225
704
  errors,
1226
- partial,
1227
- companies,
705
+ partial: rollup.partial,
706
+ companies: rollup.companies,
1228
707
  });
1229
708
  // Post-sync qmd reindex — runs AFTER `all-complete` is emitted so the
1230
709
  // menubar/CLI already shows the sync as done; this is a best-effort tail
@@ -1232,7 +711,8 @@ export async function runRunner(argv, deps = {}) {
1232
711
  // pulled in (nothing to reindex otherwise) and not explicitly disabled.
1233
712
  // Self-contained: shells out to the global `qmd` binary, no dependency on
1234
713
  // any (possibly stale) script inside the synced HQ tree. See qmd-reindex.ts.
1235
- if (totalDownloaded > 0 && process.env.HQ_QMD_REINDEX_ON_SYNC !== "0") {
714
+ if (rollup.totalDownloaded > 0 &&
715
+ process.env.HQ_QMD_REINDEX_ON_SYNC !== "0") {
1236
716
  try {
1237
717
  reindexAfterSync(parsed.hqRoot);
1238
718
  }
@@ -1290,405 +770,23 @@ const isDirectInvocation = (() => {
1290
770
  return false;
1291
771
  }
1292
772
  })();
1293
- /**
1294
- * Route a changed relative path to the push target that owns it.
1295
- *
1296
- * - `companies/<slug>/...` → a single-company push for `<slug>` (the targeted
1297
- * pass per PRD decision — push only the changed company, not a full
1298
- * `--companies` fanout).
1299
- * - anything else under hqRoot → the personal target (a `--companies` push
1300
- * restricted to personal via the personal-vault scope; modeled here as the
1301
- * "personal" route the caller maps to the right argv).
1302
- *
1303
- * Returns `null` for paths the routing cannot attribute (defensive — the
1304
- * watcher's filter already drops excluded top-levels, so this is belt-and-
1305
- * suspenders for synthetic events).
1306
- */
1307
- export function routeChangeToTarget(relPath) {
1308
- const norm = relPath.split(path.sep).join("/").replace(/^\.\//, "");
1309
- if (norm === "" || norm.startsWith(".."))
1310
- return null;
1311
- const segments = norm.split("/").filter((s) => s.length > 0);
1312
- if (segments.length === 0)
1313
- return null;
1314
- if (segments[0] === "companies") {
1315
- // companies/<slug>/... — need at least the slug segment to target.
1316
- if (segments.length < 2 || segments[1].length === 0)
1317
- return null;
1318
- return { kind: "company", slug: segments[1] };
1319
- }
1320
- return { kind: "personal" };
1321
- }
1322
- /**
1323
- * Build the argv for a targeted push pass from a routed change. The push runs
1324
- * `--direction push` for just the routed target so a local edit propagates in
1325
- * seconds without a full fanout. Company routes use `--company <slug>`;
1326
- * personal routes use `--companies --direction push` (the personal-vault scope
1327
- * is resolved inside runRunner's fanout; skipUnchanged no-ops the company
1328
- * subtrees that didn't change). Inherits `--hq-root` / `--on-conflict` from
1329
- * the base argv. Pure helper, exported for unit testing the routing→argv map.
1330
- */
1331
- export function buildTargetedPushArgv(route, baseArgv) {
1332
- const carried = carriedFlags(baseArgv);
1333
- if (route.kind === "company") {
1334
- return ["--company", route.slug, "--direction", "push", ...carried];
1335
- }
1336
- return ["--companies", "--direction", "push", ...carried];
1337
- }
1338
- /**
1339
- * Build the argv for a targeted PULL pass from a routed change (US-009 — the
1340
- * receiver's pull-on-event path). Mirrors {@link buildTargetedPushArgv} but
1341
- * with `--direction pull`: a peer device pushed a change, so this device pulls
1342
- * just the affected company/subtree instead of waiting for the next
1343
- * `--poll-remote-ms` cycle. Company routes use `--company <slug>`; personal
1344
- * routes use `--companies` (the personal-vault scope is resolved inside
1345
- * runRunner's fanout; skipUnchanged no-ops the subtrees that didn't change).
1346
- * Inherits `--hq-root` / `--on-conflict` from the base argv. Pure helper,
1347
- * exported for unit testing the event→argv map.
1348
- */
1349
- export function buildTargetedPullArgv(route, baseArgv) {
1350
- const carried = carriedFlags(baseArgv);
1351
- if (route.kind === "company") {
1352
- return ["--company", route.slug, "--direction", "pull", ...carried];
1353
- }
1354
- return ["--companies", "--direction", "pull", ...carried];
1355
- }
1356
773
  export async function runRunnerWithLoop(argv, deps = {}) {
774
+ const parsed = parseArgs(argv);
775
+ const runtime = {
776
+ runPassWithOperationLockAlreadyHeld: (passArgv) => runRunner(passArgv, { operationLockAlreadyHeld: true }),
777
+ defaultGetIdTokenClaims,
778
+ defaultGetAccessToken: () => getValidAccessToken(DEFAULT_COGNITO, { interactive: false }),
779
+ apiUrl: DEFAULT_VAULT_API_URL,
780
+ region: DEFAULT_COGNITO.region,
781
+ };
1357
782
  if (!argv.includes("--watch")) {
1358
- // One-shot cloud sync — take the per-root operation lock so it is mutually
1359
- // exclusive with rescue/reindex. The `--watch` path below is the push
1360
- // watcher and is intentionally EXEMPT (it neither takes nor is blocked by
1361
- // the lock; its in-process targeted passes call `runRunner` directly, not
1362
- // through here). If args don't parse, fall through to `runRunner` so it
1363
- // surfaces the parse error rather than us masking it with a lock failure.
1364
- const parsed = parseArgs(argv);
1365
783
  if ("error" in parsed)
1366
784
  return runRunner(argv);
1367
- // The actual sync pass — same seam the watch loop uses (deps.runPass),
1368
- // so a test can assert "waits for a short-lived holder, THEN proceeds to
1369
- // sync" without touching the network. Production passes `argv` to
1370
- // runRunner exactly as before. Regression guard for DEV-1772
1371
- // (feedback_28a1833f): instant-sync one-shots used to exit 17 and die on
1372
- // a lock conflict with the ~1-min reindex hook; they now WAIT (default)
1373
- // and proceed once the short holder releases.
1374
- const runOnce = deps.runPass ?? ((passArgv) => runRunner(passArgv));
1375
- try {
1376
- return await withOperationLock(parsed.hqRoot, "sync", () => runOnce(argv), {
1377
- timeoutSec: parsed.lockTimeoutSec,
1378
- });
1379
- }
1380
- catch (err) {
1381
- if (err instanceof OperationLockedError) {
1382
- // The lock wait was BOUNDED and tripped (a holder never released
1383
- // within --lock-timeout / HQ_OP_LOCK_TIMEOUT). Surface it loudly on
1384
- // stderr and exit with the stable OPERATION_LOCKED_EXIT (17) so the
1385
- // spawner (menubar) can recognize a lock conflict and SCHEDULE A
1386
- // RETRY rather than treating it as a hard failure and silently giving
1387
- // up (DEV-1772). With the default (no bound) we never reach here — the
1388
- // one-shot waits indefinitely and proceeds.
1389
- process.stderr.write(err.message + "\n");
1390
- return OPERATION_LOCKED_EXIT;
1391
- }
1392
- throw err;
1393
- }
1394
- }
1395
- const sleep = deps.sleep ??
1396
- ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
1397
- const runPass = deps.runPass ?? ((passArgv) => runRunner(passArgv));
1398
- const pollIdx = argv.indexOf("--poll-remote-ms");
1399
- const pollMs = pollIdx >= 0 && argv[pollIdx + 1] ? Number(argv[pollIdx + 1]) : 600_000;
1400
- const eventPush = argv.includes("--event-push");
1401
- // In `--companies` mode the sync scope is companies/*/{knowledge,projects,…}
1402
- // (per .hqinclude), so the watcher must NOT apply the personal-vault
1403
- // top-level exclusions (PERSONAL_VAULT_EXCLUDED_TOP_LEVEL drops `companies/`
1404
- // and `workspace/`) — doing so would exclude exactly the paths being synced,
1405
- // and no local edit would ever trigger an instant push. The shared ignore
1406
- // stack (createIgnoreFilter / .hqignore / .hqinclude) already scopes the
1407
- // watch filter correctly in companies mode. personalMode is only for a
1408
- // personal-vault-as-root run, where companies/ et al. genuinely aren't synced.
1409
- const companiesMode = argv.includes("--companies");
1410
- const hqIdx = argv.indexOf("--hq-root");
1411
- const hqRoot = hqIdx >= 0 && argv[hqIdx + 1] ? argv[hqIdx + 1] : DEFAULT_HQ_ROOT;
1412
- // Strip the loop-only flags before delegating: the parser inside runRunner
1413
- // accepts --watch/--poll-remote-ms/--event-push, but we don't want a per-
1414
- // iteration pass to think it's re-entering watch mode.
1415
- const passArgv = argv.filter((a, i) => {
1416
- if (a === "--watch")
1417
- return false;
1418
- if (a === "--poll-remote-ms")
1419
- return false;
1420
- if (a === "--event-push")
1421
- return false;
1422
- if (i > 0 && argv[i - 1] === "--poll-remote-ms")
1423
- return false;
1424
- return true;
1425
- });
1426
- // ---- shared in-flight guard ------------------------------------------
1427
- // The poll loop AND watcher-triggered targeted pushes funnel through this
1428
- // mutex so a watcher push never overlaps an in-flight pass (PRD AC). A
1429
- // trigger that arrives while a pass runs is collapsed by WatchPushDriver's
1430
- // own pending-while-pushing logic, then re-armed after the pass settles.
1431
- let inFlight = false;
1432
- let stopped = false;
1433
- const runGuarded = async (pass) => {
1434
- if (inFlight)
1435
- return "skipped";
1436
- inFlight = true;
1437
- try {
1438
- return await pass();
1439
- }
1440
- finally {
1441
- inFlight = false;
1442
- }
1443
- };
1444
- // ---- event-push wiring (Phase 1) -------------------------------------
1445
- let watcher = null;
1446
- let driver = null;
1447
- let detachSignal = null;
1448
- let lastChangedRel = null;
1449
- let lastBatch = null;
1450
- // ---- pull-on-event receiver (Phase 2, US-009) ------------------------
1451
- // Started after the watcher, disposed before the watcher (mirror of the
1452
- // PushTransport ordering). Dormant by default: the default factory returns
1453
- // a NoopPushReceiver, and even a real receiver stays dormant unless the
1454
- // per-tenant feature flag is on AND a queue URL is provisioned server-side.
1455
- let receiver = null;
1456
- // ---- event-driven publish + pull (Phase 3, US-017/018/019) ------------
1457
- // Brought up asynchronously after the watcher when the rollout gate
1458
- // passes; null until ready (and stays null on startup failure → poll-only).
1459
- let eventSync = null;
1460
- if (eventPush) {
1461
- const clock = deps.clock ?? systemClock;
1462
- const debounceMs = 2000;
1463
- const createWatcher = deps.createWatcher ??
1464
- ((opts) => new TreeWatcher({
1465
- hqRoot: opts.hqRoot,
1466
- debounceMs: opts.debounceMs,
1467
- clock: opts.clock,
1468
- // false in --companies mode so the watch filter matches the sync
1469
- // scope (companies/* are included via .hqinclude); true only for a
1470
- // personal-vault-as-root run.
1471
- personalMode: !companiesMode,
1472
- }));
1473
- watcher = createWatcher({ hqRoot, debounceMs, clock });
1474
- // The driver runs the targeted push when a debounced burst settles. Its
1475
- // concurrency guard collapses a trigger that lands mid-pass; we ALSO gate
1476
- // through `runGuarded` so a poll pass in flight is never overlapped.
1477
- driver = new WatchPushDriver({
1478
- debounceMs: 0, // TreeWatcher already debounces; the driver just guards.
1479
- clock,
1480
- push: async () => {
1481
- if (stopped)
1482
- return;
1483
- const rel = lastChangedRel;
1484
- // Snapshot the settled batch BEFORE the await: a change landing
1485
- // mid-pass overwrites lastBatch for the NEXT pass, and this pass
1486
- // must only announce what it actually pushed.
1487
- const batchForPublish = lastBatch;
1488
- lastBatch = null;
1489
- const route = rel
1490
- ? routeChangeToTarget(rel)
1491
- : { kind: "personal" };
1492
- if (!route)
1493
- return;
1494
- const targetedArgv = buildTargetedPushArgv(route, passArgv);
1495
- const result = await runGuarded(() => runPass(targetedArgv));
1496
- // Phase 3 (US-017): publish PushEvents only AFTER the targeted push
1497
- // pass succeeded — an event must never announce bytes that are not
1498
- // in S3 yet. A skipped pass (guard held) or a failed pass publishes
1499
- // nothing; the cadence poll covers the miss. Fall back to a
1500
- // single-path batch when the watcher emitted a bare path signal.
1501
- if (result === 0 && eventSync) {
1502
- const batch = batchForPublish ??
1503
- (rel ? { paths: new Map([[path.join(hqRoot, rel), rel]]) } : null);
1504
- if (batch)
1505
- eventSync.publishBatch(batch);
1506
- }
1507
- },
1508
- });
1509
- // A debounced TreeWatcher 'changed' signal feeds the driver, which fires
1510
- // the targeted push after its own (zero) window — i.e. immediately, but
1511
- // still serialized behind any in-flight pass. A path-aware watcher passes
1512
- // the changed relative path so the push targets just its owning company;
1513
- // the bare-signal TreeWatcher leaves it null → personal-vault route.
1514
- watcher.onChange((changedRelPath, batch) => {
1515
- if (stopped)
1516
- return;
1517
- lastChangedRel = changedRelPath ?? null;
1518
- lastBatch = batch ?? null;
1519
- driver?.notifyChange();
1520
- });
1521
- watcher.start();
1522
- // Pull-on-event receiver (US-009). The injected SyncEngineFn bridges a
1523
- // received PushEvent → a TARGETED pull pass routed by relativePath, funneled
1524
- // through the SAME `runGuarded` mutex as the poll loop + watcher push so a
1525
- // pull-on-event never overlaps an in-flight pass. Started AFTER the watcher
1526
- // so a live event can't race a half-built daemon. Default factory = noop
1527
- // (dormant); a real SqsPushReceiver is injected by a later release once the
1528
- // server-side per-client SQS queue is provisioned.
1529
- const receiverSyncFn = async (ctx) => {
1530
- if (stopped)
1531
- return;
1532
- const route = routeChangeToTarget(ctx.event.relativePath);
1533
- if (!route)
1534
- return;
1535
- const targetedArgv = buildTargetedPullArgv(route, passArgv);
1536
- await runGuarded(() => runPass(targetedArgv));
1537
- };
1538
- const createReceiver = deps.createReceiver ?? (() => new NoopPushReceiver());
1539
- receiver = createReceiver({ syncFn: receiverSyncFn, hqRoot });
1540
- // Fire-and-forget start: a receiver's start() kicks off its own poll loop
1541
- // (SqsPushReceiver) or trivially flips connected (noop) — it must NOT block
1542
- // the runner's poll loop from entering. Errors are swallowed; the cadence
1543
- // poll is the safety net regardless of receiver health. (The await-free
1544
- // start also keeps the poll loop's microtask timing identical to the
1545
- // pre-US-009 wiring.)
1546
- void Promise.resolve(receiver.start()).catch(() => undefined);
1547
- // ---- Phase 3: event-driven publish + pull (US-017/018/019) ----------
1548
- // Gated to enrolled accounts (resolveEventSync — exact-email allowlist +
1549
- // HQ_SYNC_EVENT_SYNC override). Brought up asynchronously so a slow
1550
- // subscribe/vend can't delay the first poll pass; until (and unless) the
1551
- // handles resolve, behavior is byte-identical to the gate-off path.
1552
- const getClaims = deps.getIdTokenClaims ?? defaultGetIdTokenClaims;
1553
- const email = getClaims()?.email;
1554
- if (resolveEventSync(email, process.env.HQ_SYNC_EVENT_SYNC)) {
1555
- const getAccessToken = deps.getAccessToken ??
1556
- (() => getValidAccessToken(DEFAULT_COGNITO, { interactive: false }));
1557
- const startES = deps.startEventSync ?? defaultStartEventSync;
1558
- // Entirely async + caught: NOTHING in the Phase 3 bring-up (device-id
1559
- // persistence, tenant resolution, subscribe) may crash or delay the
1560
- // daemon — any failure degrades to poll-only.
1561
- void (async () => {
1562
- const handles = await startES({
1563
- hqRoot,
1564
- apiUrl: DEFAULT_VAULT_API_URL,
1565
- authToken: getAccessToken,
1566
- deviceId: getOrCreateMachineId(hqRoot),
1567
- // The server rejects publishes whose originTenantId mismatches the
1568
- // JWT principal, so resolve the SAME canonical person uid the vault
1569
- // API derives from this token.
1570
- resolveTenantId: async () => {
1571
- const client = new VaultClient({
1572
- apiUrl: DEFAULT_VAULT_API_URL,
1573
- authToken: getAccessToken,
1574
- region: DEFAULT_COGNITO.region,
1575
- });
1576
- const persons = await client.entity.listByType("person");
1577
- const pick = pickCanonicalPersonEntity(persons);
1578
- if (!pick?.uid) {
1579
- throw new Error("no canonical person entity for this account");
1580
- }
1581
- return pick.uid;
1582
- },
1583
- syncFn: receiverSyncFn,
1584
- log: (m) => process.stderr.write(`${m}\n`),
1585
- });
1586
- if (!handles)
1587
- return;
1588
- if (stopped) {
1589
- // Shutdown raced the async bring-up — tear straight down.
1590
- void handles.dispose();
1591
- return;
1592
- }
1593
- eventSync = handles;
1594
- })().catch((err) => {
1595
- process.stderr.write(`event-sync: wiring failed, continuing poll-only: ${describeError(err)}\n`);
1596
- });
1597
- }
1598
- }
1599
- // ---- clean shutdown --------------------------------------------------
1600
- // SIGTERM (menubar stop_daemon) / SIGINT must tear down BOTH the watcher and
1601
- // the poll loop with no leaked timers or fds. We flip `stopped`, dispose the
1602
- // watcher + driver, and let the poll loop observe `stopped` on its next tick.
1603
- let resolveStopped = null;
1604
- const stoppedSignal = new Promise((resolve) => {
1605
- resolveStopped = resolve;
1606
- });
1607
- const shutdown = () => {
1608
- if (stopped)
1609
- return;
1610
- stopped = true;
1611
- // Dispose the receiver FIRST (mirror of the PushTransport ordering:
1612
- // inbound subscription torn down before the watcher) so no new
1613
- // pull-on-event fires mid-teardown. dispose() is async (it drains the
1614
- // in-flight pull up to its own deadline); fire-and-forget here — the
1615
- // receiver's internal drain + the runGuarded mutex bound the work, and
1616
- // SIGTERM teardown must not block. Errors are swallowed.
1617
- try {
1618
- void receiver?.dispose();
1619
- }
1620
- catch {
1621
- /* ignore */
1622
- }
1623
- // Phase 3 wiring (publish transport + live receiver) — torn down with
1624
- // the same fire-and-forget posture as the Phase 2 receiver above.
1625
- try {
1626
- void eventSync?.dispose();
1627
- }
1628
- catch {
1629
- /* ignore */
1630
- }
1631
- eventSync = null;
1632
- try {
1633
- driver?.dispose();
1634
- }
1635
- catch {
1636
- /* ignore */
1637
- }
1638
- try {
1639
- watcher?.dispose();
1640
- }
1641
- catch {
1642
- /* ignore */
1643
- }
1644
- resolveStopped?.();
1645
- };
1646
- const onShutdownSignal = deps.onShutdownSignal ??
1647
- ((handler) => {
1648
- const wrapped = () => handler();
1649
- process.on("SIGTERM", wrapped);
1650
- process.on("SIGINT", wrapped);
1651
- return () => {
1652
- process.off("SIGTERM", wrapped);
1653
- process.off("SIGINT", wrapped);
1654
- };
1655
- });
1656
- detachSignal = onShutdownSignal(shutdown);
1657
- try {
1658
- while (!stopped) {
1659
- const result = await runGuarded(() => runPass(passArgv));
1660
- // A poll pass that was skipped because a watcher push held the guard is
1661
- // benign — the next iteration retries after the poll interval.
1662
- if (typeof result === "number" && result !== 0) {
1663
- return result;
1664
- }
1665
- // Sleep the poll interval, but wake early on shutdown so SIGTERM stops
1666
- // the loop promptly instead of waiting out a 10-minute cycle.
1667
- await Promise.race([sleep(pollMs), stoppedSignal]);
1668
- }
1669
- return 0;
1670
- }
1671
- finally {
1672
- shutdown();
1673
- detachSignal?.();
1674
- }
1675
- }
1676
- /**
1677
- * Extract the `--hq-root` / `--on-conflict` pair-flags from a base argv so a
1678
- * re-targeted push pass inherits the same root and conflict policy. Pure
1679
- * helper used by the event-push targeted-push composer.
1680
- */
1681
- function carriedFlags(baseArgv) {
1682
- const carried = [];
1683
- for (let i = 0; i < baseArgv.length; i++) {
1684
- const a = baseArgv[i];
1685
- if (a === "--hq-root" || a === "--on-conflict") {
1686
- carried.push(a);
1687
- if (baseArgv[i + 1] !== undefined)
1688
- carried.push(baseArgv[++i]);
1689
- }
785
+ return runOneShotWithOperationLock(argv, parsed, deps, runtime);
1690
786
  }
1691
- return carried;
787
+ if ("error" in parsed)
788
+ return runRunner(argv);
789
+ return runWatchLoop(argv, parsed, deps, runtime);
1692
790
  }
1693
791
  if (isDirectInvocation) {
1694
792
  runRunnerWithLoop(process.argv.slice(2))