@indigoai-us/hq-cloud 6.11.12 → 6.11.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/bin/sync-runner-company.d.ts +35 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -0
  3. package/dist/bin/sync-runner-company.js +290 -0
  4. package/dist/bin/sync-runner-company.js.map +1 -0
  5. package/dist/bin/sync-runner-events.d.ts +12 -0
  6. package/dist/bin/sync-runner-events.d.ts.map +1 -0
  7. package/dist/bin/sync-runner-events.js +12 -0
  8. package/dist/bin/sync-runner-events.js.map +1 -0
  9. package/dist/bin/sync-runner-planning.d.ts +53 -0
  10. package/dist/bin/sync-runner-planning.d.ts.map +1 -0
  11. package/dist/bin/sync-runner-planning.js +59 -0
  12. package/dist/bin/sync-runner-planning.js.map +1 -0
  13. package/dist/bin/sync-runner-rollup.d.ts +24 -0
  14. package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
  15. package/dist/bin/sync-runner-rollup.js +46 -0
  16. package/dist/bin/sync-runner-rollup.js.map +1 -0
  17. package/dist/bin/sync-runner-telemetry.d.ts +5 -0
  18. package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
  19. package/dist/bin/sync-runner-telemetry.js +5 -0
  20. package/dist/bin/sync-runner-telemetry.js.map +1 -0
  21. package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
  22. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
  23. package/dist/bin/sync-runner-watch-loop.js +372 -0
  24. package/dist/bin/sync-runner-watch-loop.js.map +1 -0
  25. package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
  26. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
  27. package/dist/bin/sync-runner-watch-routes.js +74 -0
  28. package/dist/bin/sync-runner-watch-routes.js.map +1 -0
  29. package/dist/bin/sync-runner.d.ts +3 -54
  30. package/dist/bin/sync-runner.d.ts.map +1 -1
  31. package/dist/bin/sync-runner.js +73 -1154
  32. package/dist/bin/sync-runner.js.map +1 -1
  33. package/dist/cli/reindex.d.ts.map +1 -1
  34. package/dist/cli/reindex.js +34 -17
  35. package/dist/cli/reindex.js.map +1 -1
  36. package/dist/cli/reindex.test.js +39 -5
  37. package/dist/cli/reindex.test.js.map +1 -1
  38. package/dist/cli/rescue-classify-ordering.test.js +17 -0
  39. package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
  40. package/dist/cli/rescue-core.d.ts +45 -0
  41. package/dist/cli/rescue-core.d.ts.map +1 -1
  42. package/dist/cli/rescue-core.js +197 -170
  43. package/dist/cli/rescue-core.js.map +1 -1
  44. package/dist/cli/share.d.ts.map +1 -1
  45. package/dist/cli/share.js +224 -676
  46. package/dist/cli/share.js.map +1 -1
  47. package/dist/cli/sync.d.ts.map +1 -1
  48. package/dist/cli/sync.js +399 -726
  49. package/dist/cli/sync.js.map +1 -1
  50. package/dist/cli/sync.test.js +20 -0
  51. package/dist/cli/sync.test.js.map +1 -1
  52. package/dist/daemon-worker.d.ts +2 -2
  53. package/dist/daemon-worker.js +3 -3
  54. package/dist/daemon-worker.js.map +1 -1
  55. package/dist/object-io.js +1 -1
  56. package/dist/object-io.js.map +1 -1
  57. package/dist/remote-pull.d.ts +2 -2
  58. package/dist/remote-pull.d.ts.map +1 -1
  59. package/dist/remote-pull.js +23 -3
  60. package/dist/remote-pull.js.map +1 -1
  61. package/dist/remote-pull.test.js +24 -2
  62. package/dist/remote-pull.test.js.map +1 -1
  63. package/dist/sync/push-receiver.d.ts +6 -0
  64. package/dist/sync/push-receiver.d.ts.map +1 -1
  65. package/dist/sync/push-receiver.js +32 -2
  66. package/dist/sync/push-receiver.js.map +1 -1
  67. package/dist/sync/push-receiver.test.js +31 -0
  68. package/dist/sync/push-receiver.test.js.map +1 -1
  69. package/dist/sync-core.d.ts +27 -0
  70. package/dist/sync-core.d.ts.map +1 -0
  71. package/dist/sync-core.js +54 -0
  72. package/dist/sync-core.js.map +1 -0
  73. package/dist/vault-client.d.ts.map +1 -1
  74. package/dist/vault-client.js +284 -36
  75. package/dist/vault-client.js.map +1 -1
  76. package/dist/vault-client.test.js +59 -0
  77. package/dist/vault-client.test.js.map +1 -1
  78. package/dist/watcher.d.ts +2 -20
  79. package/dist/watcher.d.ts.map +1 -1
  80. package/dist/watcher.js +3 -113
  81. package/dist/watcher.js.map +1 -1
  82. package/package.json +1 -1
  83. package/src/bin/sync-runner-company.ts +350 -0
  84. package/src/bin/sync-runner-events.ts +25 -0
  85. package/src/bin/sync-runner-planning.ts +121 -0
  86. package/src/bin/sync-runner-rollup.ts +72 -0
  87. package/src/bin/sync-runner-telemetry.ts +8 -0
  88. package/src/bin/sync-runner-watch-loop.ts +443 -0
  89. package/src/bin/sync-runner-watch-routes.ts +86 -0
  90. package/src/bin/sync-runner.ts +96 -1253
  91. package/src/cli/reindex.test.ts +41 -3
  92. package/src/cli/reindex.ts +35 -19
  93. package/src/cli/rescue-classify-ordering.test.ts +20 -0
  94. package/src/cli/rescue-core.ts +252 -176
  95. package/src/cli/share.ts +363 -705
  96. package/src/cli/sync.test.ts +25 -0
  97. package/src/cli/sync.ts +612 -802
  98. package/src/daemon-worker.ts +3 -3
  99. package/src/object-io.ts +1 -1
  100. package/src/remote-pull.test.ts +30 -1
  101. package/src/remote-pull.ts +29 -4
  102. package/src/sync/push-receiver.test.ts +35 -0
  103. package/src/sync/push-receiver.ts +41 -2
  104. package/src/sync-core.ts +58 -0
  105. package/src/vault-client.test.ts +74 -0
  106. package/src/vault-client.ts +395 -43
  107. package/src/watcher.ts +6 -141
@@ -71,13 +71,15 @@ import { collectAndSendTelemetry } from "../telemetry.js";
71
71
  import { collectAndSendSkillTelemetry } from "../skill-telemetry.js";
72
72
  import { reindexAfterSync } from "../qmd-reindex.js";
73
73
  import { pruneConflictIndex } from "../lib/conflict-index.js";
74
- import { acquireOperationLock, withOperationLock, OperationLockedError, OPERATION_LOCKED_EXIT, } from "../operation-lock.js";
75
- import { describeError } from "../lib/describe-error.js";
76
74
  import { getOrCreateMachineId } from "../lib/machine-id.js";
77
- import { TreeWatcher, WatchPushDriver, systemClock, } from "../watcher.js";
78
- import { NoopPushReceiver, } from "../sync/push-receiver.js";
79
- import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
80
- import { PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
75
+ import { migratePersonalVaultJournal } from "../journal.js";
76
+ import { createRunnerEmitter } from "./sync-runner-events.js";
77
+ import { buildFanoutPlan, emitFanoutPlan, resolveMembershipsForRun, } from "./sync-runner-planning.js";
78
+ import { executeCompanyFanout } from "./sync-runner-company.js";
79
+ import { rollupAllComplete } from "./sync-runner-rollup.js";
80
+ import { emitTelemetry } from "./sync-runner-telemetry.js";
81
+ import { runOneShotWithOperationLock, runWatchLoop, } from "./sync-runner-watch-loop.js";
82
+ export { buildTargetedPullArgv, buildTargetedPushArgv, routeChangeToTarget, } from "./sync-runner-watch-routes.js";
81
83
  // ---------------------------------------------------------------------------
82
84
  // Defaults — mirror `hq-cli/src/utils/cognito-session.ts`. Inlined (not
83
85
  // imported) to avoid a circular dep between hq-cli and hq-cloud. If these
@@ -528,14 +530,7 @@ export async function runRunner(argv, deps = {}) {
528
530
  // events. The menubar's `HQ_CLOUD_VERSION` pin gates which runner
529
531
  // they spawn, so old menubars stay on the previous runner version
530
532
  // even after this one is published.
531
- const ERROR_TYPES = new Set([
532
- "error",
533
- "auth-error",
534
- ]);
535
- const emit = (event) => {
536
- const stream = ERROR_TYPES.has(event.type) ? stderr : stdout;
537
- stream.write(`${JSON.stringify(event)}\n`);
538
- };
533
+ const emit = createRunnerEmitter({ stdout, stderr });
539
534
  // ---- argv -------------------------------------------------------------
540
535
  const parsed = parseArgs(argv);
541
536
  if ("error" in parsed) {
@@ -619,37 +614,21 @@ export async function runRunner(argv, deps = {}) {
619
614
  // ---- resolve targets --------------------------------------------------
620
615
  let memberships;
621
616
  try {
622
- if (parsed.personal) {
623
- // Personal-vault-only mode: skip listMyMemberships entirely (and
624
- // therefore the claim-dance). The fanout plan is built solely from
625
- // the person-entity lookup below — no cloud-company targets, no
626
- // /membership/me round-trip. setup-needed is deferred to the
627
- // person-entity check (firing only when the personal entity is
628
- // also absent, not when memberships are empty by design).
629
- memberships = [];
630
- }
631
- else if (parsed.companies) {
632
- // Before giving up on memberships, run the claim-dance: new users signed
633
- // in via the tray may have email-keyed invites waiting for them. Without
634
- // this, an invited user would see "setup-needed" on every tray click.
635
- if (claims) {
636
- await runClaimDance(client, claims, stderr);
637
- }
638
- memberships = await listMembershipsWithRetry(client);
639
- if (memberships.length === 0) {
640
- // Truly empty — still a valid state (no memberships = nothing to
641
- // sync). The tray will show a friendly "create your first company"
642
- // CTA rather than an alarm banner.
643
- emit({ type: "setup-needed" });
644
- return 0;
645
- }
646
- }
647
- else {
648
- // Single-company mode: fabricate a minimal membership so the fanout
649
- // loop below treats it uniformly. We don't need to hit
650
- // /membership/me — the caller already told us which company.
651
- memberships = [{ companyUid: parsed.company }];
617
+ const resolution = await resolveMembershipsForRun({
618
+ personal: parsed.personal,
619
+ companies: parsed.companies,
620
+ company: parsed.company,
621
+ client,
622
+ claims,
623
+ stderr,
624
+ runClaimDance,
625
+ listMemberships: listMembershipsWithRetry,
626
+ });
627
+ if (resolution.status === "setup-needed") {
628
+ emit({ type: "setup-needed" });
629
+ return 0;
652
630
  }
631
+ memberships = resolution.memberships;
653
632
  }
654
633
  catch (err) {
655
634
  if (err instanceof VaultAuthError) {
@@ -669,64 +648,20 @@ export async function runRunner(argv, deps = {}) {
669
648
  });
670
649
  return 1;
671
650
  }
672
- // ---- resolve slugs for the fanout plan --------------------------------
673
- // The menubar wants "Syncing indigo" in its UI, not the raw cmp_* ULID.
674
- // If the entity fetch fails for some row (entity deleted, scoping issue),
675
- // degrade to using the UID as the slug rather than aborting the run.
676
- const plan = [];
677
- for (const m of memberships) {
678
- let slug = m.companyUid;
679
- let name;
680
- try {
681
- const info = await client.entity.get(m.companyUid);
682
- slug = info.slug || m.companyUid;
683
- name = info.name;
684
- }
685
- catch {
686
- // Best-effort — keep UID as the display identifier.
687
- }
688
- plan.push({ uid: m.companyUid, slug, ...(name ? { name } : {}) });
689
- }
690
- if ((parsed.companies || parsed.personal) &&
691
- !resolveSkipPersonal(parsed.skipPersonal)) {
692
- // Personal-target fanout slot. Skipped entirely when --skip-personal
693
- // (or HQ_SYNC_SKIP_PERSONAL=1) is set — see resolveSkipPersonal doc for
694
- // the rationale (menubar opt-out for users who only want company sync).
695
- // When skipped, the fanout-plan event below carries only company
696
- // memberships and no "personal" slug; downstream consumers (menubar
697
- // workspaces row, status surfaces) should already tolerate that
698
- // shape since pre-5.25 fanout often had it (a user with no person
699
- // entity yet, or before the canonical-person-entity machinery landed).
700
- //
701
- // `--personal` mode reaches this block with an empty `plan` and
702
- // empty `memberships`; only the personal target gets added below.
703
- const persons = await client.entity.listByType("person");
704
- const pick = pickCanonicalPersonEntity(persons);
705
- if (pick?.bucketName) {
706
- plan.push({
707
- slug: "personal",
708
- uid: pick.uid,
709
- bucketName: pick.bucketName,
710
- personalMode: true,
711
- // Reserved sentinel slug — decoupled from the `companies/personal`
712
- // company slug to avoid sharing `sync-journal.personal.json`. The
713
- // display `slug` stays "personal" (menubar UI); only the journal
714
- // shard changes. See PERSONAL_VAULT_JOURNAL_SLUG for the full
715
- // root-cause writeup.
716
- journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
717
- });
718
- }
719
- else if (parsed.personal) {
720
- // --personal mode with no canonical personal entity → setup-needed.
721
- // (In --companies mode this state is silent — companies still sync
722
- // and the missing personal target just shows an empty workspaces row.
723
- // In --personal mode it's the ONLY signal, so it gets surfaced as
724
- // setup-needed with the same shape as the empty-memberships case.)
725
- emit({ type: "setup-needed" });
726
- return 0;
727
- }
651
+ const targetPlan = await buildFanoutPlan({
652
+ memberships,
653
+ companies: parsed.companies,
654
+ personal: parsed.personal,
655
+ skipPersonal: parsed.skipPersonal,
656
+ client,
657
+ resolveSkipPersonal,
658
+ });
659
+ if (targetPlan.status === "setup-needed") {
660
+ emit({ type: "setup-needed" });
661
+ return 0;
728
662
  }
729
- emit({ type: "fanout-plan", companies: plan });
663
+ const plan = targetPlan.plan;
664
+ emitFanoutPlan(emit, plan);
730
665
  // One-time seed of the reserved personal-vault journal from the legacy
731
666
  // "personal" journal, so switching the vault slot off the colliding slug
732
667
  // does NOT trigger a mass re-upload of the HQ overlay on first run. No-op
@@ -736,508 +671,39 @@ export async function runRunner(argv, deps = {}) {
736
671
  // ---- fanout -----------------------------------------------------------
737
672
  const syncFn = deps.sync ?? defaultSync;
738
673
  const shareFn = deps.share ?? defaultShare;
739
- const doPush = parsed.direction === "push" || parsed.direction === "both";
740
- const doPull = parsed.direction === "pull" || parsed.direction === "both";
741
- const errors = [];
742
- const allConflicts = [];
743
- const stateByCompany = new Map();
744
- for (const target of plan) {
745
- const companyLabel = target.slug;
746
- const state = {
747
- company: companyLabel,
748
- // Default to "errored" so a throw before any complete-or-clean-abort
749
- // path (the original bug) leaves the entry flagged as not-clean. The
750
- // success/clean-abort paths overwrite this before the loop body exits.
751
- status: "errored",
752
- filesDownloaded: 0,
753
- bytesDownloaded: 0,
754
- filesUploaded: 0,
755
- bytesUploaded: 0,
756
- };
757
- stateByCompany.set(companyLabel, state);
758
- // Which phase is currently emitting `progress` events. Mutable closure so
759
- // tagAndEmit (defined once below) reads the latest value when each event
760
- // fires. "pull" is the default for back-compat with pull-only runs.
761
- let activePhase = doPush && !doPull ? "push" : "pull";
762
- // Per-company event tagger — shared by push and pull phases so progress
763
- // rows land on the right company regardless of which phase emitted them.
764
- // Also updates `state` for `progress` events so the rollup has accurate
765
- // partial counts even if the sync function throws before returning.
766
- let companyHadTransferError = false;
767
- const tagAndEmit = (event) => {
768
- if (event.type === "plan") {
769
- emit({
770
- type: "plan",
771
- company: companyLabel,
772
- filesToDownload: event.filesToDownload,
773
- bytesToDownload: event.bytesToDownload,
774
- filesToUpload: event.filesToUpload,
775
- bytesToUpload: event.bytesToUpload,
776
- filesToSkip: event.filesToSkip,
777
- filesToConflict: event.filesToConflict,
778
- filesToDelete: event.filesToDelete,
779
- });
780
- }
781
- else if (event.type === "progress") {
782
- if (activePhase === "push") {
783
- state.filesUploaded += 1;
784
- state.bytesUploaded += event.bytes;
785
- }
786
- else {
787
- state.filesDownloaded += 1;
788
- state.bytesDownloaded += event.bytes;
789
- }
790
- emit({
791
- type: "progress",
792
- company: companyLabel,
793
- path: event.path,
794
- bytes: event.bytes,
795
- // Stamp the transfer direction from the in-flight phase so the
796
- // menubar can label each file uploaded vs downloaded. The inner
797
- // share()/sync() emitters don't know the phase — only this tagger,
798
- // which mirrors the up/down counter bump above.
799
- direction: activePhase === "push" ? "up" : "down",
800
- ...(event.message ? { message: event.message } : {}),
801
- ...(event.deleted ? { deleted: event.deleted } : {}),
802
- });
803
- }
804
- else if (event.type === "conflict") {
805
- emit({
806
- type: "conflict",
807
- company: companyLabel,
808
- path: event.path,
809
- direction: event.direction,
810
- resolution: event.resolution,
811
- });
812
- }
813
- else if (event.type === "error") {
814
- companyHadTransferError = true;
815
- errors.push({
816
- company: companyLabel,
817
- message: event.path ? `${event.path}: ${event.message}` : event.message,
818
- });
819
- emit({
820
- type: "error",
821
- company: companyLabel,
822
- path: event.path,
823
- message: event.message,
824
- });
825
- }
826
- else if (event.type === "new-files") {
827
- emit({
828
- type: "new-files",
829
- company: companyLabel,
830
- files: event.files,
831
- });
832
- }
833
- else if (event.type === "scope-excluded") {
834
- // Push-side ACL scope exclusions — surface the named paths tagged to
835
- // this company so the menubar/CLI can show "N skipped, outside your
836
- // access" instead of the file silently never uploading.
837
- emit({
838
- type: "scope-excluded",
839
- company: companyLabel,
840
- count: event.count,
841
- samplePaths: event.samplePaths,
842
- });
843
- }
844
- };
845
- try {
846
- let pushResult = {
847
- filesUploaded: 0,
848
- bytesUploaded: 0,
849
- filesSkipped: 0,
850
- filesDeleted: 0,
851
- filesTombstoned: 0,
852
- filesRefusedStale: 0,
853
- filesRefusedStalePaths: [],
854
- filesSuppressedByTombstone: 0,
855
- filesExcludedByPolicy: 0,
856
- filesExcludedByScope: 0,
857
- conflictPaths: [],
858
- aborted: false,
859
- };
860
- let pullResult = {
861
- filesDownloaded: 0,
862
- bytesDownloaded: 0,
863
- filesSkipped: 0,
864
- conflicts: 0,
865
- conflictPaths: [],
866
- aborted: false,
867
- newFiles: [],
868
- newFilesCount: 0,
869
- filesExcludedByPolicy: 0,
870
- filesTombstoned: 0,
871
- filesOutOfScope: 0,
872
- scopeOrphansRemoved: 0,
873
- scopeOrphansBlocked: 0,
874
- };
875
- // Push first so a subsequent pull doesn't overwrite files we were about
876
- // to broadcast. Company targets walk `companies/{slug}/`; the personal
877
- // target walks every top-level entry under hqRoot minus the exclusion
878
- // list (see PERSONAL_VAULT_EXCLUDED_TOP_LEVEL). `skipUnchanged: true`
879
- // keeps both cases efficient on re-runs.
880
- // Shared between push and pull for the personal slot. Hoisted out of
881
- // the if-blocks below so doPush + doPull see the same set:
882
- //
883
- // `HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL=1` opts INTO the
884
- // cloud:false → personal-bucket fallback. Default OFF keeps the
885
- // personal vault identical to the pre-5.20 shape until operators
886
- // explicitly enable it.
887
- //
888
- // `teamSyncedSlugs` is the slug set the operator currently has
889
- // active team-bucket Memberships for, derived from the live plan.
890
- // Used by:
891
- // • push: filter `companies/` subdir enumeration so a team-synced
892
- // company never rides the personal slot; build
893
- // `decommissionPrefixes` so share() sweeps orphan keys for
894
- // any slug that's now team-backed.
895
- // • pull: filter `listRemoteFiles` so pre-decommission orphan
896
- // keys at `companies/{team-synced-slug}/...` don't get
897
- // re-downloaded into the disk paths the team-bucket pull
898
- // now manages.
899
- const includeLocalCompanies = process.env.HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL === "1";
900
- const teamSyncedSlugs = new Set(plan
901
- .filter((p) => p.personalMode !== true)
902
- .map((p) => p.slug));
903
- // Resolve the membership's effective ACL scope ONCE so BOTH the push and
904
- // pull legs respect the granted prefixes. The vault vends a child
905
- // credential scoped to these prefixes; without filtering the PUSH plan to
906
- // them, share() would HEAD/PUT keys outside the grant and the server's
907
- // correct 403 (SCOPE_EXCEEDS_PARENT) would abort the WHOLE company with a
908
- // generic error + exit 2. Personal-vault legs have no membership
909
- // sync-config — they stay full-scope ("all"). Degrades to "all" on any
910
- // error (a transient failure must never silently filter/prune the tree).
911
- // Hoisted above the push block (it used to be resolved only for pull) so
912
- // push gets the same scope; the pull leg below reuses this value.
913
- const scope = target.personalMode === true
914
- ? { syncMode: "all" }
915
- : await resolvePullScope(client, target.uid, target.slug, parsed.hqRoot);
916
- if (doPush) {
917
- activePhase = "push";
918
- // For the personal slot we hand share() both (a) the top-level
919
- // hqRoot entries that are part of the personal vault and (b) any
920
- // `companies/{slug}/` subdirs the user has opted into the personal
921
- // bucket via `cloud: false` in `company.yaml`. Slugs that already
922
- // own a team bucket (anything else in `plan`) are excluded so a
923
- // company is never double-written.
924
- const pushPaths = target.personalMode === true
925
- ? computePersonalVaultPaths(parsed.hqRoot, {
926
- includeLocalCompanies,
927
- teamSyncedSlugs,
928
- })
929
- : [path.join(parsed.hqRoot, "companies", target.slug)];
930
- // For the personal slot, hand share() a list of `companies/{slug}/`
931
- // prefixes for every team-synced slug. If the personal-bucket
932
- // journal still holds entries under any of those prefixes — i.e.
933
- // the company used to ride the personal-bucket fallback and was
934
- // since promoted to its own team bucket — share() will sweep
935
- // those orphans via DeleteObject + journal removal. Unconditional
936
- // (not gated on `HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL`) so cleanup
937
- // still runs after an operator turns the feature off: the on-ramp
938
- // is opt-in but the off-ramp is always safe to traverse.
939
- const decommissionPrefixes = target.personalMode === true
940
- ? Array.from(teamSyncedSlugs).map((slug) => `companies/${slug}`)
941
- : undefined;
942
- pushResult = await shareFn({
943
- paths: pushPaths,
944
- company: target.uid,
945
- vaultConfig,
946
- hqRoot: parsed.hqRoot,
947
- onConflict: parsed.onConflict,
948
- skipUnchanged: true,
949
- // Local deletes propagate to S3 as soft deletes (versioning is on
950
- // — DeleteObject writes a delete-marker, prior versions remain
951
- // recoverable). Without this, a deleted file resurfaces on the
952
- // next pull because the remote object is still listable.
953
- //
954
- // Policy default in 5.24 is `owned-only` (pre-5.24 behavior;
955
- // preserved for the soak window). `HQ_SYNC_DELETE_POLICY` env
956
- // can opt INTO the safer `currency-gated` (per-file HEAD + ETag
957
- // verification) or the unsafe `all` (emergency reconcile only).
958
- // Default flips to `currency-gated` in 5.25 after at least one
959
- // machine has soaked the new path. Both personal and company
960
- // targets use the same resolver — same engine, same flip.
961
- propagateDeletes: true,
962
- propagateDeletePolicy: resolveDeletePolicy(),
963
- onEvent: tagAndEmit,
964
- ...(uploadAuthor ? { author: uploadAuthor } : {}),
965
- // Mirror the pull-side seam: only spread these for the personal
966
- // slot so company-target args stay identical to the pre-Slice-2
967
- // shape (the "no personalMode/journalSlug keys" regression test
968
- // in sync-runner.test.ts pins that contract).
969
- ...(target.personalMode !== undefined ? { personalMode: target.personalMode } : {}),
970
- ...(target.journalSlug !== undefined ? { journalSlug: target.journalSlug } : {}),
971
- ...(decommissionPrefixes && decommissionPrefixes.length > 0
972
- ? { decommissionPrefixes }
973
- : {}),
974
- // US-005 symmetry: scope the PUSH plan to the membership's granted
975
- // ACL prefixes so out-of-scope keys are skipped (and surfaced via a
976
- // `scope-excluded` event) instead of drawing the server's 403 and
977
- // aborting the company. `undefined` for `syncMode: "all"` (owner /
978
- // personal) → no scope filter, identical to the pre-fix shape so the
979
- // "company-target args" contract test stays green.
980
- ...(scope.prefixSet !== undefined ? { prefixSet: scope.prefixSet } : {}),
981
- });
982
- }
983
- // Pull runs unless the push phase aborted on conflict — aborted means
984
- // the user has local edits + remote drift; blindly pulling would erase
985
- // whichever side `--on-conflict abort` just protected.
986
- if (doPull && !pushResult.aborted) {
987
- activePhase = "pull";
988
- // US-005: the pull only materializes in-scope keys (and prunes clean
989
- // orphans when scope shrank). Reuse the `scope` resolved once above so
990
- // push and pull apply the SAME granted prefixes and we avoid a second
991
- // `listMyExplicitGrants` round-trip per company.
992
- const pullScope = scope;
993
- pullResult = await syncFn({
994
- company: target.uid,
995
- vaultConfig,
996
- hqRoot: parsed.hqRoot,
997
- onConflict: parsed.onConflict,
998
- syncMode: pullScope.syncMode,
999
- // The menubar runner can take no interactive flag, so a scope shrink
1000
- // must NEVER throw here (the old `ScopeShrinkBlockedError` → exit 2
1001
- // was the permanent wedge in DEV-1768). Self-heal non-destructively:
1002
- // dirty out-of-scope files stay on disk + un-tracked, clean ones are
1003
- // quarantined (recoverable). This also clears an already-wedged
1004
- // journal — seeded by a buggy `all`-mode CLI pull — on the next sync.
1005
- scopeShrinkPolicy: "auto-recover",
1006
- // Scope-shrink authorship guard: pass the caller's own sub (the very
1007
- // sub stamped onto uploads as `created-by-sub`) so a scope shrink
1008
- // never prunes content this owner authored. Owners hold their whole
1009
- // vault by role-bypass, so without this a `shared`/`custom` pull
1010
- // would treat their own un-granted work as a foreign orphan.
1011
- ...(uploadAuthor?.userSub !== undefined
1012
- ? { callerSub: uploadAuthor.userSub }
1013
- : {}),
1014
- ...(pullScope.prefixSet !== undefined
1015
- ? { prefixSet: pullScope.prefixSet }
1016
- : {}),
1017
- ...(target.personalMode !== undefined ? { personalMode: target.personalMode } : {}),
1018
- ...(target.journalSlug !== undefined ? { journalSlug: target.journalSlug } : {}),
1019
- // Symmetric to the push side: for the personal slot, tell sync()
1020
- // how to interpret `companies/{slug}/...` keys in the personal
1021
- // bucket. With `includeLocalCompanies: true`, keys for slugs NOT
1022
- // in `teamSyncedSlugs` are downloaded (legitimate cloud:false
1023
- // opt-in content from another machine); keys for slugs IN that
1024
- // set are dropped as orphans (pre-decommission residue from a
1025
- // promoted company). With `includeLocalCompanies: false` the
1026
- // legacy pre-5.20 behavior holds: all `companies/...` keys are
1027
- // dropped. Only spread for the personal slot so company-target
1028
- // args stay identical to pre-Slice-2 shape (test C pin).
1029
- ...(target.personalMode === true
1030
- ? {
1031
- includeLocalCompanies,
1032
- teamSyncedSlugs,
1033
- }
1034
- : {}),
1035
- ...(deps.operationLockAlreadyHeld
1036
- ? { operationLockAlreadyHeld: true }
1037
- : {}),
1038
- onEvent: tagAndEmit,
1039
- });
1040
- }
1041
- // Concat push + pull conflict paths into a single per-company list,
1042
- // then dedupe — a key that legitimately conflicts on both halves of
1043
- // a bidirectional run (e.g. `.hq/install-manifest.json` round-trip
1044
- // before Fix #1) appears twice in the concat but represents a single
1045
- // logical conflict for the operator. Bug #3 in the 5.33.0 deep-test:
1046
- // every round produced `conflictPaths: [X, X]` and `conflicts: 2`,
1047
- // double-counting the conflict in every metric downstream. The push
1048
- // and pull halves each emit a `conflict` event in their own direction
1049
- // (preserved on the event stream for tracing); only the merged
1050
- // result list is collapsed. Stable first-seen order is preserved so
1051
- // consumers can rely on the pull entry coming before its push twin.
1052
- const seenConflictPaths = new Set();
1053
- const mergedConflictPaths = [];
1054
- for (const p of [...pullResult.conflictPaths, ...pushResult.conflictPaths]) {
1055
- if (seenConflictPaths.has(p))
1056
- continue;
1057
- seenConflictPaths.add(p);
1058
- mergedConflictPaths.push(p);
1059
- }
1060
- const aborted = pullResult.aborted || pushResult.aborted;
1061
- // Overwrite the progress-derived counts with the authoritative numbers
1062
- // from the sync/share return values. The `progress` stream over-counts
1063
- // when the inner walker emits a progress row for a file it then skips
1064
- // due to a journal hit — a clean return value is the source of truth.
1065
- // For the throw case below this overwrite never runs, so `state` keeps
1066
- // its progress-derived counts (which is exactly what we want there).
1067
- state.filesDownloaded = pullResult.filesDownloaded;
1068
- state.bytesDownloaded = pullResult.bytesDownloaded;
1069
- state.filesUploaded = pushResult.filesUploaded;
1070
- state.bytesUploaded = pushResult.bytesUploaded;
1071
- state.status = companyHadTransferError
1072
- ? "errored"
1073
- : aborted
1074
- ? "aborted"
1075
- : "complete";
1076
- emit({
1077
- type: "complete",
1078
- company: companyLabel,
1079
- filesDownloaded: pullResult.filesDownloaded,
1080
- bytesDownloaded: pullResult.bytesDownloaded,
1081
- filesUploaded: pushResult.filesUploaded,
1082
- bytesUploaded: pushResult.bytesUploaded,
1083
- filesSkipped: pullResult.filesSkipped + pushResult.filesSkipped,
1084
- // Push-side counters surfaced on `complete` so the menubar's
1085
- // `SyncCompleteEvent` (which carries them as Option<u32> for
1086
- // back-compat with pre-5.25 engines) can render the new totals.
1087
- // Always emitted as numbers (0 when no push leg ran) so Rust's
1088
- // serde decodes them as `Some(0)` rather than `None` — distinct
1089
- // from the legacy-engine `None` and useful when the UI wants to
1090
- // distinguish "engine ran, nothing tombstoned" from "engine
1091
- // didn't report".
1092
- // Tombstones now flow on both legs:
1093
- // - push side: `ShareResult.filesTombstoned` (remote was already
1094
- // 404 at HEAD time, journal entry dropped).
1095
- // - pull side: `SyncResult.filesTombstoned` (Bug #9 — journal-
1096
- // known key missing from remote LIST, applied as local delete).
1097
- // Sum them so the menubar's `SyncCompleteEvent` reflects the total
1098
- // delete-propagation activity for that company across the run.
1099
- filesTombstoned: pushResult.filesTombstoned + pullResult.filesTombstoned,
1100
- filesRefusedStale: pushResult.filesRefusedStale,
1101
- // Bonus diagnostic: surface the paths so operators can triage the
1102
- // recurring `filesRefusedStale: N` signal — the count alone was
1103
- // untriageable per the 5.33.0 deep-test report's "205 issue".
1104
- // Pre-capped at 50 by share() itself.
1105
- filesRefusedStalePaths: pushResult.filesRefusedStalePaths,
1106
- // Pull side now reports an `filesExcludedByPolicy` too (Bug #2 —
1107
- // ephemeral conflict-mirror refusals in the pull walker). Sum
1108
- // both legs so the `complete` event reports total excluded across
1109
- // the full bidirectional pass; pre-fix the pull half silently
1110
- // pushed legacy `.conflict-*` litter into clean trees with the
1111
- // same counter showing 0.
1112
- filesExcludedByPolicy: pushResult.filesExcludedByPolicy + pullResult.filesExcludedByPolicy,
1113
- // Sourced from the merged path list so push-side conflicts are
1114
- // counted too — `ShareResult` doesn't expose a numeric counter,
1115
- // and using `pullResult.conflicts` alone silently dropped any
1116
- // push conflict from the count while leaving its path in
1117
- // `conflictPaths`.
1118
- conflicts: mergedConflictPaths.length,
1119
- conflictPaths: mergedConflictPaths,
1120
- // Either phase aborting marks the company aborted — the UI treats
1121
- // `aborted: true` as "sync didn't complete cleanly for this company".
1122
- aborted,
1123
- newFiles: pullResult.newFiles,
1124
- newFilesCount: pullResult.newFilesCount,
1125
- // Scope-aware download counters (US-005). Pull-only — the push leg
1126
- // has no scope concept — so they pass through from `pullResult`.
1127
- filesOutOfScope: pullResult.filesOutOfScope,
1128
- scopeOrphansRemoved: pullResult.scopeOrphansRemoved,
1129
- scopeOrphansBlocked: pullResult.scopeOrphansBlocked,
1130
- });
1131
- for (const p of pullResult.conflictPaths) {
1132
- allConflicts.push({ company: companyLabel, path: p, direction: "pull" });
1133
- }
1134
- for (const p of pushResult.conflictPaths) {
1135
- allConflicts.push({ company: companyLabel, path: p, direction: "push" });
1136
- }
1137
- }
1138
- catch (err) {
1139
- // describeError walks the cause chain so AWS SDK v3's "UnknownError"
1140
- // wrapper surfaces the underlying Node networking error (ENOTFOUND,
1141
- // ECONNRESET, …) instead of an unactionable bare "UnknownError".
1142
- const message = describeError(err);
1143
- errors.push({ company: companyLabel, message });
1144
- // `state.status` was seeded as "errored" at loop entry — the throw
1145
- // path leaves it there, and `state.files{Down,Up}loaded` reflects the
1146
- // partial counts captured from `progress` events before the throw.
1147
- // Emit a `complete` event with `aborted: true` and those partial
1148
- // counts so consumers walking the `complete` event stream see every
1149
- // company in the fanout uniformly. This is the fix for the misleading
1150
- // rollup — see file header `Exit code: 2` doc.
1151
- emit({
1152
- type: "complete",
1153
- company: companyLabel,
1154
- filesDownloaded: state.filesDownloaded,
1155
- bytesDownloaded: state.bytesDownloaded,
1156
- filesUploaded: state.filesUploaded,
1157
- bytesUploaded: state.bytesUploaded,
1158
- filesSkipped: 0,
1159
- // Mid-flight throw: we have no clean ShareResult to read these
1160
- // from. Report 0 so the event shape stays stable; the partial
1161
- // counts above already reflect what actually moved before the
1162
- // throw.
1163
- filesTombstoned: 0,
1164
- filesRefusedStale: 0,
1165
- filesRefusedStalePaths: [],
1166
- filesExcludedByPolicy: 0,
1167
- conflicts: 0,
1168
- conflictPaths: [],
1169
- aborted: true,
1170
- newFiles: [],
1171
- newFilesCount: 0,
1172
- // Mid-flight throw: no clean scope counts to report. 0 keeps the
1173
- // event shape stable (US-005).
1174
- filesOutOfScope: 0,
1175
- scopeOrphansRemoved: 0,
1176
- scopeOrphansBlocked: 0,
1177
- });
1178
- emit({
1179
- type: "error",
1180
- company: companyLabel,
1181
- path: "(company)",
1182
- message,
1183
- });
1184
- // Continue — one company's failure shouldn't abort the whole fanout.
1185
- }
1186
- }
1187
- // Walk every per-company entry — the map holds one row per planned company,
1188
- // including ones that aborted via thrown exception. This is the fix for the
1189
- // bug where `all-complete` reported `filesDownloaded: 0` for an aborted
1190
- // personal-sync that had already emitted thousands of `progress` events:
1191
- // the rollup used to only sum companies that emitted a clean `complete`,
1192
- // which silently dropped partials when the sync function threw.
1193
- let totalDownloaded = 0;
1194
- let totalDownloadedBytes = 0;
1195
- let totalUploaded = 0;
1196
- let totalUploadedBytes = 0;
1197
- let partial = false;
1198
- const companies = [];
1199
- for (const target of plan) {
1200
- const s = stateByCompany.get(target.slug);
1201
- if (!s)
1202
- continue; // unreachable — every plan entry seeds the map
1203
- totalDownloaded += s.filesDownloaded;
1204
- totalDownloadedBytes += s.bytesDownloaded;
1205
- totalUploaded += s.filesUploaded;
1206
- totalUploadedBytes += s.bytesUploaded;
1207
- if (s.status !== "complete")
1208
- partial = true;
1209
- companies.push({
1210
- company: s.company,
1211
- status: s.status,
1212
- filesDownloaded: s.filesDownloaded,
1213
- bytesDownloaded: s.bytesDownloaded,
1214
- filesUploaded: s.filesUploaded,
1215
- bytesUploaded: s.bytesUploaded,
1216
- });
1217
- }
674
+ const fanout = await executeCompanyFanout({
675
+ plan,
676
+ direction: parsed.direction,
677
+ hqRoot: parsed.hqRoot,
678
+ onConflict: parsed.onConflict,
679
+ client,
680
+ vaultConfig,
681
+ ...(uploadAuthor ? { uploadAuthor } : {}),
682
+ ...(deps.operationLockAlreadyHeld ? { operationLockAlreadyHeld: true } : {}),
683
+ syncFn,
684
+ shareFn,
685
+ resolveDeletePolicy,
686
+ emit,
687
+ });
688
+ const { errors, allConflicts } = fanout;
689
+ const rollup = rollupAllComplete(plan, fanout.stateByCompany);
1218
690
  // Fire telemetry collector before the all-complete emit so the cursor at
1219
691
  // `~/.hq/telemetry-cursor.json` is consistent with what the menubar sees.
1220
- // Awaited fully — fire-and-forget would lose in-flight POSTs at process
1221
- // exit, and the previous 10s race was wrong: a first-run user with
1222
- // backlog (e.g. 60K Claude session events) takes well over 10s legitimately,
1223
- // and the race silently dropped the entire batch when it fired. Errors
1224
- // are swallowed inside `defaultCollectTelemetry`; per-request timeouts
1225
- // come from `VaultClient`'s retry loop (3 attempts × exponential backoff),
1226
- // which naturally bounds the outer wait.
1227
- const telemetryFn = deps.collectTelemetry ??
1228
- (() => defaultCollectTelemetry(client, deps.createVaultClient !== undefined, parsed.hqRoot));
1229
- await telemetryFn().catch(() => undefined);
692
+ await emitTelemetry({
693
+ collectTelemetry: deps.collectTelemetry,
694
+ defaultCollectTelemetry: () => defaultCollectTelemetry(client, deps.createVaultClient !== undefined, parsed.hqRoot),
695
+ });
1230
696
  emit({
1231
697
  type: "all-complete",
1232
698
  companiesAttempted: plan.length,
1233
- filesDownloaded: totalDownloaded,
1234
- bytesDownloaded: totalDownloadedBytes,
1235
- filesUploaded: totalUploaded,
1236
- bytesUploaded: totalUploadedBytes,
699
+ filesDownloaded: rollup.totalDownloaded,
700
+ bytesDownloaded: rollup.totalDownloadedBytes,
701
+ filesUploaded: rollup.totalUploaded,
702
+ bytesUploaded: rollup.totalUploadedBytes,
1237
703
  conflictPaths: allConflicts,
1238
704
  errors,
1239
- partial,
1240
- companies,
705
+ partial: rollup.partial,
706
+ companies: rollup.companies,
1241
707
  });
1242
708
  // Post-sync qmd reindex — runs AFTER `all-complete` is emitted so the
1243
709
  // menubar/CLI already shows the sync as done; this is a best-effort tail
@@ -1245,7 +711,8 @@ export async function runRunner(argv, deps = {}) {
1245
711
  // pulled in (nothing to reindex otherwise) and not explicitly disabled.
1246
712
  // Self-contained: shells out to the global `qmd` binary, no dependency on
1247
713
  // any (possibly stale) script inside the synced HQ tree. See qmd-reindex.ts.
1248
- if (totalDownloaded > 0 && process.env.HQ_QMD_REINDEX_ON_SYNC !== "0") {
714
+ if (rollup.totalDownloaded > 0 &&
715
+ process.env.HQ_QMD_REINDEX_ON_SYNC !== "0") {
1249
716
  try {
1250
717
  reindexAfterSync(parsed.hqRoot);
1251
718
  }
@@ -1303,571 +770,23 @@ const isDirectInvocation = (() => {
1303
770
  return false;
1304
771
  }
1305
772
  })();
1306
- /**
1307
- * Route a changed relative path to the push target that owns it.
1308
- *
1309
- * - `companies/<slug>/...` → a single-company push for `<slug>` (the targeted
1310
- * pass per PRD decision — push only the changed company, not a full
1311
- * `--companies` fanout).
1312
- * - anything else under hqRoot → the personal target (a `--companies` push
1313
- * restricted to personal via the personal-vault scope; modeled here as the
1314
- * "personal" route the caller maps to the right argv).
1315
- *
1316
- * Returns `null` for paths the routing cannot attribute (defensive — the
1317
- * watcher's filter already drops excluded top-levels, so this is belt-and-
1318
- * suspenders for synthetic events).
1319
- */
1320
- export function routeChangeToTarget(relPath) {
1321
- const norm = relPath.split(path.sep).join("/").replace(/^\.\//, "");
1322
- if (norm === "" || norm.startsWith(".."))
1323
- return null;
1324
- const segments = norm.split("/").filter((s) => s.length > 0);
1325
- if (segments.length === 0)
1326
- return null;
1327
- if (segments[0] === "companies") {
1328
- // companies/<slug>/... — need at least the slug segment to target.
1329
- if (segments.length < 2 || segments[1].length === 0)
1330
- return null;
1331
- return { kind: "company", slug: segments[1] };
1332
- }
1333
- return { kind: "personal" };
1334
- }
1335
- /**
1336
- * Build the argv for a targeted push pass from a routed change. The push runs
1337
- * `--direction push` for just the routed target so a local edit propagates in
1338
- * seconds without a full fanout. Company routes use `--company <slug>`;
1339
- * personal routes use `--companies --direction push` (the personal-vault scope
1340
- * is resolved inside runRunner's fanout; skipUnchanged no-ops the company
1341
- * subtrees that didn't change). Inherits `--hq-root` / `--on-conflict` from
1342
- * the base argv. Pure helper, exported for unit testing the routing→argv map.
1343
- */
1344
- export function buildTargetedPushArgv(route, baseArgv) {
1345
- const carried = carriedFlags(baseArgv);
1346
- if (route.kind === "company") {
1347
- return ["--company", route.slug, "--direction", "push", ...carried];
1348
- }
1349
- return ["--companies", "--direction", "push", ...carried];
1350
- }
1351
- function routeKey(route) {
1352
- return route.kind === "company" ? `company:${route.slug}` : "personal";
1353
- }
1354
- function routesForBatch(batch) {
1355
- const routes = new Map();
1356
- for (const relPath of batch.paths.values()) {
1357
- const route = routeChangeToTarget(relPath);
1358
- if (!route)
1359
- continue;
1360
- routes.set(routeKey(route), route);
1361
- }
1362
- return routes;
1363
- }
1364
- function buildFullFanoutPushArgv(baseArgv) {
1365
- return ["--companies", "--direction", "push", ...carriedFlags(baseArgv)];
1366
- }
1367
- /**
1368
- * Build the argv for a targeted PULL pass from a routed change (US-009 — the
1369
- * receiver's pull-on-event path). Mirrors {@link buildTargetedPushArgv} but
1370
- * with `--direction pull`: a peer device pushed a change, so this device pulls
1371
- * just the affected company/subtree instead of waiting for the next
1372
- * `--poll-remote-ms` cycle. Company routes use `--company <slug>`; personal
1373
- * routes use `--companies` (the personal-vault scope is resolved inside
1374
- * runRunner's fanout; skipUnchanged no-ops the subtrees that didn't change).
1375
- * Inherits `--hq-root` / `--on-conflict` from the base argv. Pure helper,
1376
- * exported for unit testing the event→argv map.
1377
- */
1378
- export function buildTargetedPullArgv(route, baseArgv) {
1379
- const carried = carriedFlags(baseArgv);
1380
- if (route.kind === "company") {
1381
- return ["--company", route.slug, "--direction", "pull", ...carried];
1382
- }
1383
- return ["--companies", "--direction", "pull", ...carried];
1384
- }
1385
773
  export async function runRunnerWithLoop(argv, deps = {}) {
1386
774
  const parsed = parseArgs(argv);
775
+ const runtime = {
776
+ runPassWithOperationLockAlreadyHeld: (passArgv) => runRunner(passArgv, { operationLockAlreadyHeld: true }),
777
+ defaultGetIdTokenClaims,
778
+ defaultGetAccessToken: () => getValidAccessToken(DEFAULT_COGNITO, { interactive: false }),
779
+ apiUrl: DEFAULT_VAULT_API_URL,
780
+ region: DEFAULT_COGNITO.region,
781
+ };
1387
782
  if (!argv.includes("--watch")) {
1388
- // One-shot cloud sync — take the per-root operation lock so it is mutually
1389
- // exclusive with rescue/reindex. If args don't parse, fall through to
1390
- // `runRunner` so it surfaces the parse error rather than us masking it
1391
- // with a lock failure.
1392
783
  if ("error" in parsed)
1393
784
  return runRunner(argv);
1394
- // The actual sync pass — same seam the watch loop uses (deps.runPass),
1395
- // so a test can assert "waits for a short-lived holder, THEN proceeds to
1396
- // sync" without touching the network. Production passes `argv` to
1397
- // runRunner exactly as before. Regression guard for DEV-1772
1398
- // (feedback_28a1833f): instant-sync one-shots used to exit 17 and die on
1399
- // a lock conflict with the ~1-min reindex hook; they now WAIT (default)
1400
- // and proceed once the short holder releases.
1401
- const runOnce = deps.runPass ??
1402
- ((passArgv) => runRunner(passArgv, { operationLockAlreadyHeld: true }));
1403
- try {
1404
- return await withOperationLock(parsed.hqRoot, "sync", () => runOnce(argv), {
1405
- timeoutSec: parsed.lockTimeoutSec,
1406
- });
1407
- }
1408
- catch (err) {
1409
- if (err instanceof OperationLockedError) {
1410
- // The lock wait was BOUNDED and tripped (a holder never released
1411
- // within --lock-timeout / HQ_OP_LOCK_TIMEOUT). Surface it loudly on
1412
- // stderr and exit with the stable OPERATION_LOCKED_EXIT (17) so the
1413
- // spawner (menubar) can recognize a lock conflict and SCHEDULE A
1414
- // RETRY rather than treating it as a hard failure and silently giving
1415
- // up (DEV-1772). With the default (no bound) we never reach here — the
1416
- // one-shot waits indefinitely and proceeds.
1417
- process.stderr.write(err.message + "\n");
1418
- return OPERATION_LOCKED_EXIT;
1419
- }
1420
- throw err;
1421
- }
785
+ return runOneShotWithOperationLock(argv, parsed, deps, runtime);
1422
786
  }
1423
787
  if ("error" in parsed)
1424
788
  return runRunner(argv);
1425
- const sleep = deps.sleep ??
1426
- ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
1427
- const runPass = deps.runPass ??
1428
- ((passArgv) => runRunner(passArgv, { operationLockAlreadyHeld: true }));
1429
- const pollIdx = argv.indexOf("--poll-remote-ms");
1430
- const pollMs = pollIdx >= 0 && argv[pollIdx + 1] ? Number(argv[pollIdx + 1]) : 600_000;
1431
- const eventPush = argv.includes("--event-push");
1432
- // In `--companies` mode the sync scope is companies/*/{knowledge,projects,…}
1433
- // (per .hqinclude), so the watcher must NOT apply the personal-vault
1434
- // top-level exclusions (PERSONAL_VAULT_EXCLUDED_TOP_LEVEL drops `companies/`
1435
- // and `workspace/`) — doing so would exclude exactly the paths being synced,
1436
- // and no local edit would ever trigger an instant push. The shared ignore
1437
- // stack (createIgnoreFilter / .hqignore / .hqinclude) already scopes the
1438
- // watch filter correctly in companies mode. personalMode is only for a
1439
- // personal-vault-as-root run, where companies/ et al. genuinely aren't synced.
1440
- const companiesMode = argv.includes("--companies");
1441
- const hqRoot = parsed.hqRoot;
1442
- // Strip the loop-only flags before delegating: the parser inside runRunner
1443
- // accepts --watch/--poll-remote-ms/--event-push, but we don't want a per-
1444
- // iteration pass to think it's re-entering watch mode.
1445
- const passArgv = argv.filter((a, i) => {
1446
- if (a === "--watch")
1447
- return false;
1448
- if (a === "--poll-remote-ms")
1449
- return false;
1450
- if (a === "--event-push")
1451
- return false;
1452
- if (i > 0 && argv[i - 1] === "--poll-remote-ms")
1453
- return false;
1454
- return true;
1455
- });
1456
- if (parsed.lockTimeoutSec === 0) {
1457
- try {
1458
- const handle = acquireOperationLock(hqRoot, "sync", {
1459
- timeoutSec: 0,
1460
- onWaitStart: () => undefined,
1461
- });
1462
- handle.release();
1463
- }
1464
- catch (err) {
1465
- if (err instanceof OperationLockedError) {
1466
- process.stderr.write(err.message + "\n");
1467
- return OPERATION_LOCKED_EXIT;
1468
- }
1469
- throw err;
1470
- }
1471
- }
1472
- const runPassWithLock = async (passArgvForRun) => {
1473
- try {
1474
- const handle = acquireOperationLock(hqRoot, "sync", {
1475
- timeoutSec: 0,
1476
- onWaitStart: () => undefined,
1477
- });
1478
- try {
1479
- return await runPass(passArgvForRun);
1480
- }
1481
- finally {
1482
- handle.release();
1483
- }
1484
- }
1485
- catch (err) {
1486
- if (!(err instanceof OperationLockedError))
1487
- throw err;
1488
- if (parsed.lockTimeoutSec === 0) {
1489
- process.stderr.write(err.message + "\n");
1490
- return OPERATION_LOCKED_EXIT;
1491
- }
1492
- }
1493
- try {
1494
- return await withOperationLock(hqRoot, "sync", () => runPass(passArgvForRun), { timeoutSec: parsed.lockTimeoutSec });
1495
- }
1496
- catch (err) {
1497
- if (err instanceof OperationLockedError) {
1498
- process.stderr.write(err.message + "\n");
1499
- return OPERATION_LOCKED_EXIT;
1500
- }
1501
- throw err;
1502
- }
1503
- };
1504
- // ---- shared pass queue -----------------------------------------------
1505
- // The poll loop, pull-on-event receiver, and watcher-triggered pushes all
1506
- // funnel through this queue so local/remote triggers never overlap, and a
1507
- // trigger arriving during an active pass runs immediately after it instead
1508
- // of being dropped.
1509
- let stopped = false;
1510
- let activePass = null;
1511
- const pendingPasses = [];
1512
- const resolveStoppedQueue = () => {
1513
- while (pendingPasses.length > 0) {
1514
- pendingPasses.shift()?.resolve(0);
1515
- }
1516
- };
1517
- const drainQueuedPasses = () => {
1518
- if (activePass !== null)
1519
- return;
1520
- if (stopped) {
1521
- resolveStoppedQueue();
1522
- return;
1523
- }
1524
- const next = pendingPasses.shift();
1525
- if (!next)
1526
- return;
1527
- const current = startGuardedPass(next.argv);
1528
- void current.then(next.resolve, next.reject);
1529
- };
1530
- const startGuardedPass = (passArgvForRun) => {
1531
- const current = stopped
1532
- ? Promise.resolve(0)
1533
- : runPassWithLock(passArgvForRun);
1534
- activePass = current;
1535
- void current
1536
- .finally(() => {
1537
- if (activePass === current) {
1538
- activePass = null;
1539
- drainQueuedPasses();
1540
- }
1541
- })
1542
- .catch(() => undefined);
1543
- return current;
1544
- };
1545
- const runGuarded = (passArgvForRun) => {
1546
- if (activePass === null && pendingPasses.length === 0) {
1547
- return startGuardedPass(passArgvForRun);
1548
- }
1549
- return new Promise((resolve, reject) => {
1550
- pendingPasses.push({ argv: passArgvForRun, resolve, reject });
1551
- drainQueuedPasses();
1552
- });
1553
- };
1554
- // ---- event-push wiring (Phase 1) -------------------------------------
1555
- let watcher = null;
1556
- let driver = null;
1557
- let detachSignal = null;
1558
- const pendingWatcherPaths = new Map();
1559
- let pendingWatcherOriginalBatch = null;
1560
- let pendingWatcherBareChange = false;
1561
- let pendingWatcherOverflowed = false;
1562
- let pendingWatcherDroppedPaths = 0;
1563
- let pendingWatcherDroppedBytes = 0;
1564
- const addPendingWatcherChange = (changedRelPath, batch) => {
1565
- if (batch) {
1566
- pendingWatcherOriginalBatch =
1567
- pendingWatcherPaths.size === 0 &&
1568
- !pendingWatcherBareChange &&
1569
- !pendingWatcherOverflowed &&
1570
- !batch.overflowed
1571
- ? batch
1572
- : null;
1573
- for (const [absolutePath, relativePath] of batch.paths.entries()) {
1574
- pendingWatcherPaths.set(absolutePath, relativePath);
1575
- }
1576
- if (batch.overflowed) {
1577
- pendingWatcherOverflowed = true;
1578
- pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
1579
- pendingWatcherDroppedBytes += batch.droppedBytes ?? 0;
1580
- }
1581
- return;
1582
- }
1583
- if (changedRelPath) {
1584
- pendingWatcherOriginalBatch = null;
1585
- pendingWatcherPaths.set(path.join(hqRoot, changedRelPath), changedRelPath);
1586
- return;
1587
- }
1588
- pendingWatcherOriginalBatch = null;
1589
- pendingWatcherBareChange = true;
1590
- };
1591
- const takePendingWatcherChange = () => {
1592
- const batch = pendingWatcherOriginalBatch !== null && !pendingWatcherOverflowed
1593
- ? pendingWatcherOriginalBatch
1594
- : pendingWatcherPaths.size > 0 || pendingWatcherOverflowed
1595
- ? {
1596
- paths: new Map(pendingWatcherPaths),
1597
- ...(pendingWatcherOverflowed
1598
- ? {
1599
- overflowed: true,
1600
- droppedPaths: pendingWatcherDroppedPaths,
1601
- droppedBytes: pendingWatcherDroppedBytes,
1602
- }
1603
- : {}),
1604
- }
1605
- : null;
1606
- const rel = [...pendingWatcherPaths.values()][0] ?? null;
1607
- const fallbackRel = rel ?? (pendingWatcherBareChange ? null : null);
1608
- pendingWatcherPaths.clear();
1609
- pendingWatcherOriginalBatch = null;
1610
- pendingWatcherBareChange = false;
1611
- pendingWatcherOverflowed = false;
1612
- pendingWatcherDroppedPaths = 0;
1613
- pendingWatcherDroppedBytes = 0;
1614
- return { rel: fallbackRel, batch };
1615
- };
1616
- // ---- pull-on-event receiver (Phase 2, US-009) ------------------------
1617
- // Started after the watcher, disposed before the watcher (mirror of the
1618
- // PushTransport ordering). Dormant by default: the default factory returns
1619
- // a NoopPushReceiver, and even a real receiver stays dormant unless the
1620
- // per-tenant feature flag is on AND a queue URL is provisioned server-side.
1621
- let receiver = null;
1622
- // ---- event-driven publish + pull (Phase 3, US-017/018/019) ------------
1623
- // Brought up asynchronously after the watcher when the rollout gate
1624
- // passes; null until ready (and stays null on startup failure → poll-only).
1625
- let eventSync = null;
1626
- if (eventPush) {
1627
- const clock = deps.clock ?? systemClock;
1628
- const debounceMs = 2000;
1629
- const createWatcher = deps.createWatcher ??
1630
- ((opts) => new TreeWatcher({
1631
- hqRoot: opts.hqRoot,
1632
- debounceMs: opts.debounceMs,
1633
- clock: opts.clock,
1634
- // false in --companies mode so the watch filter matches the sync
1635
- // scope (companies/* are included via .hqinclude); true only for a
1636
- // personal-vault-as-root run.
1637
- personalMode: !companiesMode,
1638
- }));
1639
- watcher = createWatcher({ hqRoot, debounceMs, clock });
1640
- // The driver runs the targeted push when a debounced burst settles. Its
1641
- // concurrency guard collapses a trigger that lands mid-pass; we ALSO gate
1642
- // through `runGuarded` so a poll pass in flight is never overlapped.
1643
- driver = new WatchPushDriver({
1644
- debounceMs: 0, // TreeWatcher already debounces; the driver just guards.
1645
- clock,
1646
- push: async () => {
1647
- if (stopped)
1648
- return;
1649
- // Snapshot accumulated watcher work BEFORE the await. Changes landing
1650
- // mid-pass are accumulated for the NEXT pass instead of overwriting
1651
- // this pass's publish target.
1652
- const { rel, batch: batchForPublish } = takePendingWatcherChange();
1653
- const batchRoutes = batchForPublish
1654
- ? routesForBatch(batchForPublish)
1655
- : new Map();
1656
- let targetedArgv;
1657
- if (batchForPublish?.overflowed || batchRoutes.size > 1) {
1658
- targetedArgv = buildFullFanoutPushArgv(passArgv);
1659
- }
1660
- else {
1661
- const route = batchRoutes.size === 1
1662
- ? [...batchRoutes.values()][0]
1663
- : rel
1664
- ? routeChangeToTarget(rel)
1665
- : { kind: "personal" };
1666
- if (!route)
1667
- return;
1668
- targetedArgv = buildTargetedPushArgv(route, passArgv);
1669
- }
1670
- const result = await runGuarded(targetedArgv);
1671
- // Phase 3 (US-017): publish PushEvents only AFTER the targeted push
1672
- // pass succeeded — an event must never announce bytes that are not
1673
- // in S3 yet. A failed pass publishes nothing; queued passes run after
1674
- // the active pass instead of dropping the watcher-triggered push. Fall
1675
- // back to a single-path batch when the watcher emitted a bare path
1676
- // signal. Overflowed batches deliberately publish nothing because the
1677
- // exact path set was dropped; the full fanout push plus cadence poll is
1678
- // the resync signal.
1679
- if (result === 0 && !stopped && eventSync && !batchForPublish?.overflowed) {
1680
- const batch = batchForPublish ??
1681
- (rel ? { paths: new Map([[path.join(hqRoot, rel), rel]]) } : null);
1682
- if (batch)
1683
- eventSync.publishBatch(batch);
1684
- }
1685
- },
1686
- });
1687
- // A debounced TreeWatcher 'changed' signal feeds the driver, which fires
1688
- // the targeted push after its own (zero) window — i.e. immediately, but
1689
- // still serialized behind any in-flight pass. A path-aware watcher passes
1690
- // the changed relative path so the push targets just its owning company;
1691
- // the bare-signal TreeWatcher leaves it null → personal-vault route.
1692
- watcher.onChange((changedRelPath, batch) => {
1693
- if (stopped)
1694
- return;
1695
- addPendingWatcherChange(changedRelPath, batch);
1696
- driver?.notifyChange();
1697
- });
1698
- watcher.start();
1699
- // Pull-on-event receiver (US-009). The injected SyncEngineFn bridges a
1700
- // received PushEvent → a TARGETED pull pass routed by relativePath, funneled
1701
- // through the SAME `runGuarded` mutex as the poll loop + watcher push so a
1702
- // pull-on-event never overlaps an in-flight pass. Started AFTER the watcher
1703
- // so a live event can't race a half-built daemon. Default factory = noop
1704
- // (dormant); a real SqsPushReceiver is injected by a later release once the
1705
- // server-side per-client SQS queue is provisioned.
1706
- const receiverSyncFn = async (ctx) => {
1707
- if (stopped)
1708
- return;
1709
- const route = routeChangeToTarget(ctx.event.relativePath);
1710
- if (!route)
1711
- return;
1712
- const targetedArgv = buildTargetedPullArgv(route, passArgv);
1713
- const result = await runGuarded(targetedArgv);
1714
- if (result !== 0) {
1715
- throw new Error(`targeted pull failed with exit code ${result}`);
1716
- }
1717
- };
1718
- const createReceiver = deps.createReceiver ?? (() => new NoopPushReceiver());
1719
- receiver = createReceiver({ syncFn: receiverSyncFn, hqRoot });
1720
- // Fire-and-forget start: a receiver's start() kicks off its own poll loop
1721
- // (SqsPushReceiver) or trivially flips connected (noop) — it must NOT block
1722
- // the runner's poll loop from entering. Errors are swallowed; the cadence
1723
- // poll is the safety net regardless of receiver health. (The await-free
1724
- // start also keeps the poll loop's microtask timing identical to the
1725
- // pre-US-009 wiring.)
1726
- void Promise.resolve(receiver.start()).catch(() => undefined);
1727
- // ---- Phase 3: event-driven publish + pull (US-017/018/019) ----------
1728
- // Gated to enrolled accounts (resolveEventSync — exact-email allowlist +
1729
- // HQ_SYNC_EVENT_SYNC override). Brought up asynchronously so a slow
1730
- // subscribe/vend can't delay the first poll pass; until (and unless) the
1731
- // handles resolve, behavior is byte-identical to the gate-off path.
1732
- const getClaims = deps.getIdTokenClaims ?? defaultGetIdTokenClaims;
1733
- const email = getClaims()?.email;
1734
- if (resolveEventSync(email, process.env.HQ_SYNC_EVENT_SYNC)) {
1735
- const getAccessToken = deps.getAccessToken ??
1736
- (() => getValidAccessToken(DEFAULT_COGNITO, { interactive: false }));
1737
- const startES = deps.startEventSync ?? defaultStartEventSync;
1738
- // Entirely async + caught: NOTHING in the Phase 3 bring-up (device-id
1739
- // persistence, tenant resolution, subscribe) may crash or delay the
1740
- // daemon — any failure degrades to poll-only.
1741
- void (async () => {
1742
- const handles = await startES({
1743
- hqRoot,
1744
- apiUrl: DEFAULT_VAULT_API_URL,
1745
- authToken: getAccessToken,
1746
- deviceId: getOrCreateMachineId(hqRoot),
1747
- // The server rejects publishes whose originTenantId mismatches the
1748
- // JWT principal, so resolve the SAME canonical person uid the vault
1749
- // API derives from this token.
1750
- resolveTenantId: async () => {
1751
- const client = new VaultClient({
1752
- apiUrl: DEFAULT_VAULT_API_URL,
1753
- authToken: getAccessToken,
1754
- region: DEFAULT_COGNITO.region,
1755
- });
1756
- const persons = await client.entity.listByType("person");
1757
- const pick = pickCanonicalPersonEntity(persons);
1758
- if (!pick?.uid) {
1759
- throw new Error("no canonical person entity for this account");
1760
- }
1761
- return pick.uid;
1762
- },
1763
- syncFn: receiverSyncFn,
1764
- log: (m) => process.stderr.write(`${m}\n`),
1765
- });
1766
- if (!handles)
1767
- return;
1768
- if (stopped) {
1769
- // Shutdown raced the async bring-up — tear straight down.
1770
- void handles.dispose();
1771
- return;
1772
- }
1773
- eventSync = handles;
1774
- })().catch((err) => {
1775
- process.stderr.write(`event-sync: wiring failed, continuing poll-only: ${describeError(err)}\n`);
1776
- });
1777
- }
1778
- }
1779
- // ---- clean shutdown --------------------------------------------------
1780
- // SIGTERM (menubar stop_daemon) / SIGINT must tear down BOTH the watcher and
1781
- // the poll loop with no leaked timers or fds. We flip `stopped`, dispose the
1782
- // watcher + driver, and let the poll loop observe `stopped` on its next tick.
1783
- let resolveStopped = null;
1784
- const stoppedSignal = new Promise((resolve) => {
1785
- resolveStopped = resolve;
1786
- });
1787
- const shutdown = () => {
1788
- if (stopped)
1789
- return;
1790
- stopped = true;
1791
- resolveStoppedQueue();
1792
- // Dispose the receiver FIRST (mirror of the PushTransport ordering:
1793
- // inbound subscription torn down before the watcher) so no new
1794
- // pull-on-event fires mid-teardown. dispose() is async (it drains the
1795
- // in-flight pull up to its own deadline); fire-and-forget here — the
1796
- // receiver's internal drain + the runGuarded mutex bound the work, and
1797
- // SIGTERM teardown must not block. Errors are swallowed.
1798
- try {
1799
- void receiver?.dispose();
1800
- }
1801
- catch {
1802
- /* ignore */
1803
- }
1804
- // Phase 3 wiring (publish transport + live receiver) — torn down with
1805
- // the same fire-and-forget posture as the Phase 2 receiver above.
1806
- try {
1807
- void eventSync?.dispose();
1808
- }
1809
- catch {
1810
- /* ignore */
1811
- }
1812
- eventSync = null;
1813
- try {
1814
- driver?.dispose();
1815
- }
1816
- catch {
1817
- /* ignore */
1818
- }
1819
- try {
1820
- watcher?.dispose();
1821
- }
1822
- catch {
1823
- /* ignore */
1824
- }
1825
- resolveStopped?.();
1826
- };
1827
- const onShutdownSignal = deps.onShutdownSignal ??
1828
- ((handler) => {
1829
- const wrapped = () => handler();
1830
- process.on("SIGTERM", wrapped);
1831
- process.on("SIGINT", wrapped);
1832
- return () => {
1833
- process.off("SIGTERM", wrapped);
1834
- process.off("SIGINT", wrapped);
1835
- };
1836
- });
1837
- detachSignal = onShutdownSignal(shutdown);
1838
- try {
1839
- while (!stopped) {
1840
- const result = await runGuarded(passArgv);
1841
- if (result !== 0) {
1842
- return result;
1843
- }
1844
- // Sleep the poll interval, but wake early on shutdown so SIGTERM stops
1845
- // the loop promptly instead of waiting out a 10-minute cycle.
1846
- await Promise.race([sleep(pollMs), stoppedSignal]);
1847
- }
1848
- return 0;
1849
- }
1850
- finally {
1851
- shutdown();
1852
- detachSignal?.();
1853
- }
1854
- }
1855
- /**
1856
- * Extract the `--hq-root` / `--on-conflict` pair-flags from a base argv so a
1857
- * re-targeted push pass inherits the same root and conflict policy. Pure
1858
- * helper used by the event-push targeted-push composer.
1859
- */
1860
- function carriedFlags(baseArgv) {
1861
- const carried = [];
1862
- for (let i = 0; i < baseArgv.length; i++) {
1863
- const a = baseArgv[i];
1864
- if (a === "--hq-root" || a === "--on-conflict") {
1865
- carried.push(a);
1866
- if (baseArgv[i + 1] !== undefined)
1867
- carried.push(baseArgv[++i]);
1868
- }
1869
- }
1870
- return carried;
789
+ return runWatchLoop(argv, parsed, deps, runtime);
1871
790
  }
1872
791
  if (isDirectInvocation) {
1873
792
  runRunnerWithLoop(process.argv.slice(2))