@indigoai-us/hq-cloud 6.11.11 → 6.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +35 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -0
- package/dist/bin/sync-runner-company.js +290 -0
- package/dist/bin/sync-runner-company.js.map +1 -0
- package/dist/bin/sync-runner-events.d.ts +12 -0
- package/dist/bin/sync-runner-events.d.ts.map +1 -0
- package/dist/bin/sync-runner-events.js +12 -0
- package/dist/bin/sync-runner-events.js.map +1 -0
- package/dist/bin/sync-runner-planning.d.ts +53 -0
- package/dist/bin/sync-runner-planning.d.ts.map +1 -0
- package/dist/bin/sync-runner-planning.js +59 -0
- package/dist/bin/sync-runner-planning.js.map +1 -0
- package/dist/bin/sync-runner-rollup.d.ts +24 -0
- package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
- package/dist/bin/sync-runner-rollup.js +46 -0
- package/dist/bin/sync-runner-rollup.js.map +1 -0
- package/dist/bin/sync-runner-telemetry.d.ts +5 -0
- package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
- package/dist/bin/sync-runner-telemetry.js +5 -0
- package/dist/bin/sync-runner-telemetry.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-loop.js +372 -0
- package/dist/bin/sync-runner-watch-loop.js.map +1 -0
- package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-routes.js +74 -0
- package/dist/bin/sync-runner-watch-routes.js.map +1 -0
- package/dist/bin/sync-runner.d.ts +5 -54
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +76 -978
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +265 -11
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +34 -17
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +39 -5
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +75 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +45 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +320 -170
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/share.d.ts +2 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +276 -660
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +30 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +28 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +541 -748
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +382 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +55 -10
- package/dist/cognito-auth.js.map +1 -1
- package/dist/cognito-auth.test.js +61 -0
- package/dist/cognito-auth.test.js.map +1 -1
- package/dist/daemon-worker.d.ts +2 -2
- package/dist/daemon-worker.js +3 -3
- package/dist/daemon-worker.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +93 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +59 -0
- package/dist/journal.test.js.map +1 -1
- package/dist/machine-auth.test.js +60 -2
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +37 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +149 -30
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +121 -0
- package/dist/object-io.test.js.map +1 -1
- package/dist/operation-lock.d.ts +8 -8
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +99 -32
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +51 -4
- package/dist/operation-lock.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +8 -2
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +34 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/prefix-coalesce.d.ts +20 -9
- package/dist/prefix-coalesce.d.ts.map +1 -1
- package/dist/prefix-coalesce.js +124 -28
- package/dist/prefix-coalesce.js.map +1 -1
- package/dist/prefix-coalesce.test.js +57 -2
- package/dist/prefix-coalesce.test.js.map +1 -1
- package/dist/remote-pull.d.ts +8 -3
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +85 -16
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +213 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +197 -116
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +109 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/scope-shrink.d.ts +3 -2
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +1 -1
- package/dist/scope-shrink.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +69 -9
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +86 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +34 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +73 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/metrics.d.ts +17 -1
- package/dist/sync/metrics.d.ts.map +1 -1
- package/dist/sync/metrics.js +32 -1
- package/dist/sync/metrics.js.map +1 -1
- package/dist/sync/metrics.test.js +74 -1
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +15 -7
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +12 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +45 -17
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +67 -1
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync-core.d.ts +27 -0
- package/dist/sync-core.d.ts.map +1 -0
- package/dist/sync-core.js +54 -0
- package/dist/sync-core.js.map +1 -0
- package/dist/telemetry.d.ts +1 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +59 -6
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +74 -0
- package/dist/telemetry.test.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +284 -36
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +59 -0
- package/dist/vault-client.test.js.map +1 -1
- package/dist/watcher.d.ts +38 -20
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +155 -143
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +103 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner-company.ts +350 -0
- package/src/bin/sync-runner-events.ts +25 -0
- package/src/bin/sync-runner-planning.ts +121 -0
- package/src/bin/sync-runner-rollup.ts +72 -0
- package/src/bin/sync-runner-telemetry.ts +8 -0
- package/src/bin/sync-runner-watch-loop.ts +443 -0
- package/src/bin/sync-runner-watch-routes.ts +86 -0
- package/src/bin/sync-runner.test.ts +298 -11
- package/src/bin/sync-runner.ts +99 -1054
- package/src/cli/reindex.test.ts +41 -3
- package/src/cli/reindex.ts +35 -19
- package/src/cli/rescue-classify-ordering.test.ts +81 -0
- package/src/cli/rescue-core.ts +400 -165
- package/src/cli/share.test.ts +38 -0
- package/src/cli/share.ts +420 -693
- package/src/cli/sync.test.ts +460 -1
- package/src/cli/sync.ts +788 -825
- package/src/cognito-auth.test.ts +77 -0
- package/src/cognito-auth.ts +73 -11
- package/src/daemon-worker.ts +3 -3
- package/src/index.ts +8 -0
- package/src/journal.test.ts +72 -0
- package/src/journal.ts +95 -8
- package/src/machine-auth.test.ts +64 -2
- package/src/object-io.test.ts +142 -0
- package/src/object-io.ts +183 -31
- package/src/operation-lock.test.ts +63 -4
- package/src/operation-lock.ts +99 -31
- package/src/personal-vault.test.ts +42 -0
- package/src/personal-vault.ts +8 -2
- package/src/prefix-coalesce.test.ts +71 -1
- package/src/prefix-coalesce.ts +155 -30
- package/src/remote-pull.test.ts +235 -1
- package/src/remote-pull.ts +106 -18
- package/src/s3.test.ts +126 -0
- package/src/s3.ts +237 -122
- package/src/scope-shrink.ts +6 -3
- package/src/skill-telemetry.test.ts +109 -0
- package/src/skill-telemetry.ts +82 -14
- package/src/sync/event-sync.test.ts +75 -0
- package/src/sync/event-sync.ts +54 -1
- package/src/sync/metrics.test.ts +81 -0
- package/src/sync/metrics.ts +59 -4
- package/src/sync/pull-scope.ts +23 -7
- package/src/sync/push-receiver.test.ts +73 -1
- package/src/sync/push-receiver.ts +56 -20
- package/src/sync-core.ts +58 -0
- package/src/telemetry.test.ts +85 -0
- package/src/telemetry.ts +69 -6
- package/src/types.ts +8 -0
- package/src/vault-client.test.ts +74 -0
- package/src/vault-client.ts +395 -43
- package/src/watcher.test.ts +117 -0
- package/src/watcher.ts +215 -174
package/src/bin/sync-runner.ts
CHANGED
|
@@ -106,35 +106,32 @@ import { collectAndSendTelemetry } from "../telemetry.js";
|
|
|
106
106
|
import { collectAndSendSkillTelemetry } from "../skill-telemetry.js";
|
|
107
107
|
import { reindexAfterSync } from "../qmd-reindex.js";
|
|
108
108
|
import { pruneConflictIndex } from "../lib/conflict-index.js";
|
|
109
|
-
import {
|
|
110
|
-
withOperationLock,
|
|
111
|
-
OperationLockedError,
|
|
112
|
-
OPERATION_LOCKED_EXIT,
|
|
113
|
-
} from "../operation-lock.js";
|
|
114
|
-
import { describeError } from "../lib/describe-error.js";
|
|
115
109
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
110
|
+
import type { Clock, TreeChangeBatch } from "../watcher.js";
|
|
111
|
+
import type { PushReceiver, SyncEngineFn } from "../sync/push-receiver.js";
|
|
116
112
|
import {
|
|
117
|
-
TreeWatcher,
|
|
118
|
-
WatchPushDriver,
|
|
119
|
-
systemClock,
|
|
120
|
-
type Clock,
|
|
121
|
-
type TreeChangeBatch,
|
|
122
|
-
} from "../watcher.js";
|
|
123
|
-
import {
|
|
124
|
-
NoopPushReceiver,
|
|
125
|
-
type PushReceiver,
|
|
126
|
-
type SyncEngineFn,
|
|
127
|
-
} from "../sync/push-receiver.js";
|
|
128
|
-
import {
|
|
129
|
-
resolveEventSync,
|
|
130
|
-
startEventSync as defaultStartEventSync,
|
|
131
113
|
type EventSyncHandles,
|
|
132
114
|
type StartEventSyncOptions,
|
|
133
115
|
} from "../sync/event-sync.js";
|
|
116
|
+
import { migratePersonalVaultJournal } from "../journal.js";
|
|
117
|
+
import { createRunnerEmitter } from "./sync-runner-events.js";
|
|
134
118
|
import {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
119
|
+
buildFanoutPlan,
|
|
120
|
+
emitFanoutPlan,
|
|
121
|
+
resolveMembershipsForRun,
|
|
122
|
+
} from "./sync-runner-planning.js";
|
|
123
|
+
import { executeCompanyFanout } from "./sync-runner-company.js";
|
|
124
|
+
import { rollupAllComplete } from "./sync-runner-rollup.js";
|
|
125
|
+
import { emitTelemetry } from "./sync-runner-telemetry.js";
|
|
126
|
+
import {
|
|
127
|
+
runOneShotWithOperationLock,
|
|
128
|
+
runWatchLoop,
|
|
129
|
+
} from "./sync-runner-watch-loop.js";
|
|
130
|
+
export {
|
|
131
|
+
buildTargetedPullArgv,
|
|
132
|
+
buildTargetedPushArgv,
|
|
133
|
+
routeChangeToTarget,
|
|
134
|
+
} from "./sync-runner-watch-routes.js";
|
|
138
135
|
|
|
139
136
|
/**
|
|
140
137
|
* Sync direction for a run.
|
|
@@ -486,6 +483,8 @@ export interface RunnerDeps {
|
|
|
486
483
|
createVaultClient?: (config: VaultServiceConfig) => VaultClientSurface;
|
|
487
484
|
/** Sync function. Defaults to `cli/sync.sync`. */
|
|
488
485
|
sync?: (options: SyncOptions) => Promise<SyncResult>;
|
|
486
|
+
/** Internal: set when runRunner is invoked under the per-root operation lock. */
|
|
487
|
+
operationLockAlreadyHeld?: boolean;
|
|
489
488
|
/** Share function (push phase). Defaults to `cli/share.share`. */
|
|
490
489
|
share?: (options: ShareOptions) => Promise<ShareResult>;
|
|
491
490
|
/**
|
|
@@ -891,14 +890,7 @@ export async function runRunner(
|
|
|
891
890
|
// events. The menubar's `HQ_CLOUD_VERSION` pin gates which runner
|
|
892
891
|
// they spawn, so old menubars stay on the previous runner version
|
|
893
892
|
// even after this one is published.
|
|
894
|
-
const
|
|
895
|
-
"error",
|
|
896
|
-
"auth-error",
|
|
897
|
-
]);
|
|
898
|
-
const emit = (event: RunnerEvent): void => {
|
|
899
|
-
const stream = ERROR_TYPES.has(event.type) ? stderr : stdout;
|
|
900
|
-
stream.write(`${JSON.stringify(event)}\n`);
|
|
901
|
-
};
|
|
893
|
+
const emit = createRunnerEmitter({ stdout, stderr });
|
|
902
894
|
|
|
903
895
|
// ---- argv -------------------------------------------------------------
|
|
904
896
|
const parsed = parseArgs(argv);
|
|
@@ -990,36 +982,21 @@ export async function runRunner(
|
|
|
990
982
|
// ---- resolve targets --------------------------------------------------
|
|
991
983
|
let memberships: Pick<Membership, "companyUid">[];
|
|
992
984
|
try {
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
await runClaimDance(client, claims, stderr);
|
|
1007
|
-
}
|
|
1008
|
-
|
|
1009
|
-
memberships = await listMembershipsWithRetry(client);
|
|
1010
|
-
if (memberships.length === 0) {
|
|
1011
|
-
// Truly empty — still a valid state (no memberships = nothing to
|
|
1012
|
-
// sync). The tray will show a friendly "create your first company"
|
|
1013
|
-
// CTA rather than an alarm banner.
|
|
1014
|
-
emit({ type: "setup-needed" });
|
|
1015
|
-
return 0;
|
|
1016
|
-
}
|
|
1017
|
-
} else {
|
|
1018
|
-
// Single-company mode: fabricate a minimal membership so the fanout
|
|
1019
|
-
// loop below treats it uniformly. We don't need to hit
|
|
1020
|
-
// /membership/me — the caller already told us which company.
|
|
1021
|
-
memberships = [{ companyUid: parsed.company! }];
|
|
985
|
+
const resolution = await resolveMembershipsForRun({
|
|
986
|
+
personal: parsed.personal,
|
|
987
|
+
companies: parsed.companies,
|
|
988
|
+
company: parsed.company,
|
|
989
|
+
client,
|
|
990
|
+
claims,
|
|
991
|
+
stderr,
|
|
992
|
+
runClaimDance,
|
|
993
|
+
listMemberships: listMembershipsWithRetry,
|
|
994
|
+
});
|
|
995
|
+
if (resolution.status === "setup-needed") {
|
|
996
|
+
emit({ type: "setup-needed" });
|
|
997
|
+
return 0;
|
|
1022
998
|
}
|
|
999
|
+
memberships = resolution.memberships;
|
|
1023
1000
|
} catch (err) {
|
|
1024
1001
|
if (err instanceof VaultAuthError) {
|
|
1025
1002
|
emit({
|
|
@@ -1039,73 +1016,21 @@ export async function runRunner(
|
|
|
1039
1016
|
return 1;
|
|
1040
1017
|
}
|
|
1041
1018
|
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
}> = [];
|
|
1054
|
-
for (const m of memberships) {
|
|
1055
|
-
let slug = m.companyUid;
|
|
1056
|
-
let name: string | undefined;
|
|
1057
|
-
try {
|
|
1058
|
-
const info = await client.entity.get(m.companyUid);
|
|
1059
|
-
slug = info.slug || m.companyUid;
|
|
1060
|
-
name = info.name;
|
|
1061
|
-
} catch {
|
|
1062
|
-
// Best-effort — keep UID as the display identifier.
|
|
1063
|
-
}
|
|
1064
|
-
plan.push({ uid: m.companyUid, slug, ...(name ? { name } : {}) });
|
|
1065
|
-
}
|
|
1066
|
-
|
|
1067
|
-
if (
|
|
1068
|
-
(parsed.companies || parsed.personal) &&
|
|
1069
|
-
!resolveSkipPersonal(parsed.skipPersonal)
|
|
1070
|
-
) {
|
|
1071
|
-
// Personal-target fanout slot. Skipped entirely when --skip-personal
|
|
1072
|
-
// (or HQ_SYNC_SKIP_PERSONAL=1) is set — see resolveSkipPersonal doc for
|
|
1073
|
-
// the rationale (menubar opt-out for users who only want company sync).
|
|
1074
|
-
// When skipped, the fanout-plan event below carries only company
|
|
1075
|
-
// memberships and no "personal" slug; downstream consumers (menubar
|
|
1076
|
-
// workspaces row, status surfaces) should already tolerate that
|
|
1077
|
-
// shape since pre-5.25 fanout often had it (a user with no person
|
|
1078
|
-
// entity yet, or before the canonical-person-entity machinery landed).
|
|
1079
|
-
//
|
|
1080
|
-
// `--personal` mode reaches this block with an empty `plan` and
|
|
1081
|
-
// empty `memberships`; only the personal target gets added below.
|
|
1082
|
-
const persons = await client.entity.listByType("person");
|
|
1083
|
-
const pick = pickCanonicalPersonEntity(persons);
|
|
1084
|
-
if (pick?.bucketName) {
|
|
1085
|
-
plan.push({
|
|
1086
|
-
slug: "personal",
|
|
1087
|
-
uid: pick.uid,
|
|
1088
|
-
bucketName: pick.bucketName,
|
|
1089
|
-
personalMode: true,
|
|
1090
|
-
// Reserved sentinel slug — decoupled from the `companies/personal`
|
|
1091
|
-
// company slug to avoid sharing `sync-journal.personal.json`. The
|
|
1092
|
-
// display `slug` stays "personal" (menubar UI); only the journal
|
|
1093
|
-
// shard changes. See PERSONAL_VAULT_JOURNAL_SLUG for the full
|
|
1094
|
-
// root-cause writeup.
|
|
1095
|
-
journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
|
|
1096
|
-
});
|
|
1097
|
-
} else if (parsed.personal) {
|
|
1098
|
-
// --personal mode with no canonical personal entity → setup-needed.
|
|
1099
|
-
// (In --companies mode this state is silent — companies still sync
|
|
1100
|
-
// and the missing personal target just shows an empty workspaces row.
|
|
1101
|
-
// In --personal mode it's the ONLY signal, so it gets surfaced as
|
|
1102
|
-
// setup-needed with the same shape as the empty-memberships case.)
|
|
1103
|
-
emit({ type: "setup-needed" });
|
|
1104
|
-
return 0;
|
|
1105
|
-
}
|
|
1019
|
+
const targetPlan = await buildFanoutPlan({
|
|
1020
|
+
memberships,
|
|
1021
|
+
companies: parsed.companies,
|
|
1022
|
+
personal: parsed.personal,
|
|
1023
|
+
skipPersonal: parsed.skipPersonal,
|
|
1024
|
+
client,
|
|
1025
|
+
resolveSkipPersonal,
|
|
1026
|
+
});
|
|
1027
|
+
if (targetPlan.status === "setup-needed") {
|
|
1028
|
+
emit({ type: "setup-needed" });
|
|
1029
|
+
return 0;
|
|
1106
1030
|
}
|
|
1031
|
+
const plan = targetPlan.plan;
|
|
1107
1032
|
|
|
1108
|
-
emit
|
|
1033
|
+
emitFanoutPlan(emit, plan);
|
|
1109
1034
|
|
|
1110
1035
|
// One-time seed of the reserved personal-vault journal from the legacy
|
|
1111
1036
|
// "personal" journal, so switching the vault slot off the colliding slug
|
|
@@ -1117,539 +1042,46 @@ export async function runRunner(
|
|
|
1117
1042
|
// ---- fanout -----------------------------------------------------------
|
|
1118
1043
|
const syncFn = deps.sync ?? defaultSync;
|
|
1119
1044
|
const shareFn = deps.share ?? defaultShare;
|
|
1120
|
-
const
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
// at event time, so progress events route to the right column.
|
|
1137
|
-
type CompanyStatus = "complete" | "aborted" | "errored";
|
|
1138
|
-
interface CompanyState {
|
|
1139
|
-
company: string;
|
|
1140
|
-
status: CompanyStatus;
|
|
1141
|
-
filesDownloaded: number;
|
|
1142
|
-
bytesDownloaded: number;
|
|
1143
|
-
filesUploaded: number;
|
|
1144
|
-
bytesUploaded: number;
|
|
1145
|
-
}
|
|
1146
|
-
const stateByCompany = new Map<string, CompanyState>();
|
|
1147
|
-
|
|
1148
|
-
for (const target of plan) {
|
|
1149
|
-
const companyLabel = target.slug;
|
|
1150
|
-
const state: CompanyState = {
|
|
1151
|
-
company: companyLabel,
|
|
1152
|
-
// Default to "errored" so a throw before any complete-or-clean-abort
|
|
1153
|
-
// path (the original bug) leaves the entry flagged as not-clean. The
|
|
1154
|
-
// success/clean-abort paths overwrite this before the loop body exits.
|
|
1155
|
-
status: "errored",
|
|
1156
|
-
filesDownloaded: 0,
|
|
1157
|
-
bytesDownloaded: 0,
|
|
1158
|
-
filesUploaded: 0,
|
|
1159
|
-
bytesUploaded: 0,
|
|
1160
|
-
};
|
|
1161
|
-
stateByCompany.set(companyLabel, state);
|
|
1162
|
-
|
|
1163
|
-
// Which phase is currently emitting `progress` events. Mutable closure so
|
|
1164
|
-
// tagAndEmit (defined once below) reads the latest value when each event
|
|
1165
|
-
// fires. "pull" is the default for back-compat with pull-only runs.
|
|
1166
|
-
let activePhase: "pull" | "push" = doPush && !doPull ? "push" : "pull";
|
|
1167
|
-
|
|
1168
|
-
// Per-company event tagger — shared by push and pull phases so progress
|
|
1169
|
-
// rows land on the right company regardless of which phase emitted them.
|
|
1170
|
-
// Also updates `state` for `progress` events so the rollup has accurate
|
|
1171
|
-
// partial counts even if the sync function throws before returning.
|
|
1172
|
-
const tagAndEmit = (event: SyncProgressEvent): void => {
|
|
1173
|
-
if (event.type === "plan") {
|
|
1174
|
-
emit({
|
|
1175
|
-
type: "plan",
|
|
1176
|
-
company: companyLabel,
|
|
1177
|
-
filesToDownload: event.filesToDownload,
|
|
1178
|
-
bytesToDownload: event.bytesToDownload,
|
|
1179
|
-
filesToUpload: event.filesToUpload,
|
|
1180
|
-
bytesToUpload: event.bytesToUpload,
|
|
1181
|
-
filesToSkip: event.filesToSkip,
|
|
1182
|
-
filesToConflict: event.filesToConflict,
|
|
1183
|
-
filesToDelete: event.filesToDelete,
|
|
1184
|
-
});
|
|
1185
|
-
} else if (event.type === "progress") {
|
|
1186
|
-
if (activePhase === "push") {
|
|
1187
|
-
state.filesUploaded += 1;
|
|
1188
|
-
state.bytesUploaded += event.bytes;
|
|
1189
|
-
} else {
|
|
1190
|
-
state.filesDownloaded += 1;
|
|
1191
|
-
state.bytesDownloaded += event.bytes;
|
|
1192
|
-
}
|
|
1193
|
-
emit({
|
|
1194
|
-
type: "progress",
|
|
1195
|
-
company: companyLabel,
|
|
1196
|
-
path: event.path,
|
|
1197
|
-
bytes: event.bytes,
|
|
1198
|
-
// Stamp the transfer direction from the in-flight phase so the
|
|
1199
|
-
// menubar can label each file uploaded vs downloaded. The inner
|
|
1200
|
-
// share()/sync() emitters don't know the phase — only this tagger,
|
|
1201
|
-
// which mirrors the up/down counter bump above.
|
|
1202
|
-
direction: activePhase === "push" ? "up" : "down",
|
|
1203
|
-
...(event.message ? { message: event.message } : {}),
|
|
1204
|
-
...(event.deleted ? { deleted: event.deleted } : {}),
|
|
1205
|
-
});
|
|
1206
|
-
} else if (event.type === "conflict") {
|
|
1207
|
-
emit({
|
|
1208
|
-
type: "conflict",
|
|
1209
|
-
company: companyLabel,
|
|
1210
|
-
path: event.path,
|
|
1211
|
-
direction: event.direction,
|
|
1212
|
-
resolution: event.resolution,
|
|
1213
|
-
});
|
|
1214
|
-
} else if (event.type === "error") {
|
|
1215
|
-
emit({
|
|
1216
|
-
type: "error",
|
|
1217
|
-
company: companyLabel,
|
|
1218
|
-
path: event.path,
|
|
1219
|
-
message: event.message,
|
|
1220
|
-
});
|
|
1221
|
-
} else if (event.type === "new-files") {
|
|
1222
|
-
emit({
|
|
1223
|
-
type: "new-files",
|
|
1224
|
-
company: companyLabel,
|
|
1225
|
-
files: event.files,
|
|
1226
|
-
});
|
|
1227
|
-
} else if (event.type === "scope-excluded") {
|
|
1228
|
-
// Push-side ACL scope exclusions — surface the named paths tagged to
|
|
1229
|
-
// this company so the menubar/CLI can show "N skipped, outside your
|
|
1230
|
-
// access" instead of the file silently never uploading.
|
|
1231
|
-
emit({
|
|
1232
|
-
type: "scope-excluded",
|
|
1233
|
-
company: companyLabel,
|
|
1234
|
-
count: event.count,
|
|
1235
|
-
samplePaths: event.samplePaths,
|
|
1236
|
-
});
|
|
1237
|
-
}
|
|
1238
|
-
};
|
|
1239
|
-
|
|
1240
|
-
try {
|
|
1241
|
-
let pushResult: ShareResult = {
|
|
1242
|
-
filesUploaded: 0,
|
|
1243
|
-
bytesUploaded: 0,
|
|
1244
|
-
filesSkipped: 0,
|
|
1245
|
-
filesDeleted: 0,
|
|
1246
|
-
filesTombstoned: 0,
|
|
1247
|
-
filesRefusedStale: 0,
|
|
1248
|
-
filesRefusedStalePaths: [],
|
|
1249
|
-
filesSuppressedByTombstone: 0,
|
|
1250
|
-
filesExcludedByPolicy: 0,
|
|
1251
|
-
filesExcludedByScope: 0,
|
|
1252
|
-
conflictPaths: [],
|
|
1253
|
-
aborted: false,
|
|
1254
|
-
};
|
|
1255
|
-
let pullResult: SyncResult = {
|
|
1256
|
-
filesDownloaded: 0,
|
|
1257
|
-
bytesDownloaded: 0,
|
|
1258
|
-
filesSkipped: 0,
|
|
1259
|
-
conflicts: 0,
|
|
1260
|
-
conflictPaths: [],
|
|
1261
|
-
aborted: false,
|
|
1262
|
-
newFiles: [],
|
|
1263
|
-
newFilesCount: 0,
|
|
1264
|
-
filesExcludedByPolicy: 0,
|
|
1265
|
-
filesTombstoned: 0,
|
|
1266
|
-
filesOutOfScope: 0,
|
|
1267
|
-
scopeOrphansRemoved: 0,
|
|
1268
|
-
scopeOrphansBlocked: 0,
|
|
1269
|
-
};
|
|
1270
|
-
|
|
1271
|
-
// Push first so a subsequent pull doesn't overwrite files we were about
|
|
1272
|
-
// to broadcast. Company targets walk `companies/{slug}/`; the personal
|
|
1273
|
-
// target walks every top-level entry under hqRoot minus the exclusion
|
|
1274
|
-
// list (see PERSONAL_VAULT_EXCLUDED_TOP_LEVEL). `skipUnchanged: true`
|
|
1275
|
-
// keeps both cases efficient on re-runs.
|
|
1276
|
-
// Shared between push and pull for the personal slot. Hoisted out of
|
|
1277
|
-
// the if-blocks below so doPush + doPull see the same set:
|
|
1278
|
-
//
|
|
1279
|
-
// `HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL=1` opts INTO the
|
|
1280
|
-
// cloud:false → personal-bucket fallback. Default OFF keeps the
|
|
1281
|
-
// personal vault identical to the pre-5.20 shape until operators
|
|
1282
|
-
// explicitly enable it.
|
|
1283
|
-
//
|
|
1284
|
-
// `teamSyncedSlugs` is the slug set the operator currently has
|
|
1285
|
-
// active team-bucket Memberships for, derived from the live plan.
|
|
1286
|
-
// Used by:
|
|
1287
|
-
// • push: filter `companies/` subdir enumeration so a team-synced
|
|
1288
|
-
// company never rides the personal slot; build
|
|
1289
|
-
// `decommissionPrefixes` so share() sweeps orphan keys for
|
|
1290
|
-
// any slug that's now team-backed.
|
|
1291
|
-
// • pull: filter `listRemoteFiles` so pre-decommission orphan
|
|
1292
|
-
// keys at `companies/{team-synced-slug}/...` don't get
|
|
1293
|
-
// re-downloaded into the disk paths the team-bucket pull
|
|
1294
|
-
// now manages.
|
|
1295
|
-
const includeLocalCompanies =
|
|
1296
|
-
process.env.HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL === "1";
|
|
1297
|
-
const teamSyncedSlugs = new Set(
|
|
1298
|
-
plan
|
|
1299
|
-
.filter((p) => p.personalMode !== true)
|
|
1300
|
-
.map((p) => p.slug),
|
|
1301
|
-
);
|
|
1302
|
-
|
|
1303
|
-
// Resolve the membership's effective ACL scope ONCE so BOTH the push and
|
|
1304
|
-
// pull legs respect the granted prefixes. The vault vends a child
|
|
1305
|
-
// credential scoped to these prefixes; without filtering the PUSH plan to
|
|
1306
|
-
// them, share() would HEAD/PUT keys outside the grant and the server's
|
|
1307
|
-
// correct 403 (SCOPE_EXCEEDS_PARENT) would abort the WHOLE company with a
|
|
1308
|
-
// generic error + exit 2. Personal-vault legs have no membership
|
|
1309
|
-
// sync-config — they stay full-scope ("all"). Degrades to "all" on any
|
|
1310
|
-
// error (a transient failure must never silently filter/prune the tree).
|
|
1311
|
-
// Hoisted above the push block (it used to be resolved only for pull) so
|
|
1312
|
-
// push gets the same scope; the pull leg below reuses this value.
|
|
1313
|
-
const scope: PullScope =
|
|
1314
|
-
target.personalMode === true
|
|
1315
|
-
? { syncMode: "all" }
|
|
1316
|
-
: await resolvePullScope(
|
|
1317
|
-
client,
|
|
1318
|
-
target.uid,
|
|
1319
|
-
target.slug,
|
|
1320
|
-
parsed.hqRoot,
|
|
1321
|
-
);
|
|
1322
|
-
|
|
1323
|
-
if (doPush) {
|
|
1324
|
-
activePhase = "push";
|
|
1325
|
-
// For the personal slot we hand share() both (a) the top-level
|
|
1326
|
-
// hqRoot entries that are part of the personal vault and (b) any
|
|
1327
|
-
// `companies/{slug}/` subdirs the user has opted into the personal
|
|
1328
|
-
// bucket via `cloud: false` in `company.yaml`. Slugs that already
|
|
1329
|
-
// own a team bucket (anything else in `plan`) are excluded so a
|
|
1330
|
-
// company is never double-written.
|
|
1331
|
-
const pushPaths = target.personalMode === true
|
|
1332
|
-
? computePersonalVaultPaths(parsed.hqRoot, {
|
|
1333
|
-
includeLocalCompanies,
|
|
1334
|
-
teamSyncedSlugs,
|
|
1335
|
-
})
|
|
1336
|
-
: [path.join(parsed.hqRoot, "companies", target.slug)];
|
|
1337
|
-
// For the personal slot, hand share() a list of `companies/{slug}/`
|
|
1338
|
-
// prefixes for every team-synced slug. If the personal-bucket
|
|
1339
|
-
// journal still holds entries under any of those prefixes — i.e.
|
|
1340
|
-
// the company used to ride the personal-bucket fallback and was
|
|
1341
|
-
// since promoted to its own team bucket — share() will sweep
|
|
1342
|
-
// those orphans via DeleteObject + journal removal. Unconditional
|
|
1343
|
-
// (not gated on `HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL`) so cleanup
|
|
1344
|
-
// still runs after an operator turns the feature off: the on-ramp
|
|
1345
|
-
// is opt-in but the off-ramp is always safe to traverse.
|
|
1346
|
-
const decommissionPrefixes = target.personalMode === true
|
|
1347
|
-
? Array.from(teamSyncedSlugs).map((slug) => `companies/${slug}`)
|
|
1348
|
-
: undefined;
|
|
1349
|
-
pushResult = await shareFn({
|
|
1350
|
-
paths: pushPaths,
|
|
1351
|
-
company: target.uid,
|
|
1352
|
-
vaultConfig,
|
|
1353
|
-
hqRoot: parsed.hqRoot,
|
|
1354
|
-
onConflict: parsed.onConflict,
|
|
1355
|
-
skipUnchanged: true,
|
|
1356
|
-
// Local deletes propagate to S3 as soft deletes (versioning is on
|
|
1357
|
-
// — DeleteObject writes a delete-marker, prior versions remain
|
|
1358
|
-
// recoverable). Without this, a deleted file resurfaces on the
|
|
1359
|
-
// next pull because the remote object is still listable.
|
|
1360
|
-
//
|
|
1361
|
-
// Policy default in 5.24 is `owned-only` (pre-5.24 behavior;
|
|
1362
|
-
// preserved for the soak window). `HQ_SYNC_DELETE_POLICY` env
|
|
1363
|
-
// can opt INTO the safer `currency-gated` (per-file HEAD + ETag
|
|
1364
|
-
// verification) or the unsafe `all` (emergency reconcile only).
|
|
1365
|
-
// Default flips to `currency-gated` in 5.25 after at least one
|
|
1366
|
-
// machine has soaked the new path. Both personal and company
|
|
1367
|
-
// targets use the same resolver — same engine, same flip.
|
|
1368
|
-
propagateDeletes: true,
|
|
1369
|
-
propagateDeletePolicy: resolveDeletePolicy(),
|
|
1370
|
-
onEvent: tagAndEmit,
|
|
1371
|
-
...(uploadAuthor ? { author: uploadAuthor } : {}),
|
|
1372
|
-
// Mirror the pull-side seam: only spread these for the personal
|
|
1373
|
-
// slot so company-target args stay identical to the pre-Slice-2
|
|
1374
|
-
// shape (the "no personalMode/journalSlug keys" regression test
|
|
1375
|
-
// in sync-runner.test.ts pins that contract).
|
|
1376
|
-
...(target.personalMode !== undefined ? { personalMode: target.personalMode } : {}),
|
|
1377
|
-
...(target.journalSlug !== undefined ? { journalSlug: target.journalSlug } : {}),
|
|
1378
|
-
...(decommissionPrefixes && decommissionPrefixes.length > 0
|
|
1379
|
-
? { decommissionPrefixes }
|
|
1380
|
-
: {}),
|
|
1381
|
-
// US-005 symmetry: scope the PUSH plan to the membership's granted
|
|
1382
|
-
// ACL prefixes so out-of-scope keys are skipped (and surfaced via a
|
|
1383
|
-
// `scope-excluded` event) instead of drawing the server's 403 and
|
|
1384
|
-
// aborting the company. `undefined` for `syncMode: "all"` (owner /
|
|
1385
|
-
// personal) → no scope filter, identical to the pre-fix shape so the
|
|
1386
|
-
// "company-target args" contract test stays green.
|
|
1387
|
-
...(scope.prefixSet !== undefined ? { prefixSet: scope.prefixSet } : {}),
|
|
1388
|
-
});
|
|
1389
|
-
}
|
|
1390
|
-
|
|
1391
|
-
// Pull runs unless the push phase aborted on conflict — aborted means
|
|
1392
|
-
// the user has local edits + remote drift; blindly pulling would erase
|
|
1393
|
-
// whichever side `--on-conflict abort` just protected.
|
|
1394
|
-
if (doPull && !pushResult.aborted) {
|
|
1395
|
-
activePhase = "pull";
|
|
1396
|
-
// US-005: the pull only materializes in-scope keys (and prunes clean
|
|
1397
|
-
// orphans when scope shrank). Reuse the `scope` resolved once above so
|
|
1398
|
-
// push and pull apply the SAME granted prefixes and we avoid a second
|
|
1399
|
-
// `listMyExplicitGrants` round-trip per company.
|
|
1400
|
-
const pullScope: PullScope = scope;
|
|
1401
|
-
pullResult = await syncFn({
|
|
1402
|
-
company: target.uid,
|
|
1403
|
-
vaultConfig,
|
|
1404
|
-
hqRoot: parsed.hqRoot,
|
|
1405
|
-
onConflict: parsed.onConflict,
|
|
1406
|
-
syncMode: pullScope.syncMode,
|
|
1407
|
-
// The menubar runner can take no interactive flag, so a scope shrink
|
|
1408
|
-
// must NEVER throw here (the old `ScopeShrinkBlockedError` → exit 2
|
|
1409
|
-
// was the permanent wedge in DEV-1768). Self-heal non-destructively:
|
|
1410
|
-
// dirty out-of-scope files stay on disk + un-tracked, clean ones are
|
|
1411
|
-
// quarantined (recoverable). This also clears an already-wedged
|
|
1412
|
-
// journal — seeded by a buggy `all`-mode CLI pull — on the next sync.
|
|
1413
|
-
scopeShrinkPolicy: "auto-recover",
|
|
1414
|
-
// Scope-shrink authorship guard: pass the caller's own sub (the very
|
|
1415
|
-
// sub stamped onto uploads as `created-by-sub`) so a scope shrink
|
|
1416
|
-
// never prunes content this owner authored. Owners hold their whole
|
|
1417
|
-
// vault by role-bypass, so without this a `shared`/`custom` pull
|
|
1418
|
-
// would treat their own un-granted work as a foreign orphan.
|
|
1419
|
-
...(uploadAuthor?.userSub !== undefined
|
|
1420
|
-
? { callerSub: uploadAuthor.userSub }
|
|
1421
|
-
: {}),
|
|
1422
|
-
...(pullScope.prefixSet !== undefined
|
|
1423
|
-
? { prefixSet: pullScope.prefixSet }
|
|
1424
|
-
: {}),
|
|
1425
|
-
...(target.personalMode !== undefined ? { personalMode: target.personalMode } : {}),
|
|
1426
|
-
...(target.journalSlug !== undefined ? { journalSlug: target.journalSlug } : {}),
|
|
1427
|
-
// Symmetric to the push side: for the personal slot, tell sync()
|
|
1428
|
-
// how to interpret `companies/{slug}/...` keys in the personal
|
|
1429
|
-
// bucket. With `includeLocalCompanies: true`, keys for slugs NOT
|
|
1430
|
-
// in `teamSyncedSlugs` are downloaded (legitimate cloud:false
|
|
1431
|
-
// opt-in content from another machine); keys for slugs IN that
|
|
1432
|
-
// set are dropped as orphans (pre-decommission residue from a
|
|
1433
|
-
// promoted company). With `includeLocalCompanies: false` the
|
|
1434
|
-
// legacy pre-5.20 behavior holds: all `companies/...` keys are
|
|
1435
|
-
// dropped. Only spread for the personal slot so company-target
|
|
1436
|
-
// args stay identical to pre-Slice-2 shape (test C pin).
|
|
1437
|
-
...(target.personalMode === true
|
|
1438
|
-
? {
|
|
1439
|
-
includeLocalCompanies,
|
|
1440
|
-
teamSyncedSlugs,
|
|
1441
|
-
}
|
|
1442
|
-
: {}),
|
|
1443
|
-
onEvent: tagAndEmit,
|
|
1444
|
-
});
|
|
1445
|
-
}
|
|
1446
|
-
|
|
1447
|
-
// Concat push + pull conflict paths into a single per-company list,
|
|
1448
|
-
// then dedupe — a key that legitimately conflicts on both halves of
|
|
1449
|
-
// a bidirectional run (e.g. `.hq/install-manifest.json` round-trip
|
|
1450
|
-
// before Fix #1) appears twice in the concat but represents a single
|
|
1451
|
-
// logical conflict for the operator. Bug #3 in the 5.33.0 deep-test:
|
|
1452
|
-
// every round produced `conflictPaths: [X, X]` and `conflicts: 2`,
|
|
1453
|
-
// double-counting the conflict in every metric downstream. The push
|
|
1454
|
-
// and pull halves each emit a `conflict` event in their own direction
|
|
1455
|
-
// (preserved on the event stream for tracing); only the merged
|
|
1456
|
-
// result list is collapsed. Stable first-seen order is preserved so
|
|
1457
|
-
// consumers can rely on the pull entry coming before its push twin.
|
|
1458
|
-
const seenConflictPaths = new Set<string>();
|
|
1459
|
-
const mergedConflictPaths: string[] = [];
|
|
1460
|
-
for (const p of [...pullResult.conflictPaths, ...pushResult.conflictPaths]) {
|
|
1461
|
-
if (seenConflictPaths.has(p)) continue;
|
|
1462
|
-
seenConflictPaths.add(p);
|
|
1463
|
-
mergedConflictPaths.push(p);
|
|
1464
|
-
}
|
|
1465
|
-
const aborted = pullResult.aborted || pushResult.aborted;
|
|
1466
|
-
|
|
1467
|
-
// Overwrite the progress-derived counts with the authoritative numbers
|
|
1468
|
-
// from the sync/share return values. The `progress` stream over-counts
|
|
1469
|
-
// when the inner walker emits a progress row for a file it then skips
|
|
1470
|
-
// due to a journal hit — a clean return value is the source of truth.
|
|
1471
|
-
// For the throw case below this overwrite never runs, so `state` keeps
|
|
1472
|
-
// its progress-derived counts (which is exactly what we want there).
|
|
1473
|
-
state.filesDownloaded = pullResult.filesDownloaded;
|
|
1474
|
-
state.bytesDownloaded = pullResult.bytesDownloaded;
|
|
1475
|
-
state.filesUploaded = pushResult.filesUploaded;
|
|
1476
|
-
state.bytesUploaded = pushResult.bytesUploaded;
|
|
1477
|
-
state.status = aborted ? "aborted" : "complete";
|
|
1478
|
-
|
|
1479
|
-
emit({
|
|
1480
|
-
type: "complete",
|
|
1481
|
-
company: companyLabel,
|
|
1482
|
-
filesDownloaded: pullResult.filesDownloaded,
|
|
1483
|
-
bytesDownloaded: pullResult.bytesDownloaded,
|
|
1484
|
-
filesUploaded: pushResult.filesUploaded,
|
|
1485
|
-
bytesUploaded: pushResult.bytesUploaded,
|
|
1486
|
-
filesSkipped: pullResult.filesSkipped + pushResult.filesSkipped,
|
|
1487
|
-
// Push-side counters surfaced on `complete` so the menubar's
|
|
1488
|
-
// `SyncCompleteEvent` (which carries them as Option<u32> for
|
|
1489
|
-
// back-compat with pre-5.25 engines) can render the new totals.
|
|
1490
|
-
// Always emitted as numbers (0 when no push leg ran) so Rust's
|
|
1491
|
-
// serde decodes them as `Some(0)` rather than `None` — distinct
|
|
1492
|
-
// from the legacy-engine `None` and useful when the UI wants to
|
|
1493
|
-
// distinguish "engine ran, nothing tombstoned" from "engine
|
|
1494
|
-
// didn't report".
|
|
1495
|
-
// Tombstones now flow on both legs:
|
|
1496
|
-
// - push side: `ShareResult.filesTombstoned` (remote was already
|
|
1497
|
-
// 404 at HEAD time, journal entry dropped).
|
|
1498
|
-
// - pull side: `SyncResult.filesTombstoned` (Bug #9 — journal-
|
|
1499
|
-
// known key missing from remote LIST, applied as local delete).
|
|
1500
|
-
// Sum them so the menubar's `SyncCompleteEvent` reflects the total
|
|
1501
|
-
// delete-propagation activity for that company across the run.
|
|
1502
|
-
filesTombstoned: pushResult.filesTombstoned + pullResult.filesTombstoned,
|
|
1503
|
-
filesRefusedStale: pushResult.filesRefusedStale,
|
|
1504
|
-
// Bonus diagnostic: surface the paths so operators can triage the
|
|
1505
|
-
// recurring `filesRefusedStale: N` signal — the count alone was
|
|
1506
|
-
// untriageable per the 5.33.0 deep-test report's "205 issue".
|
|
1507
|
-
// Pre-capped at 50 by share() itself.
|
|
1508
|
-
filesRefusedStalePaths: pushResult.filesRefusedStalePaths,
|
|
1509
|
-
// Pull side now reports an `filesExcludedByPolicy` too (Bug #2 —
|
|
1510
|
-
// ephemeral conflict-mirror refusals in the pull walker). Sum
|
|
1511
|
-
// both legs so the `complete` event reports total excluded across
|
|
1512
|
-
// the full bidirectional pass; pre-fix the pull half silently
|
|
1513
|
-
// pushed legacy `.conflict-*` litter into clean trees with the
|
|
1514
|
-
// same counter showing 0.
|
|
1515
|
-
filesExcludedByPolicy:
|
|
1516
|
-
pushResult.filesExcludedByPolicy + pullResult.filesExcludedByPolicy,
|
|
1517
|
-
// Sourced from the merged path list so push-side conflicts are
|
|
1518
|
-
// counted too — `ShareResult` doesn't expose a numeric counter,
|
|
1519
|
-
// and using `pullResult.conflicts` alone silently dropped any
|
|
1520
|
-
// push conflict from the count while leaving its path in
|
|
1521
|
-
// `conflictPaths`.
|
|
1522
|
-
conflicts: mergedConflictPaths.length,
|
|
1523
|
-
conflictPaths: mergedConflictPaths,
|
|
1524
|
-
// Either phase aborting marks the company aborted — the UI treats
|
|
1525
|
-
// `aborted: true` as "sync didn't complete cleanly for this company".
|
|
1526
|
-
aborted,
|
|
1527
|
-
newFiles: pullResult.newFiles,
|
|
1528
|
-
newFilesCount: pullResult.newFilesCount,
|
|
1529
|
-
// Scope-aware download counters (US-005). Pull-only — the push leg
|
|
1530
|
-
// has no scope concept — so they pass through from `pullResult`.
|
|
1531
|
-
filesOutOfScope: pullResult.filesOutOfScope,
|
|
1532
|
-
scopeOrphansRemoved: pullResult.scopeOrphansRemoved,
|
|
1533
|
-
scopeOrphansBlocked: pullResult.scopeOrphansBlocked,
|
|
1534
|
-
});
|
|
1535
|
-
for (const p of pullResult.conflictPaths) {
|
|
1536
|
-
allConflicts.push({ company: companyLabel, path: p, direction: "pull" });
|
|
1537
|
-
}
|
|
1538
|
-
for (const p of pushResult.conflictPaths) {
|
|
1539
|
-
allConflicts.push({ company: companyLabel, path: p, direction: "push" });
|
|
1540
|
-
}
|
|
1541
|
-
} catch (err) {
|
|
1542
|
-
// describeError walks the cause chain so AWS SDK v3's "UnknownError"
|
|
1543
|
-
// wrapper surfaces the underlying Node networking error (ENOTFOUND,
|
|
1544
|
-
// ECONNRESET, …) instead of an unactionable bare "UnknownError".
|
|
1545
|
-
const message = describeError(err);
|
|
1546
|
-
errors.push({ company: companyLabel, message });
|
|
1547
|
-
// `state.status` was seeded as "errored" at loop entry — the throw
|
|
1548
|
-
// path leaves it there, and `state.files{Down,Up}loaded` reflects the
|
|
1549
|
-
// partial counts captured from `progress` events before the throw.
|
|
1550
|
-
// Emit a `complete` event with `aborted: true` and those partial
|
|
1551
|
-
// counts so consumers walking the `complete` event stream see every
|
|
1552
|
-
// company in the fanout uniformly. This is the fix for the misleading
|
|
1553
|
-
// rollup — see file header `Exit code: 2` doc.
|
|
1554
|
-
emit({
|
|
1555
|
-
type: "complete",
|
|
1556
|
-
company: companyLabel,
|
|
1557
|
-
filesDownloaded: state.filesDownloaded,
|
|
1558
|
-
bytesDownloaded: state.bytesDownloaded,
|
|
1559
|
-
filesUploaded: state.filesUploaded,
|
|
1560
|
-
bytesUploaded: state.bytesUploaded,
|
|
1561
|
-
filesSkipped: 0,
|
|
1562
|
-
// Mid-flight throw: we have no clean ShareResult to read these
|
|
1563
|
-
// from. Report 0 so the event shape stays stable; the partial
|
|
1564
|
-
// counts above already reflect what actually moved before the
|
|
1565
|
-
// throw.
|
|
1566
|
-
filesTombstoned: 0,
|
|
1567
|
-
filesRefusedStale: 0,
|
|
1568
|
-
filesRefusedStalePaths: [],
|
|
1569
|
-
filesExcludedByPolicy: 0,
|
|
1570
|
-
conflicts: 0,
|
|
1571
|
-
conflictPaths: [],
|
|
1572
|
-
aborted: true,
|
|
1573
|
-
newFiles: [],
|
|
1574
|
-
newFilesCount: 0,
|
|
1575
|
-
// Mid-flight throw: no clean scope counts to report. 0 keeps the
|
|
1576
|
-
// event shape stable (US-005).
|
|
1577
|
-
filesOutOfScope: 0,
|
|
1578
|
-
scopeOrphansRemoved: 0,
|
|
1579
|
-
scopeOrphansBlocked: 0,
|
|
1580
|
-
});
|
|
1581
|
-
emit({
|
|
1582
|
-
type: "error",
|
|
1583
|
-
company: companyLabel,
|
|
1584
|
-
path: "(company)",
|
|
1585
|
-
message,
|
|
1586
|
-
});
|
|
1587
|
-
// Continue — one company's failure shouldn't abort the whole fanout.
|
|
1588
|
-
}
|
|
1589
|
-
}
|
|
1590
|
-
|
|
1591
|
-
// Walk every per-company entry — the map holds one row per planned company,
|
|
1592
|
-
// including ones that aborted via thrown exception. This is the fix for the
|
|
1593
|
-
// bug where `all-complete` reported `filesDownloaded: 0` for an aborted
|
|
1594
|
-
// personal-sync that had already emitted thousands of `progress` events:
|
|
1595
|
-
// the rollup used to only sum companies that emitted a clean `complete`,
|
|
1596
|
-
// which silently dropped partials when the sync function threw.
|
|
1597
|
-
let totalDownloaded = 0;
|
|
1598
|
-
let totalDownloadedBytes = 0;
|
|
1599
|
-
let totalUploaded = 0;
|
|
1600
|
-
let totalUploadedBytes = 0;
|
|
1601
|
-
let partial = false;
|
|
1602
|
-
const companies: Array<{
|
|
1603
|
-
company: string;
|
|
1604
|
-
status: CompanyStatus;
|
|
1605
|
-
filesDownloaded: number;
|
|
1606
|
-
bytesDownloaded: number;
|
|
1607
|
-
filesUploaded: number;
|
|
1608
|
-
bytesUploaded: number;
|
|
1609
|
-
}> = [];
|
|
1610
|
-
for (const target of plan) {
|
|
1611
|
-
const s = stateByCompany.get(target.slug);
|
|
1612
|
-
if (!s) continue; // unreachable — every plan entry seeds the map
|
|
1613
|
-
totalDownloaded += s.filesDownloaded;
|
|
1614
|
-
totalDownloadedBytes += s.bytesDownloaded;
|
|
1615
|
-
totalUploaded += s.filesUploaded;
|
|
1616
|
-
totalUploadedBytes += s.bytesUploaded;
|
|
1617
|
-
if (s.status !== "complete") partial = true;
|
|
1618
|
-
companies.push({
|
|
1619
|
-
company: s.company,
|
|
1620
|
-
status: s.status,
|
|
1621
|
-
filesDownloaded: s.filesDownloaded,
|
|
1622
|
-
bytesDownloaded: s.bytesDownloaded,
|
|
1623
|
-
filesUploaded: s.filesUploaded,
|
|
1624
|
-
bytesUploaded: s.bytesUploaded,
|
|
1625
|
-
});
|
|
1626
|
-
}
|
|
1045
|
+
const fanout = await executeCompanyFanout({
|
|
1046
|
+
plan,
|
|
1047
|
+
direction: parsed.direction,
|
|
1048
|
+
hqRoot: parsed.hqRoot,
|
|
1049
|
+
onConflict: parsed.onConflict,
|
|
1050
|
+
client,
|
|
1051
|
+
vaultConfig,
|
|
1052
|
+
...(uploadAuthor ? { uploadAuthor } : {}),
|
|
1053
|
+
...(deps.operationLockAlreadyHeld ? { operationLockAlreadyHeld: true } : {}),
|
|
1054
|
+
syncFn,
|
|
1055
|
+
shareFn,
|
|
1056
|
+
resolveDeletePolicy,
|
|
1057
|
+
emit,
|
|
1058
|
+
});
|
|
1059
|
+
const { errors, allConflicts } = fanout;
|
|
1060
|
+
const rollup = rollupAllComplete(plan, fanout.stateByCompany);
|
|
1627
1061
|
|
|
1628
1062
|
// Fire telemetry collector before the all-complete emit so the cursor at
|
|
1629
1063
|
// `~/.hq/telemetry-cursor.json` is consistent with what the menubar sees.
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
(() => defaultCollectTelemetry(client, deps.createVaultClient !== undefined, parsed.hqRoot));
|
|
1640
|
-
await telemetryFn().catch(() => undefined);
|
|
1064
|
+
await emitTelemetry({
|
|
1065
|
+
collectTelemetry: deps.collectTelemetry,
|
|
1066
|
+
defaultCollectTelemetry: () =>
|
|
1067
|
+
defaultCollectTelemetry(
|
|
1068
|
+
client,
|
|
1069
|
+
deps.createVaultClient !== undefined,
|
|
1070
|
+
parsed.hqRoot,
|
|
1071
|
+
),
|
|
1072
|
+
});
|
|
1641
1073
|
|
|
1642
1074
|
emit({
|
|
1643
1075
|
type: "all-complete",
|
|
1644
1076
|
companiesAttempted: plan.length,
|
|
1645
|
-
filesDownloaded: totalDownloaded,
|
|
1646
|
-
bytesDownloaded: totalDownloadedBytes,
|
|
1647
|
-
filesUploaded: totalUploaded,
|
|
1648
|
-
bytesUploaded: totalUploadedBytes,
|
|
1077
|
+
filesDownloaded: rollup.totalDownloaded,
|
|
1078
|
+
bytesDownloaded: rollup.totalDownloadedBytes,
|
|
1079
|
+
filesUploaded: rollup.totalUploaded,
|
|
1080
|
+
bytesUploaded: rollup.totalUploadedBytes,
|
|
1649
1081
|
conflictPaths: allConflicts,
|
|
1650
1082
|
errors,
|
|
1651
|
-
partial,
|
|
1652
|
-
companies,
|
|
1083
|
+
partial: rollup.partial,
|
|
1084
|
+
companies: rollup.companies,
|
|
1653
1085
|
});
|
|
1654
1086
|
|
|
1655
1087
|
// Post-sync qmd reindex — runs AFTER `all-complete` is emitted so the
|
|
@@ -1658,7 +1090,10 @@ export async function runRunner(
|
|
|
1658
1090
|
// pulled in (nothing to reindex otherwise) and not explicitly disabled.
|
|
1659
1091
|
// Self-contained: shells out to the global `qmd` binary, no dependency on
|
|
1660
1092
|
// any (possibly stale) script inside the synced HQ tree. See qmd-reindex.ts.
|
|
1661
|
-
if (
|
|
1093
|
+
if (
|
|
1094
|
+
rollup.totalDownloaded > 0 &&
|
|
1095
|
+
process.env.HQ_QMD_REINDEX_ON_SYNC !== "0"
|
|
1096
|
+
) {
|
|
1662
1097
|
try {
|
|
1663
1098
|
reindexAfterSync(parsed.hqRoot);
|
|
1664
1099
|
} catch {
|
|
@@ -1833,418 +1268,28 @@ export interface WatcherSurface {
|
|
|
1833
1268
|
dispose(): void;
|
|
1834
1269
|
}
|
|
1835
1270
|
|
|
1836
|
-
/**
|
|
1837
|
-
* Route a changed relative path to the push target that owns it.
|
|
1838
|
-
*
|
|
1839
|
-
* - `companies/<slug>/...` → a single-company push for `<slug>` (the targeted
|
|
1840
|
-
* pass per PRD decision — push only the changed company, not a full
|
|
1841
|
-
* `--companies` fanout).
|
|
1842
|
-
* - anything else under hqRoot → the personal target (a `--companies` push
|
|
1843
|
-
* restricted to personal via the personal-vault scope; modeled here as the
|
|
1844
|
-
* "personal" route the caller maps to the right argv).
|
|
1845
|
-
*
|
|
1846
|
-
* Returns `null` for paths the routing cannot attribute (defensive — the
|
|
1847
|
-
* watcher's filter already drops excluded top-levels, so this is belt-and-
|
|
1848
|
-
* suspenders for synthetic events).
|
|
1849
|
-
*/
|
|
1850
|
-
export function routeChangeToTarget(
|
|
1851
|
-
relPath: string,
|
|
1852
|
-
): { kind: "company"; slug: string } | { kind: "personal" } | null {
|
|
1853
|
-
const norm = relPath.split(path.sep).join("/").replace(/^\.\//, "");
|
|
1854
|
-
if (norm === "" || norm.startsWith("..")) return null;
|
|
1855
|
-
const segments = norm.split("/").filter((s) => s.length > 0);
|
|
1856
|
-
if (segments.length === 0) return null;
|
|
1857
|
-
if (segments[0] === "companies") {
|
|
1858
|
-
// companies/<slug>/... — need at least the slug segment to target.
|
|
1859
|
-
if (segments.length < 2 || segments[1].length === 0) return null;
|
|
1860
|
-
return { kind: "company", slug: segments[1] };
|
|
1861
|
-
}
|
|
1862
|
-
return { kind: "personal" };
|
|
1863
|
-
}
|
|
1864
|
-
|
|
1865
|
-
/**
|
|
1866
|
-
* Build the argv for a targeted push pass from a routed change. The push runs
|
|
1867
|
-
* `--direction push` for just the routed target so a local edit propagates in
|
|
1868
|
-
* seconds without a full fanout. Company routes use `--company <slug>`;
|
|
1869
|
-
* personal routes use `--companies --direction push` (the personal-vault scope
|
|
1870
|
-
* is resolved inside runRunner's fanout; skipUnchanged no-ops the company
|
|
1871
|
-
* subtrees that didn't change). Inherits `--hq-root` / `--on-conflict` from
|
|
1872
|
-
* the base argv. Pure helper, exported for unit testing the routing→argv map.
|
|
1873
|
-
*/
|
|
1874
|
-
export function buildTargetedPushArgv(
|
|
1875
|
-
route: { kind: "company"; slug: string } | { kind: "personal" },
|
|
1876
|
-
baseArgv: string[],
|
|
1877
|
-
): string[] {
|
|
1878
|
-
const carried = carriedFlags(baseArgv);
|
|
1879
|
-
if (route.kind === "company") {
|
|
1880
|
-
return ["--company", route.slug, "--direction", "push", ...carried];
|
|
1881
|
-
}
|
|
1882
|
-
return ["--companies", "--direction", "push", ...carried];
|
|
1883
|
-
}
|
|
1884
|
-
|
|
1885
|
-
/**
|
|
1886
|
-
* Build the argv for a targeted PULL pass from a routed change (US-009 — the
|
|
1887
|
-
* receiver's pull-on-event path). Mirrors {@link buildTargetedPushArgv} but
|
|
1888
|
-
* with `--direction pull`: a peer device pushed a change, so this device pulls
|
|
1889
|
-
* just the affected company/subtree instead of waiting for the next
|
|
1890
|
-
* `--poll-remote-ms` cycle. Company routes use `--company <slug>`; personal
|
|
1891
|
-
* routes use `--companies` (the personal-vault scope is resolved inside
|
|
1892
|
-
* runRunner's fanout; skipUnchanged no-ops the subtrees that didn't change).
|
|
1893
|
-
* Inherits `--hq-root` / `--on-conflict` from the base argv. Pure helper,
|
|
1894
|
-
* exported for unit testing the event→argv map.
|
|
1895
|
-
*/
|
|
1896
|
-
export function buildTargetedPullArgv(
|
|
1897
|
-
route: { kind: "company"; slug: string } | { kind: "personal" },
|
|
1898
|
-
baseArgv: string[],
|
|
1899
|
-
): string[] {
|
|
1900
|
-
const carried = carriedFlags(baseArgv);
|
|
1901
|
-
if (route.kind === "company") {
|
|
1902
|
-
return ["--company", route.slug, "--direction", "pull", ...carried];
|
|
1903
|
-
}
|
|
1904
|
-
return ["--companies", "--direction", "pull", ...carried];
|
|
1905
|
-
}
|
|
1906
|
-
|
|
1907
1271
|
export async function runRunnerWithLoop(
|
|
1908
1272
|
argv: string[],
|
|
1909
1273
|
deps: RunnerLoopDeps = {},
|
|
1910
1274
|
): Promise<number> {
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
// The actual sync pass — same seam the watch loop uses (deps.runPass),
|
|
1921
|
-
// so a test can assert "waits for a short-lived holder, THEN proceeds to
|
|
1922
|
-
// sync" without touching the network. Production passes `argv` to
|
|
1923
|
-
// runRunner exactly as before. Regression guard for DEV-1772
|
|
1924
|
-
// (feedback_28a1833f): instant-sync one-shots used to exit 17 and die on
|
|
1925
|
-
// a lock conflict with the ~1-min reindex hook; they now WAIT (default)
|
|
1926
|
-
// and proceed once the short holder releases.
|
|
1927
|
-
const runOnce = deps.runPass ?? ((passArgv: string[]) => runRunner(passArgv));
|
|
1928
|
-
try {
|
|
1929
|
-
return await withOperationLock(parsed.hqRoot, "sync", () => runOnce(argv), {
|
|
1930
|
-
timeoutSec: parsed.lockTimeoutSec,
|
|
1931
|
-
});
|
|
1932
|
-
} catch (err) {
|
|
1933
|
-
if (err instanceof OperationLockedError) {
|
|
1934
|
-
// The lock wait was BOUNDED and tripped (a holder never released
|
|
1935
|
-
// within --lock-timeout / HQ_OP_LOCK_TIMEOUT). Surface it loudly on
|
|
1936
|
-
// stderr and exit with the stable OPERATION_LOCKED_EXIT (17) so the
|
|
1937
|
-
// spawner (menubar) can recognize a lock conflict and SCHEDULE A
|
|
1938
|
-
// RETRY rather than treating it as a hard failure and silently giving
|
|
1939
|
-
// up (DEV-1772). With the default (no bound) we never reach here — the
|
|
1940
|
-
// one-shot waits indefinitely and proceeds.
|
|
1941
|
-
process.stderr.write(err.message + "\n");
|
|
1942
|
-
return OPERATION_LOCKED_EXIT;
|
|
1943
|
-
}
|
|
1944
|
-
throw err;
|
|
1945
|
-
}
|
|
1946
|
-
}
|
|
1947
|
-
const sleep =
|
|
1948
|
-
deps.sleep ??
|
|
1949
|
-
((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
|
|
1950
|
-
const runPass = deps.runPass ?? ((passArgv: string[]) => runRunner(passArgv));
|
|
1951
|
-
const pollIdx = argv.indexOf("--poll-remote-ms");
|
|
1952
|
-
const pollMs =
|
|
1953
|
-
pollIdx >= 0 && argv[pollIdx + 1] ? Number(argv[pollIdx + 1]) : 600_000;
|
|
1954
|
-
const eventPush = argv.includes("--event-push");
|
|
1955
|
-
// In `--companies` mode the sync scope is companies/*/{knowledge,projects,…}
|
|
1956
|
-
// (per .hqinclude), so the watcher must NOT apply the personal-vault
|
|
1957
|
-
// top-level exclusions (PERSONAL_VAULT_EXCLUDED_TOP_LEVEL drops `companies/`
|
|
1958
|
-
// and `workspace/`) — doing so would exclude exactly the paths being synced,
|
|
1959
|
-
// and no local edit would ever trigger an instant push. The shared ignore
|
|
1960
|
-
// stack (createIgnoreFilter / .hqignore / .hqinclude) already scopes the
|
|
1961
|
-
// watch filter correctly in companies mode. personalMode is only for a
|
|
1962
|
-
// personal-vault-as-root run, where companies/ et al. genuinely aren't synced.
|
|
1963
|
-
const companiesMode = argv.includes("--companies");
|
|
1964
|
-
const hqIdx = argv.indexOf("--hq-root");
|
|
1965
|
-
const hqRoot =
|
|
1966
|
-
hqIdx >= 0 && argv[hqIdx + 1] ? argv[hqIdx + 1] : DEFAULT_HQ_ROOT;
|
|
1967
|
-
|
|
1968
|
-
// Strip the loop-only flags before delegating: the parser inside runRunner
|
|
1969
|
-
// accepts --watch/--poll-remote-ms/--event-push, but we don't want a per-
|
|
1970
|
-
// iteration pass to think it's re-entering watch mode.
|
|
1971
|
-
const passArgv = argv.filter((a, i) => {
|
|
1972
|
-
if (a === "--watch") return false;
|
|
1973
|
-
if (a === "--poll-remote-ms") return false;
|
|
1974
|
-
if (a === "--event-push") return false;
|
|
1975
|
-
if (i > 0 && argv[i - 1] === "--poll-remote-ms") return false;
|
|
1976
|
-
return true;
|
|
1977
|
-
});
|
|
1978
|
-
|
|
1979
|
-
// ---- shared in-flight guard ------------------------------------------
|
|
1980
|
-
// The poll loop AND watcher-triggered targeted pushes funnel through this
|
|
1981
|
-
// mutex so a watcher push never overlaps an in-flight pass (PRD AC). A
|
|
1982
|
-
// trigger that arrives while a pass runs is collapsed by WatchPushDriver's
|
|
1983
|
-
// own pending-while-pushing logic, then re-armed after the pass settles.
|
|
1984
|
-
let inFlight = false;
|
|
1985
|
-
let stopped = false;
|
|
1986
|
-
const runGuarded = async (
|
|
1987
|
-
pass: () => Promise<number>,
|
|
1988
|
-
): Promise<number | "skipped"> => {
|
|
1989
|
-
if (inFlight) return "skipped";
|
|
1990
|
-
inFlight = true;
|
|
1991
|
-
try {
|
|
1992
|
-
return await pass();
|
|
1993
|
-
} finally {
|
|
1994
|
-
inFlight = false;
|
|
1995
|
-
}
|
|
1996
|
-
};
|
|
1997
|
-
|
|
1998
|
-
// ---- event-push wiring (Phase 1) -------------------------------------
|
|
1999
|
-
let watcher: WatcherSurface | null = null;
|
|
2000
|
-
let driver: WatchPushDriver | null = null;
|
|
2001
|
-
let detachSignal: (() => void) | null = null;
|
|
2002
|
-
let lastChangedRel: string | null = null;
|
|
2003
|
-
let lastBatch: TreeChangeBatch | null = null;
|
|
2004
|
-
// ---- pull-on-event receiver (Phase 2, US-009) ------------------------
|
|
2005
|
-
// Started after the watcher, disposed before the watcher (mirror of the
|
|
2006
|
-
// PushTransport ordering). Dormant by default: the default factory returns
|
|
2007
|
-
// a NoopPushReceiver, and even a real receiver stays dormant unless the
|
|
2008
|
-
// per-tenant feature flag is on AND a queue URL is provisioned server-side.
|
|
2009
|
-
let receiver: PushReceiver | null = null;
|
|
2010
|
-
// ---- event-driven publish + pull (Phase 3, US-017/018/019) ------------
|
|
2011
|
-
// Brought up asynchronously after the watcher when the rollout gate
|
|
2012
|
-
// passes; null until ready (and stays null on startup failure → poll-only).
|
|
2013
|
-
let eventSync: EventSyncHandles | null = null;
|
|
2014
|
-
|
|
2015
|
-
if (eventPush) {
|
|
2016
|
-
const clock = deps.clock ?? systemClock;
|
|
2017
|
-
const debounceMs = 2000;
|
|
2018
|
-
const createWatcher =
|
|
2019
|
-
deps.createWatcher ??
|
|
2020
|
-
((opts) =>
|
|
2021
|
-
new TreeWatcher({
|
|
2022
|
-
hqRoot: opts.hqRoot,
|
|
2023
|
-
debounceMs: opts.debounceMs,
|
|
2024
|
-
clock: opts.clock,
|
|
2025
|
-
// false in --companies mode so the watch filter matches the sync
|
|
2026
|
-
// scope (companies/* are included via .hqinclude); true only for a
|
|
2027
|
-
// personal-vault-as-root run.
|
|
2028
|
-
personalMode: !companiesMode,
|
|
2029
|
-
}));
|
|
2030
|
-
watcher = createWatcher({ hqRoot, debounceMs, clock });
|
|
2031
|
-
|
|
2032
|
-
// The driver runs the targeted push when a debounced burst settles. Its
|
|
2033
|
-
// concurrency guard collapses a trigger that lands mid-pass; we ALSO gate
|
|
2034
|
-
// through `runGuarded` so a poll pass in flight is never overlapped.
|
|
2035
|
-
driver = new WatchPushDriver({
|
|
2036
|
-
debounceMs: 0, // TreeWatcher already debounces; the driver just guards.
|
|
2037
|
-
clock,
|
|
2038
|
-
push: async () => {
|
|
2039
|
-
if (stopped) return;
|
|
2040
|
-
const rel = lastChangedRel;
|
|
2041
|
-
// Snapshot the settled batch BEFORE the await: a change landing
|
|
2042
|
-
// mid-pass overwrites lastBatch for the NEXT pass, and this pass
|
|
2043
|
-
// must only announce what it actually pushed.
|
|
2044
|
-
const batchForPublish = lastBatch;
|
|
2045
|
-
lastBatch = null;
|
|
2046
|
-
const route = rel
|
|
2047
|
-
? routeChangeToTarget(rel)
|
|
2048
|
-
: { kind: "personal" as const };
|
|
2049
|
-
if (!route) return;
|
|
2050
|
-
const targetedArgv = buildTargetedPushArgv(route, passArgv);
|
|
2051
|
-
const result = await runGuarded(() => runPass(targetedArgv));
|
|
2052
|
-
// Phase 3 (US-017): publish PushEvents only AFTER the targeted push
|
|
2053
|
-
// pass succeeded — an event must never announce bytes that are not
|
|
2054
|
-
// in S3 yet. A skipped pass (guard held) or a failed pass publishes
|
|
2055
|
-
// nothing; the cadence poll covers the miss. Fall back to a
|
|
2056
|
-
// single-path batch when the watcher emitted a bare path signal.
|
|
2057
|
-
if (result === 0 && eventSync) {
|
|
2058
|
-
const batch: TreeChangeBatch | null =
|
|
2059
|
-
batchForPublish ??
|
|
2060
|
-
(rel ? { paths: new Map([[path.join(hqRoot, rel), rel]]) } : null);
|
|
2061
|
-
if (batch) eventSync.publishBatch(batch);
|
|
2062
|
-
}
|
|
2063
|
-
},
|
|
2064
|
-
});
|
|
2065
|
-
|
|
2066
|
-
// A debounced TreeWatcher 'changed' signal feeds the driver, which fires
|
|
2067
|
-
// the targeted push after its own (zero) window — i.e. immediately, but
|
|
2068
|
-
// still serialized behind any in-flight pass. A path-aware watcher passes
|
|
2069
|
-
// the changed relative path so the push targets just its owning company;
|
|
2070
|
-
// the bare-signal TreeWatcher leaves it null → personal-vault route.
|
|
2071
|
-
watcher.onChange((changedRelPath, batch) => {
|
|
2072
|
-
if (stopped) return;
|
|
2073
|
-
lastChangedRel = changedRelPath ?? null;
|
|
2074
|
-
lastBatch = batch ?? null;
|
|
2075
|
-
driver?.notifyChange();
|
|
2076
|
-
});
|
|
2077
|
-
watcher.start();
|
|
2078
|
-
|
|
2079
|
-
// Pull-on-event receiver (US-009). The injected SyncEngineFn bridges a
|
|
2080
|
-
// received PushEvent → a TARGETED pull pass routed by relativePath, funneled
|
|
2081
|
-
// through the SAME `runGuarded` mutex as the poll loop + watcher push so a
|
|
2082
|
-
// pull-on-event never overlaps an in-flight pass. Started AFTER the watcher
|
|
2083
|
-
// so a live event can't race a half-built daemon. Default factory = noop
|
|
2084
|
-
// (dormant); a real SqsPushReceiver is injected by a later release once the
|
|
2085
|
-
// server-side per-client SQS queue is provisioned.
|
|
2086
|
-
const receiverSyncFn: SyncEngineFn = async (ctx) => {
|
|
2087
|
-
if (stopped) return;
|
|
2088
|
-
const route = routeChangeToTarget(ctx.event.relativePath);
|
|
2089
|
-
if (!route) return;
|
|
2090
|
-
const targetedArgv = buildTargetedPullArgv(route, passArgv);
|
|
2091
|
-
await runGuarded(() => runPass(targetedArgv));
|
|
2092
|
-
};
|
|
2093
|
-
const createReceiver =
|
|
2094
|
-
deps.createReceiver ?? (() => new NoopPushReceiver());
|
|
2095
|
-
receiver = createReceiver({ syncFn: receiverSyncFn, hqRoot });
|
|
2096
|
-
// Fire-and-forget start: a receiver's start() kicks off its own poll loop
|
|
2097
|
-
// (SqsPushReceiver) or trivially flips connected (noop) — it must NOT block
|
|
2098
|
-
// the runner's poll loop from entering. Errors are swallowed; the cadence
|
|
2099
|
-
// poll is the safety net regardless of receiver health. (The await-free
|
|
2100
|
-
// start also keeps the poll loop's microtask timing identical to the
|
|
2101
|
-
// pre-US-009 wiring.)
|
|
2102
|
-
void Promise.resolve(receiver.start()).catch(() => undefined);
|
|
2103
|
-
|
|
2104
|
-
// ---- Phase 3: event-driven publish + pull (US-017/018/019) ----------
|
|
2105
|
-
// Gated to enrolled accounts (resolveEventSync — exact-email allowlist +
|
|
2106
|
-
// HQ_SYNC_EVENT_SYNC override). Brought up asynchronously so a slow
|
|
2107
|
-
// subscribe/vend can't delay the first poll pass; until (and unless) the
|
|
2108
|
-
// handles resolve, behavior is byte-identical to the gate-off path.
|
|
2109
|
-
const getClaims = deps.getIdTokenClaims ?? defaultGetIdTokenClaims;
|
|
2110
|
-
const email = getClaims()?.email;
|
|
2111
|
-
if (resolveEventSync(email, process.env.HQ_SYNC_EVENT_SYNC)) {
|
|
2112
|
-
const getAccessToken =
|
|
2113
|
-
deps.getAccessToken ??
|
|
2114
|
-
(() => getValidAccessToken(DEFAULT_COGNITO, { interactive: false }));
|
|
2115
|
-
const startES = deps.startEventSync ?? defaultStartEventSync;
|
|
2116
|
-
// Entirely async + caught: NOTHING in the Phase 3 bring-up (device-id
|
|
2117
|
-
// persistence, tenant resolution, subscribe) may crash or delay the
|
|
2118
|
-
// daemon — any failure degrades to poll-only.
|
|
2119
|
-
void (async () => {
|
|
2120
|
-
const handles = await startES({
|
|
2121
|
-
hqRoot,
|
|
2122
|
-
apiUrl: DEFAULT_VAULT_API_URL,
|
|
2123
|
-
authToken: getAccessToken,
|
|
2124
|
-
deviceId: getOrCreateMachineId(hqRoot),
|
|
2125
|
-
// The server rejects publishes whose originTenantId mismatches the
|
|
2126
|
-
// JWT principal, so resolve the SAME canonical person uid the vault
|
|
2127
|
-
// API derives from this token.
|
|
2128
|
-
resolveTenantId: async () => {
|
|
2129
|
-
const client = new VaultClient({
|
|
2130
|
-
apiUrl: DEFAULT_VAULT_API_URL,
|
|
2131
|
-
authToken: getAccessToken,
|
|
2132
|
-
region: DEFAULT_COGNITO.region,
|
|
2133
|
-
});
|
|
2134
|
-
const persons = await client.entity.listByType("person");
|
|
2135
|
-
const pick = pickCanonicalPersonEntity(persons);
|
|
2136
|
-
if (!pick?.uid) {
|
|
2137
|
-
throw new Error("no canonical person entity for this account");
|
|
2138
|
-
}
|
|
2139
|
-
return pick.uid;
|
|
2140
|
-
},
|
|
2141
|
-
syncFn: receiverSyncFn,
|
|
2142
|
-
log: (m) => process.stderr.write(`${m}\n`),
|
|
2143
|
-
});
|
|
2144
|
-
if (!handles) return;
|
|
2145
|
-
if (stopped) {
|
|
2146
|
-
// Shutdown raced the async bring-up — tear straight down.
|
|
2147
|
-
void handles.dispose();
|
|
2148
|
-
return;
|
|
2149
|
-
}
|
|
2150
|
-
eventSync = handles;
|
|
2151
|
-
})().catch((err) => {
|
|
2152
|
-
process.stderr.write(
|
|
2153
|
-
`event-sync: wiring failed, continuing poll-only: ${describeError(err)}\n`,
|
|
2154
|
-
);
|
|
2155
|
-
});
|
|
2156
|
-
}
|
|
2157
|
-
}
|
|
2158
|
-
|
|
2159
|
-
// ---- clean shutdown --------------------------------------------------
|
|
2160
|
-
// SIGTERM (menubar stop_daemon) / SIGINT must tear down BOTH the watcher and
|
|
2161
|
-
// the poll loop with no leaked timers or fds. We flip `stopped`, dispose the
|
|
2162
|
-
// watcher + driver, and let the poll loop observe `stopped` on its next tick.
|
|
2163
|
-
let resolveStopped: (() => void) | null = null;
|
|
2164
|
-
const stoppedSignal = new Promise<void>((resolve) => {
|
|
2165
|
-
resolveStopped = resolve;
|
|
2166
|
-
});
|
|
2167
|
-
const shutdown = (): void => {
|
|
2168
|
-
if (stopped) return;
|
|
2169
|
-
stopped = true;
|
|
2170
|
-
// Dispose the receiver FIRST (mirror of the PushTransport ordering:
|
|
2171
|
-
// inbound subscription torn down before the watcher) so no new
|
|
2172
|
-
// pull-on-event fires mid-teardown. dispose() is async (it drains the
|
|
2173
|
-
// in-flight pull up to its own deadline); fire-and-forget here — the
|
|
2174
|
-
// receiver's internal drain + the runGuarded mutex bound the work, and
|
|
2175
|
-
// SIGTERM teardown must not block. Errors are swallowed.
|
|
2176
|
-
try {
|
|
2177
|
-
void receiver?.dispose();
|
|
2178
|
-
} catch {
|
|
2179
|
-
/* ignore */
|
|
2180
|
-
}
|
|
2181
|
-
// Phase 3 wiring (publish transport + live receiver) — torn down with
|
|
2182
|
-
// the same fire-and-forget posture as the Phase 2 receiver above.
|
|
2183
|
-
try {
|
|
2184
|
-
void eventSync?.dispose();
|
|
2185
|
-
} catch {
|
|
2186
|
-
/* ignore */
|
|
2187
|
-
}
|
|
2188
|
-
eventSync = null;
|
|
2189
|
-
try {
|
|
2190
|
-
driver?.dispose();
|
|
2191
|
-
} catch {
|
|
2192
|
-
/* ignore */
|
|
2193
|
-
}
|
|
2194
|
-
try {
|
|
2195
|
-
watcher?.dispose();
|
|
2196
|
-
} catch {
|
|
2197
|
-
/* ignore */
|
|
2198
|
-
}
|
|
2199
|
-
resolveStopped?.();
|
|
1275
|
+
const parsed = parseArgs(argv);
|
|
1276
|
+
const runtime = {
|
|
1277
|
+
runPassWithOperationLockAlreadyHeld: (passArgv: string[]) =>
|
|
1278
|
+
runRunner(passArgv, { operationLockAlreadyHeld: true }),
|
|
1279
|
+
defaultGetIdTokenClaims,
|
|
1280
|
+
defaultGetAccessToken: () =>
|
|
1281
|
+
getValidAccessToken(DEFAULT_COGNITO, { interactive: false }),
|
|
1282
|
+
apiUrl: DEFAULT_VAULT_API_URL,
|
|
1283
|
+
region: DEFAULT_COGNITO.region,
|
|
2200
1284
|
};
|
|
2201
|
-
const onShutdownSignal =
|
|
2202
|
-
deps.onShutdownSignal ??
|
|
2203
|
-
((handler: () => void) => {
|
|
2204
|
-
const wrapped = () => handler();
|
|
2205
|
-
process.on("SIGTERM", wrapped);
|
|
2206
|
-
process.on("SIGINT", wrapped);
|
|
2207
|
-
return () => {
|
|
2208
|
-
process.off("SIGTERM", wrapped);
|
|
2209
|
-
process.off("SIGINT", wrapped);
|
|
2210
|
-
};
|
|
2211
|
-
});
|
|
2212
|
-
detachSignal = onShutdownSignal(shutdown);
|
|
2213
1285
|
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
// A poll pass that was skipped because a watcher push held the guard is
|
|
2218
|
-
// benign — the next iteration retries after the poll interval.
|
|
2219
|
-
if (typeof result === "number" && result !== 0) {
|
|
2220
|
-
return result;
|
|
2221
|
-
}
|
|
2222
|
-
// Sleep the poll interval, but wake early on shutdown so SIGTERM stops
|
|
2223
|
-
// the loop promptly instead of waiting out a 10-minute cycle.
|
|
2224
|
-
await Promise.race([sleep(pollMs), stoppedSignal]);
|
|
2225
|
-
}
|
|
2226
|
-
return 0;
|
|
2227
|
-
} finally {
|
|
2228
|
-
shutdown();
|
|
2229
|
-
detachSignal?.();
|
|
1286
|
+
if (!argv.includes("--watch")) {
|
|
1287
|
+
if ("error" in parsed) return runRunner(argv);
|
|
1288
|
+
return runOneShotWithOperationLock(argv, parsed, deps, runtime);
|
|
2230
1289
|
}
|
|
2231
|
-
}
|
|
2232
1290
|
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
* re-targeted push pass inherits the same root and conflict policy. Pure
|
|
2236
|
-
* helper used by the event-push targeted-push composer.
|
|
2237
|
-
*/
|
|
2238
|
-
function carriedFlags(baseArgv: string[]): string[] {
|
|
2239
|
-
const carried: string[] = [];
|
|
2240
|
-
for (let i = 0; i < baseArgv.length; i++) {
|
|
2241
|
-
const a = baseArgv[i];
|
|
2242
|
-
if (a === "--hq-root" || a === "--on-conflict") {
|
|
2243
|
-
carried.push(a);
|
|
2244
|
-
if (baseArgv[i + 1] !== undefined) carried.push(baseArgv[++i]);
|
|
2245
|
-
}
|
|
2246
|
-
}
|
|
2247
|
-
return carried;
|
|
1291
|
+
if ("error" in parsed) return runRunner(argv);
|
|
1292
|
+
return runWatchLoop(argv, parsed, deps, runtime);
|
|
2248
1293
|
}
|
|
2249
1294
|
|
|
2250
1295
|
if (isDirectInvocation) {
|