@indigoai-us/hq-cloud 6.15.80 → 6.15.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +18 -0
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +20 -0
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +30 -1
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/backup-prune-runner.d.ts +4 -1
- package/dist/bin/backup-prune-runner.d.ts.map +1 -1
- package/dist/bin/backup-prune-runner.js +61 -38
- package/dist/bin/backup-prune-runner.js.map +1 -1
- package/dist/bin/backup-prune-runner.test.d.ts +2 -0
- package/dist/bin/backup-prune-runner.test.d.ts.map +1 -0
- package/dist/bin/backup-prune-runner.test.js +22 -0
- package/dist/bin/backup-prune-runner.test.js.map +1 -0
- package/dist/bin/layout-compatibility.d.ts +12 -0
- package/dist/bin/layout-compatibility.d.ts.map +1 -0
- package/dist/bin/layout-compatibility.js +50 -0
- package/dist/bin/layout-compatibility.js.map +1 -0
- package/dist/bin/layout-compatibility.test.d.ts +2 -0
- package/dist/bin/layout-compatibility.test.d.ts.map +1 -0
- package/dist/bin/layout-compatibility.test.js +87 -0
- package/dist/bin/layout-compatibility.test.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +15 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +65 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-events.test.js +20 -0
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +9 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +369 -21
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +5 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +11 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +165 -2
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +29 -5
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +131 -24
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +23 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +72 -7
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +199 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +22 -11
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts +18 -0
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +41 -17
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +8 -1
- package/dist/journal.test.js.map +1 -1
- package/dist/outposts/self-deploy.d.ts.map +1 -1
- package/dist/outposts/self-deploy.js +10 -1
- package/dist/outposts/self-deploy.js.map +1 -1
- package/dist/outposts/self-deploy.test.js +4 -1
- package/dist/outposts/self-deploy.test.js.map +1 -1
- package/dist/s3.d.ts +62 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +91 -1
- package/dist/s3.js.map +1 -1
- package/dist/s3.rename-retry.test.d.ts +2 -0
- package/dist/s3.rename-retry.test.d.ts.map +1 -0
- package/dist/s3.rename-retry.test.js +153 -0
- package/dist/s3.rename-retry.test.js.map +1 -0
- package/dist/sync/area-ledger-migration.d.ts +8 -1
- package/dist/sync/area-ledger-migration.d.ts.map +1 -1
- package/dist/sync/area-ledger-migration.js +23 -0
- package/dist/sync/area-ledger-migration.js.map +1 -1
- package/dist/sync/area-ledger-migration.test.js +20 -0
- package/dist/sync/area-ledger-migration.test.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +10 -0
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +11 -0
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/feature-flags.d.ts +29 -0
- package/dist/sync/feature-flags.d.ts.map +1 -1
- package/dist/sync/feature-flags.js +44 -0
- package/dist/sync/feature-flags.js.map +1 -1
- package/dist/sync/feature-flags.test.js +38 -1
- package/dist/sync/feature-flags.test.js.map +1 -1
- package/dist/sync/index.d.ts +6 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +3 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/layout-compatibility.d.ts +17 -0
- package/dist/sync/layout-compatibility.d.ts.map +1 -0
- package/dist/sync/layout-compatibility.js +29 -0
- package/dist/sync/layout-compatibility.js.map +1 -0
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
- package/dist/sync/parallel-sync-coordinator.js +0 -3
- package/dist/sync/parallel-sync-coordinator.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +14 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/per-area-maintenance.d.ts +45 -0
- package/dist/sync/per-area-maintenance.d.ts.map +1 -0
- package/dist/sync/per-area-maintenance.js +131 -0
- package/dist/sync/per-area-maintenance.js.map +1 -0
- package/dist/sync/per-area-maintenance.test.d.ts +2 -0
- package/dist/sync/per-area-maintenance.test.d.ts.map +1 -0
- package/dist/sync/per-area-maintenance.test.js +100 -0
- package/dist/sync/per-area-maintenance.test.js.map +1 -0
- package/dist/sync/realtime-telemetry.d.ts +49 -0
- package/dist/sync/realtime-telemetry.d.ts.map +1 -0
- package/dist/sync/realtime-telemetry.js +101 -0
- package/dist/sync/realtime-telemetry.js.map +1 -0
- package/dist/sync/realtime-telemetry.test.d.ts +2 -0
- package/dist/sync/realtime-telemetry.test.d.ts.map +1 -0
- package/dist/sync/realtime-telemetry.test.js +64 -0
- package/dist/sync/realtime-telemetry.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +10 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +18 -0
- package/dist/sync/state-store.js.map +1 -1
- package/package.json +3 -2
|
@@ -12,13 +12,22 @@ import { PARTIAL_SYNC_EXIT } from "../lib/exit-codes.js";
|
|
|
12
12
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
13
13
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
14
14
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
15
|
-
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournals, readJournal, writeJournal, } from "../journal.js";
|
|
15
|
+
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, readJournal, writeJournal, } from "../journal.js";
|
|
16
16
|
import { NoopPushReceiver, } from "../sync/push-receiver.js";
|
|
17
17
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
18
|
+
import { resolveParallelSyncRollout } from "../sync/feature-flags.js";
|
|
19
|
+
import { resolveAreaLayoutAuthority } from "../sync/area-ledger-migration.js";
|
|
20
|
+
import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
|
|
21
|
+
import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr } from "../sync/realtime-telemetry.js";
|
|
22
|
+
import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
18
23
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
19
24
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
20
25
|
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
21
26
|
const MAX_FULL_RECONCILE_REASONS = 7;
|
|
27
|
+
/** Area storage is an explicit canary and only applies after durable cutover. */
|
|
28
|
+
export function shouldUseAreaStorage(areaStorageEnabled, stateDir, journalSlug) {
|
|
29
|
+
return areaStorageEnabled && resolveAreaLayoutAuthority(stateDir, journalSlug) === "areas";
|
|
30
|
+
}
|
|
22
31
|
function emitFullReconcileDecision({ mode, reasons, triggers, pollTick, configuredIntervalMs, elapsedSinceLastCompletedFullReconcileMs, batchPathCount, }) {
|
|
23
32
|
process.stderr.write(JSON.stringify({
|
|
24
33
|
// Keep aggregate decision fields first: journald may truncate oversized
|
|
@@ -62,6 +71,16 @@ export const DEFAULT_FAST_LANE_MAX_BYTES = 4 * 1024 * 1024;
|
|
|
62
71
|
* leaving session logs and other transfer-heavy files in the slow lane.
|
|
63
72
|
*/
|
|
64
73
|
export const DEFAULT_FAST_LANE_MAX_FILE_BYTES = 256 * 1024;
|
|
74
|
+
/**
|
|
75
|
+
* A coordinator submission must not contain both a directory and one of its
|
|
76
|
+
* descendants: they are one mutation scope, and the lease manager correctly
|
|
77
|
+
* rejects internally-conflicting submissions. Prefer the widest scope so an
|
|
78
|
+
* unlinkDir following child unlink events remains admissible.
|
|
79
|
+
*/
|
|
80
|
+
export function normalizeCoordinatorPaths(paths) {
|
|
81
|
+
const ordered = [...new Set(paths)].sort((left, right) => left.length - right.length || left.localeCompare(right));
|
|
82
|
+
return ordered.filter((candidate, index) => !ordered.slice(0, index).some((ancestor) => candidate.startsWith(`${ancestor}/`)));
|
|
83
|
+
}
|
|
65
84
|
function positiveEnvInt(name, fallback) {
|
|
66
85
|
const value = Number.parseInt(process.env[name] ?? "", 10);
|
|
67
86
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
@@ -443,6 +462,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
443
462
|
// Fast work may bypass the lock only while this process *actually owns* it;
|
|
444
463
|
// otherwise it must participate in normal lock acquisition.
|
|
445
464
|
let processOwnsOperationLock = false;
|
|
465
|
+
// Compatibility chores still mutate whole routes/vaults. While one is
|
|
466
|
+
// executing, a coordinator realtime task must not take the normal fast-path
|
|
467
|
+
// lock carve-out: it has no bounded chore lease to prove independence.
|
|
468
|
+
let coordinatorWholeMutationChores = 0;
|
|
446
469
|
let operationLockUsers = 0;
|
|
447
470
|
let resolveNoOperationLockUsers = null;
|
|
448
471
|
const acquireOperationLockUser = () => {
|
|
@@ -469,7 +492,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
469
492
|
// its keyed journal delta and per-object mutation are independent from the
|
|
470
493
|
// slow pass's bulk paths, and the slow pass already owns the root lock.
|
|
471
494
|
// Never bypass the lock when no slow pass is active.
|
|
472
|
-
if (lane === "fast" &&
|
|
495
|
+
if (lane === "fast" &&
|
|
496
|
+
processOwnsOperationLock &&
|
|
497
|
+
coordinatorWholeMutationChores === 0) {
|
|
473
498
|
acquireOperationLockUser();
|
|
474
499
|
try {
|
|
475
500
|
prepare?.();
|
|
@@ -731,6 +756,137 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
731
756
|
});
|
|
732
757
|
};
|
|
733
758
|
const runGuarded = (passArgvForRun, prepare, lane = "slow") => runGuardedTask(() => runPassWithLock(passArgvForRun, prepare, lane), lane);
|
|
759
|
+
/*
|
|
760
|
+
* The bridge is intentionally opt-in. The legacy guarded pass arrays stay
|
|
761
|
+
* exactly as they are until an outpost enables HQ_SYNC_PARALLEL_COORDINATOR.
|
|
762
|
+
* Once enabled, the coordinator owns admission and path fencing; the small
|
|
763
|
+
* inline adapters are only the compatibility execution seam while the
|
|
764
|
+
* runner's real slice executors are being moved into child workers.
|
|
765
|
+
*/
|
|
766
|
+
const parallelRollout = resolveParallelSyncRollout();
|
|
767
|
+
const areaStorageEnabled = parallelRollout.areaStorageEnabled;
|
|
768
|
+
const parallelCoordinatorEnabled = parallelRollout.parallelExecutionEnabled;
|
|
769
|
+
const coordinatorExecutions = new WeakMap();
|
|
770
|
+
const coordinatorWorker = async (slice) => {
|
|
771
|
+
const execution = coordinatorExecutions.get(slice.work);
|
|
772
|
+
if (!execution) {
|
|
773
|
+
return {
|
|
774
|
+
paths: slice.paths.map((item) => ({
|
|
775
|
+
path: item.canonicalPath,
|
|
776
|
+
epoch: item.epoch,
|
|
777
|
+
status: "retryable",
|
|
778
|
+
reason: "coordinator execution was not registered",
|
|
779
|
+
})),
|
|
780
|
+
};
|
|
781
|
+
}
|
|
782
|
+
if (slice.work.class === "chore" && execution.wholeMutation)
|
|
783
|
+
coordinatorWholeMutationChores += 1;
|
|
784
|
+
try {
|
|
785
|
+
execution.result = await execution.run(slice);
|
|
786
|
+
}
|
|
787
|
+
finally {
|
|
788
|
+
if (slice.work.class === "chore" && execution.wholeMutation)
|
|
789
|
+
coordinatorWholeMutationChores -= 1;
|
|
790
|
+
}
|
|
791
|
+
const exitCode = passExitCode(execution.result);
|
|
792
|
+
return {
|
|
793
|
+
paths: slice.paths.map((item) => ({
|
|
794
|
+
path: item.canonicalPath,
|
|
795
|
+
epoch: item.epoch,
|
|
796
|
+
status: exitCode === 0 ? "durable" : "retryable",
|
|
797
|
+
...(exitCode === 0
|
|
798
|
+
? {}
|
|
799
|
+
: { reason: `sync pass exited ${exitCode}` }),
|
|
800
|
+
})),
|
|
801
|
+
};
|
|
802
|
+
};
|
|
803
|
+
const coordinator = parallelCoordinatorEnabled
|
|
804
|
+
? new DefaultParallelSyncCoordinator({
|
|
805
|
+
realtimeWorker: new InlineSyncWorkerAdapter(coordinatorWorker),
|
|
806
|
+
choreWorker: new InlineSyncWorkerAdapter(coordinatorWorker),
|
|
807
|
+
identifyPath: (canonicalPath) => {
|
|
808
|
+
const route = routeChangeToTarget(canonicalPath);
|
|
809
|
+
if (!route)
|
|
810
|
+
throw new Error(`coordinator cannot route ${canonicalPath}`);
|
|
811
|
+
return route.kind === "company"
|
|
812
|
+
? { targetUid: `company:${route.slug}`, areaId: `company:${route.slug}` }
|
|
813
|
+
: { targetUid: "personal", areaId: "personal" };
|
|
814
|
+
},
|
|
815
|
+
// Compatibility chores still use whole-vault/whole-route executors,
|
|
816
|
+
// which cannot yet name sound bounded mutation paths. Their execution
|
|
817
|
+
// therefore remains behind the runner's guarded operation queue until
|
|
818
|
+
// those executors can provide concrete scopes.
|
|
819
|
+
planChorePaths: () => [],
|
|
820
|
+
})
|
|
821
|
+
: null;
|
|
822
|
+
const runCoordinatorWork = async (work, task, options = {}) => {
|
|
823
|
+
const telemetry = work.class === "realtime" && parallelCoordinatorEnabled
|
|
824
|
+
? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr)
|
|
825
|
+
: undefined;
|
|
826
|
+
if (!coordinator) {
|
|
827
|
+
return {
|
|
828
|
+
result: await task({
|
|
829
|
+
id: "legacy-direct",
|
|
830
|
+
work,
|
|
831
|
+
paths: [],
|
|
832
|
+
signal: new AbortController().signal,
|
|
833
|
+
}),
|
|
834
|
+
accepted: true,
|
|
835
|
+
};
|
|
836
|
+
}
|
|
837
|
+
const execution = {
|
|
838
|
+
run: async (slice) => {
|
|
839
|
+
telemetry?.mark("worker");
|
|
840
|
+
telemetry?.mark("conflict");
|
|
841
|
+
telemetry?.mark("operationGate");
|
|
842
|
+
telemetry?.mark("appendLock");
|
|
843
|
+
telemetry?.mark("network");
|
|
844
|
+
telemetry?.mark("commit");
|
|
845
|
+
return task(slice);
|
|
846
|
+
},
|
|
847
|
+
wholeMutation: options.wholeMutation ?? false,
|
|
848
|
+
};
|
|
849
|
+
coordinatorExecutions.set(work, execution);
|
|
850
|
+
try {
|
|
851
|
+
const outcome = await coordinator.submit(work);
|
|
852
|
+
const executorSucceeded = execution.result !== undefined && passExitCode(execution.result) === 0;
|
|
853
|
+
const accepted = executorSucceeded && outcome.paths.every((path) => path.status === "durable");
|
|
854
|
+
if (telemetry) {
|
|
855
|
+
telemetry.finish(accepted
|
|
856
|
+
? "durable"
|
|
857
|
+
: outcome.paths.some((path) => path.status === "refused")
|
|
858
|
+
? "refused"
|
|
859
|
+
: "retryable");
|
|
860
|
+
}
|
|
861
|
+
return {
|
|
862
|
+
result: execution.result ?? 1,
|
|
863
|
+
accepted,
|
|
864
|
+
};
|
|
865
|
+
}
|
|
866
|
+
catch (error) {
|
|
867
|
+
telemetry?.finish("retryable");
|
|
868
|
+
throw error;
|
|
869
|
+
}
|
|
870
|
+
finally {
|
|
871
|
+
coordinatorExecutions.delete(work);
|
|
872
|
+
}
|
|
873
|
+
};
|
|
874
|
+
let nextCoordinatorCausality = 0;
|
|
875
|
+
const realtimeWork = (source, direction, paths) => ({
|
|
876
|
+
class: "realtime",
|
|
877
|
+
source,
|
|
878
|
+
direction,
|
|
879
|
+
paths: normalizeCoordinatorPaths(paths),
|
|
880
|
+
causality: `${source}-${++nextCoordinatorCausality}`,
|
|
881
|
+
});
|
|
882
|
+
const choreWork = (kind) => ({ class: "chore", kind, areas: [] });
|
|
883
|
+
const runRealtimePass = (source, direction, paths, task) => runCoordinatorWork(realtimeWork(source, direction, paths), task);
|
|
884
|
+
const enqueueChore = (kind, task, onSettled, options = { wholeMutation: true }) => {
|
|
885
|
+
void runCoordinatorWork(choreWork(kind), task, options).then(onSettled).catch((error) => {
|
|
886
|
+
process.stderr.write(`hq-sync-runner: coordinator ${kind} chore failed: ${describeError(error)}\n`);
|
|
887
|
+
onSettled({ result: 1, accepted: false });
|
|
888
|
+
});
|
|
889
|
+
};
|
|
734
890
|
let watcher = null;
|
|
735
891
|
let eventPushSurfacesActivated = false;
|
|
736
892
|
let watcherUnavailableWarned = false;
|
|
@@ -1417,6 +1573,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1417
1573
|
const skipCompanies = grouped ? resolveSkipCompanies() : null;
|
|
1418
1574
|
const isPausedRoute = (route) => route.kind === "company" &&
|
|
1419
1575
|
(skipCompanies?.has(route.slug.toLowerCase()) ?? false);
|
|
1576
|
+
const changesForAdmittedPaths = (group, admittedPaths) => new Map([...group.changes.entries()].filter(([absolutePath]) => {
|
|
1577
|
+
const relativePath = group.paths.get(absolutePath);
|
|
1578
|
+
return relativePath !== undefined && admittedPaths.some((scope) => relativePath === scope || relativePath.startsWith(`${scope}/`));
|
|
1579
|
+
}));
|
|
1420
1580
|
if (watcherDrainRunsPush && grouped) {
|
|
1421
1581
|
for (const group of grouped.values()) {
|
|
1422
1582
|
// Paused workspaces: drop the scoped push without uploading or
|
|
@@ -1424,15 +1584,24 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1424
1584
|
if (isPausedRoute(group.route)) {
|
|
1425
1585
|
continue;
|
|
1426
1586
|
}
|
|
1427
|
-
const scopedArgv = buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots);
|
|
1428
1587
|
try {
|
|
1429
1588
|
const lane = isFastLocalBatch(group.paths.keys(), group.changes)
|
|
1430
1589
|
? "fast"
|
|
1431
1590
|
: "slow";
|
|
1432
|
-
const
|
|
1433
|
-
? (
|
|
1434
|
-
|
|
1435
|
-
|
|
1591
|
+
const execution = parallelCoordinatorEnabled
|
|
1592
|
+
? await runRealtimePass("watcher", "push", group.relPaths, async (slice) => {
|
|
1593
|
+
const admittedPaths = slice.paths.map((path) => path.canonicalPath);
|
|
1594
|
+
const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
|
|
1595
|
+
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
1596
|
+
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
1597
|
+
? () => prepareWatcherChanges(admittedChanges)
|
|
1598
|
+
: undefined, lane);
|
|
1599
|
+
})
|
|
1600
|
+
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
1601
|
+
? () => prepareWatcherChanges(group.changes)
|
|
1602
|
+
: undefined, lane), accepted: true };
|
|
1603
|
+
const result = execution.result;
|
|
1604
|
+
if (execution.accepted && passExitCode(result) === 0) {
|
|
1436
1605
|
const refusedPaths = new Set(typeof result === "number"
|
|
1437
1606
|
? []
|
|
1438
1607
|
: (result.result?.pushPathResults ?? [])
|
|
@@ -1492,7 +1661,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1492
1661
|
continue;
|
|
1493
1662
|
ranPersonalDrain = true;
|
|
1494
1663
|
}
|
|
1495
|
-
const
|
|
1664
|
+
const execution = parallelCoordinatorEnabled
|
|
1665
|
+
? await runRealtimePass("watcher", "pull", group.relPaths, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, isFastLocalBatch(group.paths.keys(), group.changes) ? "fast" : "slow"))
|
|
1666
|
+
: { result: await runGuarded(pullArgv, undefined, isFastLocalBatch(group.paths.keys(), group.changes) ? "fast" : "slow"), accepted: true };
|
|
1667
|
+
const exit = execution.accepted ? passExitCode(execution.result) : 1;
|
|
1496
1668
|
if (exit !== 0 && worstExit === 0)
|
|
1497
1669
|
worstExit = exit;
|
|
1498
1670
|
}
|
|
@@ -1527,6 +1699,30 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1527
1699
|
for (const hint of batch.droppedRouteHints ?? [])
|
|
1528
1700
|
addRoute(hint);
|
|
1529
1701
|
const skipCompanies = resolveSkipCompanies();
|
|
1702
|
+
if (parallelCoordinatorEnabled) {
|
|
1703
|
+
for (const route of routes.values()) {
|
|
1704
|
+
if (route.kind === "company" &&
|
|
1705
|
+
skipCompanies.has(route.slug.toLowerCase())) {
|
|
1706
|
+
continue;
|
|
1707
|
+
}
|
|
1708
|
+
enqueueChore("backlog", async () => {
|
|
1709
|
+
if (watcherDrainRunsPush) {
|
|
1710
|
+
const push = await runGuarded(buildScopedPushArgv(route, [], passArgv));
|
|
1711
|
+
if (passExitCode(push) !== 0)
|
|
1712
|
+
return push;
|
|
1713
|
+
}
|
|
1714
|
+
return drainRunsPull
|
|
1715
|
+
? runGuarded(buildRoutePullArgv(route, passArgv))
|
|
1716
|
+
: 0;
|
|
1717
|
+
}, (execution) => {
|
|
1718
|
+
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
1719
|
+
process.stderr.write(`hq-sync-runner: backlog chore for ${routeKey(route)} failed; will retry on cadence\n`);
|
|
1720
|
+
interruptWait();
|
|
1721
|
+
}
|
|
1722
|
+
});
|
|
1723
|
+
}
|
|
1724
|
+
return 0;
|
|
1725
|
+
}
|
|
1530
1726
|
let worstExit = 0;
|
|
1531
1727
|
for (const route of routes.values()) {
|
|
1532
1728
|
// Paused workspaces sync in NO direction while Off.
|
|
@@ -1647,6 +1843,29 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1647
1843
|
const lanes = selectedBatch && !selectedBatch.overflowed
|
|
1648
1844
|
? partitionLocalBatch(selectedBatch)
|
|
1649
1845
|
: null;
|
|
1846
|
+
if (parallelCoordinatorEnabled && lanes) {
|
|
1847
|
+
// Preserve the ingress partition/debounce, but both partitions now
|
|
1848
|
+
// submit semantic realtime work. In particular a fresh large file
|
|
1849
|
+
// never waits for a whole-backlog chore merely because it exceeded
|
|
1850
|
+
// the old fast byte cap.
|
|
1851
|
+
for (const lane of [lanes.fast, lanes.slow]) {
|
|
1852
|
+
if (!lane)
|
|
1853
|
+
continue;
|
|
1854
|
+
const generation = ++nextWatcherBatchGeneration;
|
|
1855
|
+
void runScopedDrain(lane, generation, true).then((exit) => {
|
|
1856
|
+
if (exit === 0)
|
|
1857
|
+
return;
|
|
1858
|
+
restorePendingWatcherPaths(lane.paths, generation);
|
|
1859
|
+
interruptWait();
|
|
1860
|
+
}).catch((err) => {
|
|
1861
|
+
restorePendingWatcherPaths(lane.paths, generation);
|
|
1862
|
+
process.stderr.write(`watch coordinator scoped push failed, will retry next tick: ${describeError(err)}\n`);
|
|
1863
|
+
interruptWait();
|
|
1864
|
+
});
|
|
1865
|
+
}
|
|
1866
|
+
realtimeState.scheduler?.signal("watcher");
|
|
1867
|
+
return;
|
|
1868
|
+
}
|
|
1650
1869
|
if (lanes?.fast &&
|
|
1651
1870
|
activeSlowPass !== null) {
|
|
1652
1871
|
const generation = ++nextWatcherBatchGeneration;
|
|
@@ -1768,6 +1987,28 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1768
1987
|
// Apply the caps once across the whole delivery, then retain the residue
|
|
1769
1988
|
// in the normal lane. The receiver already coalesces to one latest event
|
|
1770
1989
|
// per path, so the partition cannot split revisions of one path.
|
|
1990
|
+
if (parallelCoordinatorEnabled) {
|
|
1991
|
+
// #423's coalescing/caps remain ingress protection only. A newly
|
|
1992
|
+
// delivered single large path is realtime work, not a slow-pass item;
|
|
1993
|
+
// byte size is handled by the transfer budget below the coordinator.
|
|
1994
|
+
const pathsByRoute = new Map();
|
|
1995
|
+
for (const event of events) {
|
|
1996
|
+
const route = routeChangeToTarget(event.relativePath);
|
|
1997
|
+
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
1998
|
+
continue;
|
|
1999
|
+
const key = routeKey(route);
|
|
2000
|
+
const group = pathsByRoute.get(key) ?? { route, paths: [] };
|
|
2001
|
+
group.paths.push(event.relativePath);
|
|
2002
|
+
pathsByRoute.set(key, group);
|
|
2003
|
+
}
|
|
2004
|
+
for (const { route, paths } of pathsByRoute.values()) {
|
|
2005
|
+
const execution = await runRealtimePass("receiver", "pull", paths, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, "slow"));
|
|
2006
|
+
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
2007
|
+
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
2008
|
+
}
|
|
2009
|
+
}
|
|
2010
|
+
return;
|
|
2011
|
+
}
|
|
1771
2012
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
1772
2013
|
const drainLane = async (lane, laneEvents) => {
|
|
1773
2014
|
const pathsByRoute = new Map();
|
|
@@ -1800,6 +2041,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1800
2041
|
await drainLane("slow", lanes.slow);
|
|
1801
2042
|
};
|
|
1802
2043
|
receiverSyncBatchFn.partition = (events) => {
|
|
2044
|
+
if (parallelCoordinatorEnabled)
|
|
2045
|
+
return events.length > 0 ? [events] : [];
|
|
1803
2046
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
1804
2047
|
return [lanes.fast, lanes.slow].filter((lane) => lane.length > 0);
|
|
1805
2048
|
};
|
|
@@ -1821,10 +2064,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1821
2064
|
runGuarded: async (drain) => {
|
|
1822
2065
|
let outcome;
|
|
1823
2066
|
try {
|
|
1824
|
-
|
|
1825
|
-
await
|
|
1826
|
-
|
|
1827
|
-
|
|
2067
|
+
if (parallelCoordinatorEnabled) {
|
|
2068
|
+
const execution = await runRealtimePass("wake", "pull", [], async () => runGuardedTask(() => runTaskWithLock(async () => {
|
|
2069
|
+
await drain();
|
|
2070
|
+
return 0;
|
|
2071
|
+
})));
|
|
2072
|
+
outcome = execution.accepted ? execution.result : 1;
|
|
2073
|
+
}
|
|
2074
|
+
else {
|
|
2075
|
+
outcome = await runGuardedTask(() => runTaskWithLock(async () => {
|
|
2076
|
+
await drain();
|
|
2077
|
+
return 0;
|
|
2078
|
+
}));
|
|
2079
|
+
}
|
|
1828
2080
|
}
|
|
1829
2081
|
catch (err) {
|
|
1830
2082
|
// Scheduler clients may have already been disposed when a queued
|
|
@@ -1899,11 +2151,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1899
2151
|
});
|
|
1900
2152
|
}
|
|
1901
2153
|
}
|
|
2154
|
+
let coordinatorShutdown = null;
|
|
1902
2155
|
const shutdown = () => {
|
|
1903
2156
|
if (stopped)
|
|
1904
|
-
return;
|
|
2157
|
+
return coordinatorShutdown ?? Promise.resolve();
|
|
1905
2158
|
stopped = true;
|
|
1906
2159
|
resolveStoppedQueue();
|
|
2160
|
+
// Reject queued coordinator work before tearing down ingress. Active
|
|
2161
|
+
// work receives its abort signal; receiver receipts remain unacknowledged
|
|
2162
|
+
// unless their durable pull has already completed.
|
|
2163
|
+
coordinatorShutdown = coordinator?.shutdown() ?? Promise.resolve();
|
|
1907
2164
|
try {
|
|
1908
2165
|
void receiver?.dispose();
|
|
1909
2166
|
}
|
|
@@ -1931,10 +2188,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1931
2188
|
/* ignore */
|
|
1932
2189
|
}
|
|
1933
2190
|
resolveStopped?.();
|
|
2191
|
+
return coordinatorShutdown;
|
|
1934
2192
|
};
|
|
1935
2193
|
const onShutdownSignal = deps.onShutdownSignal ??
|
|
1936
2194
|
((handler) => {
|
|
1937
|
-
const wrapped = () => handler();
|
|
2195
|
+
const wrapped = () => { void handler(); };
|
|
1938
2196
|
process.on("SIGTERM", wrapped);
|
|
1939
2197
|
process.on("SIGINT", wrapped);
|
|
1940
2198
|
return () => {
|
|
@@ -1948,8 +2206,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1948
2206
|
let nextDueAt = monotonicNow();
|
|
1949
2207
|
let pollTick = 0;
|
|
1950
2208
|
let lastCompletedFullReconcileAt = null;
|
|
2209
|
+
let fullChorePending = false;
|
|
2210
|
+
let fullChoreFollowupPending = false;
|
|
2211
|
+
let coordinatorAuthRequired = false;
|
|
1951
2212
|
try {
|
|
1952
2213
|
while (!stopped) {
|
|
2214
|
+
if (coordinatorAuthRequired)
|
|
2215
|
+
return 0;
|
|
1953
2216
|
pollTick++;
|
|
1954
2217
|
const pending = takePendingWatcherChange();
|
|
1955
2218
|
// The quarantine is scoped to the delete episode it refused, and only
|
|
@@ -2020,10 +2283,58 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2020
2283
|
};
|
|
2021
2284
|
let result;
|
|
2022
2285
|
let ranFullReconcile = false;
|
|
2286
|
+
let enqueuedFullChore = false;
|
|
2023
2287
|
if (shouldRunFull) {
|
|
2024
|
-
ranFullReconcile = true;
|
|
2025
2288
|
emitDecision("full", reasons);
|
|
2026
|
-
|
|
2289
|
+
if (parallelCoordinatorEnabled) {
|
|
2290
|
+
// Cadence/full/backlog work is admitted to the reserved chore worker
|
|
2291
|
+
// and deliberately never awaited by the watch loop. The completion
|
|
2292
|
+
// callback, rather than submission, advances the full-reconcile
|
|
2293
|
+
// clock and turns on event surfaces.
|
|
2294
|
+
if (!fullChorePending) {
|
|
2295
|
+
fullChorePending = true;
|
|
2296
|
+
enqueueChore("full-reconcile", () => runGuarded(passArgv), (execution) => {
|
|
2297
|
+
fullChorePending = false;
|
|
2298
|
+
const exit = passExitCode(execution.result);
|
|
2299
|
+
if (exit === runtime.authRequiredPassExit) {
|
|
2300
|
+
coordinatorAuthRequired = true;
|
|
2301
|
+
interruptWait();
|
|
2302
|
+
return;
|
|
2303
|
+
}
|
|
2304
|
+
if (execution.accepted && exit === 0) {
|
|
2305
|
+
lastCompletedFullReconcileAt = monotonicNow();
|
|
2306
|
+
activateEventPushSurfaces();
|
|
2307
|
+
if (fullChoreFollowupPending) {
|
|
2308
|
+
fullChoreFollowupPending = false;
|
|
2309
|
+
interruptWait();
|
|
2310
|
+
}
|
|
2311
|
+
return;
|
|
2312
|
+
}
|
|
2313
|
+
process.stderr.write(`hq-sync-runner: full reconcile chore failed (exit=${exit}); will retry on cadence\n`);
|
|
2314
|
+
if (fullChoreFollowupPending) {
|
|
2315
|
+
fullChoreFollowupPending = false;
|
|
2316
|
+
interruptWait();
|
|
2317
|
+
}
|
|
2318
|
+
});
|
|
2319
|
+
}
|
|
2320
|
+
else {
|
|
2321
|
+
// A full pass can have already scanned this route. Preserve any
|
|
2322
|
+
// incoming evidence and make the completion schedule one more
|
|
2323
|
+
// reconcile rather than dropping the taken batch.
|
|
2324
|
+
if (pending.batch) {
|
|
2325
|
+
restorePendingWatcherPaths(pending.batch.paths, pending.generation);
|
|
2326
|
+
}
|
|
2327
|
+
if (pending.bare)
|
|
2328
|
+
pendingWatcherBareChange = true;
|
|
2329
|
+
fullChoreFollowupPending ||= pending.batch !== null || pending.bare;
|
|
2330
|
+
}
|
|
2331
|
+
enqueuedFullChore = true;
|
|
2332
|
+
result = 0;
|
|
2333
|
+
}
|
|
2334
|
+
else {
|
|
2335
|
+
ranFullReconcile = true;
|
|
2336
|
+
result = await runGuarded(passArgv);
|
|
2337
|
+
}
|
|
2027
2338
|
}
|
|
2028
2339
|
else if (pending.batch === null) {
|
|
2029
2340
|
// There is no watcher batch and no full-reconcile trigger. Do not
|
|
@@ -2104,9 +2415,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2104
2415
|
else if (exitCode !== 0) {
|
|
2105
2416
|
return exitCode;
|
|
2106
2417
|
}
|
|
2107
|
-
const completedPostRepairPass = typeof result === "number"
|
|
2418
|
+
const completedPostRepairPass = !enqueuedFullChore && (typeof result === "number"
|
|
2108
2419
|
? exitCode === 0
|
|
2109
|
-
: result.result !== undefined;
|
|
2420
|
+
: result.result !== undefined);
|
|
2110
2421
|
if (completedPostRepairPass) {
|
|
2111
2422
|
// The production adapter returns RunnerPassResult only after fanout,
|
|
2112
2423
|
// which follows journal repair. Its numeric seam has the same
|
|
@@ -2118,9 +2429,32 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2118
2429
|
if (Date.now() >= nextBackupPruneAt) {
|
|
2119
2430
|
// Advance before invoking: a failed/unavailable backstop cannot become
|
|
2120
2431
|
// an every-tick retry loop. It is diagnostic only and never restarts
|
|
2121
|
-
// the sync daemon.
|
|
2122
|
-
//
|
|
2432
|
+
// the sync daemon. Under coordinator mode it is a reserved-worker
|
|
2433
|
+
// chore, never a dependency of realtime admission.
|
|
2123
2434
|
nextBackupPruneAt = Date.now() + 24 * 60 * 60 * 1000;
|
|
2435
|
+
if (parallelCoordinatorEnabled) {
|
|
2436
|
+
enqueueChore("rotation", async () => runGuardedTask(() => runTaskWithLock(async () => {
|
|
2437
|
+
await runBackupPrune(hqRoot);
|
|
2438
|
+
return 0;
|
|
2439
|
+
})), (execution) => {
|
|
2440
|
+
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
2441
|
+
process.stderr.write("hq-sync-runner: backup retention chore failed; will retry tomorrow\n");
|
|
2442
|
+
}
|
|
2443
|
+
}, { wholeMutation: true });
|
|
2444
|
+
const epoch = new Date().toISOString().slice(0, 10);
|
|
2445
|
+
for (const slug of listJournalSlugs()) {
|
|
2446
|
+
if (!shouldUseAreaStorage(areaStorageEnabled, getStateDir(), slug))
|
|
2447
|
+
continue;
|
|
2448
|
+
enqueueChore("rotation", async () => {
|
|
2449
|
+
const maintenance = maintainAreaJournalSlice(slug, epoch);
|
|
2450
|
+
if (maintenance?.outcome?.status === "isolated") {
|
|
2451
|
+
process.stderr.write(`hq-sync-runner: isolated area maintenance failure for ${slug}/${maintenance.outcome.areaId}: ${maintenance.outcome.error}\n`);
|
|
2452
|
+
}
|
|
2453
|
+
return 0;
|
|
2454
|
+
}, () => undefined, { wholeMutation: false });
|
|
2455
|
+
}
|
|
2456
|
+
continue;
|
|
2457
|
+
}
|
|
2124
2458
|
try {
|
|
2125
2459
|
await runGuardedTask(() => runTaskWithLock(async () => {
|
|
2126
2460
|
await runBackupPrune(hqRoot);
|
|
@@ -2130,6 +2464,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2130
2464
|
catch (error) {
|
|
2131
2465
|
process.stderr.write(`hq-sync-runner: backup retention dry-run failed — ${error instanceof Error ? error.message : String(error)}\n`);
|
|
2132
2466
|
}
|
|
2467
|
+
if (areaStorageEnabled) {
|
|
2468
|
+
const epoch = new Date().toISOString().slice(0, 10);
|
|
2469
|
+
for (const slug of listJournalSlugs()) {
|
|
2470
|
+
if (!shouldUseAreaStorage(true, getStateDir(), slug))
|
|
2471
|
+
continue;
|
|
2472
|
+
await runGuardedTask(async () => {
|
|
2473
|
+
const maintenance = maintainAreaJournalSlice(slug, epoch);
|
|
2474
|
+
if (maintenance?.outcome?.status === "isolated") {
|
|
2475
|
+
process.stderr.write(`hq-sync-runner: isolated area maintenance failure for ${slug}/${maintenance.outcome.areaId}: ${maintenance.outcome.error}\n`);
|
|
2476
|
+
}
|
|
2477
|
+
return 0;
|
|
2478
|
+
});
|
|
2479
|
+
}
|
|
2480
|
+
}
|
|
2133
2481
|
}
|
|
2134
2482
|
const nextPollMs = explicitPollMs ?? resolveWatchPollMs(undefined, receiverHasQueuedWork, receiverHasQueuedWork ? 0 : sampleLoadAvg(), cpuCount);
|
|
2135
2483
|
const activeRealtimeScheduler = realtimeState.scheduler;
|
|
@@ -2160,7 +2508,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2160
2508
|
return 0;
|
|
2161
2509
|
}
|
|
2162
2510
|
finally {
|
|
2163
|
-
shutdown();
|
|
2511
|
+
await shutdown();
|
|
2164
2512
|
detachSignal?.();
|
|
2165
2513
|
}
|
|
2166
2514
|
}
|