@indigoai-us/hq-cloud 6.16.64 → 6.16.66-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +14 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +42 -8
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +62 -7
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +172 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +52 -3
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +451 -167
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +19 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +40 -5
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +47 -0
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +64 -6
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +48 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +1577 -169
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/push-scan-exclusions.test.js +4 -4
- package/dist/cli/push-scan-exclusions.test.js.map +1 -1
- package/dist/cli/share.d.ts +31 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +136 -6
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +295 -5
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +12 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +302 -123
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +255 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/hq-root-recovery.d.ts +30 -0
- package/dist/hq-root-recovery.d.ts.map +1 -0
- package/dist/hq-root-recovery.js +131 -0
- package/dist/hq-root-recovery.js.map +1 -0
- package/dist/hq-root-recovery.test.d.ts +2 -0
- package/dist/hq-root-recovery.test.d.ts.map +1 -0
- package/dist/hq-root-recovery.test.js +65 -0
- package/dist/hq-root-recovery.test.js.map +1 -0
- package/dist/journal-row-store.d.ts +40 -5
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +240 -21
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +54 -0
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +11 -6
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +324 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +51 -16
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +303 -79
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/object-io.d.ts +33 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +145 -8
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +248 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +20 -2
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/sync/area-ledger.d.ts +27 -3
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +149 -39
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +192 -5
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +29 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/presign-denial-backoff-flag.d.ts +22 -0
- package/dist/sync/presign-denial-backoff-flag.d.ts.map +1 -0
- package/dist/sync/presign-denial-backoff-flag.js +75 -0
- package/dist/sync/presign-denial-backoff-flag.js.map +1 -0
- package/dist/sync/presign-denial-backoff-flag.test.d.ts +2 -0
- package/dist/sync/presign-denial-backoff-flag.test.d.ts.map +1 -0
- package/dist/sync/presign-denial-backoff-flag.test.js +103 -0
- package/dist/sync/presign-denial-backoff-flag.test.js.map +1 -0
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +6 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +14 -2
- package/dist/sync/state-store.js.map +1 -1
- package/dist/watch-coverage.d.ts +24 -0
- package/dist/watch-coverage.d.ts.map +1 -0
- package/dist/watch-coverage.js +46 -0
- package/dist/watch-coverage.js.map +1 -0
- package/dist/watch-coverage.test.d.ts +2 -0
- package/dist/watch-coverage.test.d.ts.map +1 -0
- package/dist/watch-coverage.test.js +56 -0
- package/dist/watch-coverage.test.js.map +1 -0
- package/dist/watcher.d.ts +31 -1
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +172 -0
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +78 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +3 -1
|
@@ -10,11 +10,12 @@ import { pickCanonicalPersonEntity } from "../vault-client.js";
|
|
|
10
10
|
import { acquireOperationLock, withOperationLock, withOperationLockSync, OperationLockedError, OPERATION_LOCKED_EXIT, } from "../operation-lock.js";
|
|
11
11
|
import { describeError } from "../lib/describe-error.js";
|
|
12
12
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
13
|
-
import { PARTIAL_SYNC_EXIT } from "../lib/exit-codes.js";
|
|
13
|
+
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
14
14
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
15
15
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
16
16
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
17
|
-
import {
|
|
17
|
+
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
18
|
+
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, journalRows, readJournalScoped, writeJournal, } from "../journal.js";
|
|
18
19
|
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
19
20
|
import { NoopPushReceiver, } from "../sync/push-receiver.js";
|
|
20
21
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
@@ -24,12 +25,226 @@ import { AREA_AUTO_MIGRATION_SOURCE, createAreaAutoMigrationScheduler, openPerso
|
|
|
24
25
|
import { emitCloudTelemetry } from "../telemetry-events.js";
|
|
25
26
|
import { HeapCensus, startRunnerHeapSampling, heapSampleCloudProperties, heapPressureCloudProperties, heapPressureStderrLine, heapRecycleCloudProperties, heapRecycleStderrLine, performHeapHardExit, nodeHeapReaders, nodeMajorGcObserver, resolveDeclaredHeapCeilingBytes, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled, resolveHeapRecycleLiveFraction, resolveHeapRecycleSustainWindowMs, resolveHeapRecycleEmergencyFraction, resolveHeapRecycleMinUptimeMs, resolveHeapCheckpointMs, resolveHeapPinnedFraction, resolveHeapPinnedSustainMs, resolveHeapHardExitFraction, resolveHeapRecycleLatchHardExitMs, resolveHeapFootprintRecycleEnabled, resolveHeapFootprintRecycleBytes, resolveHeapFootprintHardExitBytes, HEAP_SAMPLE_INTERVAL_ENV, HEAP_PRESSURE_FRACTION_ENV, HEAP_RECYCLE_ENABLED_ENV, HEAP_RECYCLE_LIVE_FRACTION_ENV, HEAP_RECYCLE_SUSTAIN_WINDOW_MS_ENV, HEAP_RECYCLE_EMERGENCY_FRACTION_ENV, HEAP_RECYCLE_MIN_UPTIME_MS_ENV, HEAP_CHECKPOINT_MS_ENV, HEAP_PINNED_FRACTION_ENV, HEAP_PINNED_SUSTAIN_MS_ENV, HEAP_HARD_EXIT_FRACTION_ENV, HEAP_RECYCLE_LATCH_HARD_EXIT_MS_ENV, HEAP_FOOTPRINT_RECYCLE_ENABLED_ENV, HEAP_FOOTPRINT_RECYCLE_MB_ENV, HEAP_FOOTPRINT_HARD_EXIT_MB_ENV, } from "./sync-runner-heap-telemetry.js";
|
|
26
27
|
import { setRunnerEmitObserver } from "./sync-runner-events.js";
|
|
28
|
+
import { refreshPresignDenialBackoffFlagWithin } from "../sync/presign-denial-backoff-flag.js";
|
|
27
29
|
import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
|
|
28
30
|
import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr } from "../sync/realtime-telemetry.js";
|
|
29
31
|
import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
30
32
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
31
33
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
32
34
|
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
35
|
+
/** A deferred full chore must get the next admission after this long. */
|
|
36
|
+
export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
37
|
+
/**
|
|
38
|
+
* One durable-state owner at a time for a full chore. Realtime work never runs
|
|
39
|
+
* beside that chore: it explicitly awaits `checkpoint()` after a completed
|
|
40
|
+
* durable unit, then resumes once the hand-off is finished. A separately
|
|
41
|
+
* tracked, explicitly opted-in watcher fast pass may use the pre-existing
|
|
42
|
+
* scoped fast-over-slow exception, but never while a chore owns durable state.
|
|
43
|
+
*/
|
|
44
|
+
export class CooperativePassScheduler {
|
|
45
|
+
now;
|
|
46
|
+
starvationCapMs;
|
|
47
|
+
pending = [];
|
|
48
|
+
active = null;
|
|
49
|
+
// Scoped fast realtime work has the established in-process lock exception
|
|
50
|
+
// while a scoped slow realtime upload is in flight. Keep that slot separate
|
|
51
|
+
// from `active`: a chore still has exactly one durable-state owner.
|
|
52
|
+
activeFastRealtime = null;
|
|
53
|
+
stopped = false;
|
|
54
|
+
_handoffActive = false;
|
|
55
|
+
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS) {
|
|
56
|
+
this.now = now;
|
|
57
|
+
this.starvationCapMs = starvationCapMs;
|
|
58
|
+
}
|
|
59
|
+
get handoffActive() {
|
|
60
|
+
return this._handoffActive;
|
|
61
|
+
}
|
|
62
|
+
get hasActiveChore() {
|
|
63
|
+
return this.active?.workClass === "chore";
|
|
64
|
+
}
|
|
65
|
+
get hasActiveSlowRealtime() {
|
|
66
|
+
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
67
|
+
}
|
|
68
|
+
pendingLaneCount(lane) {
|
|
69
|
+
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
70
|
+
}
|
|
71
|
+
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false) {
|
|
72
|
+
if (this.stopped)
|
|
73
|
+
return Promise.resolve(0);
|
|
74
|
+
const directEntry = {
|
|
75
|
+
task,
|
|
76
|
+
workClass,
|
|
77
|
+
lane,
|
|
78
|
+
allowFastRealtimeOvertake,
|
|
79
|
+
// An immediately admitted task cannot starve, so avoid sampling the
|
|
80
|
+
// cadence clock merely to stamp it. This preserves the loop's deadline
|
|
81
|
+
// accounting and keeps the timestamp meaningful only for real queues.
|
|
82
|
+
enqueuedAt: 0,
|
|
83
|
+
resolve: () => undefined,
|
|
84
|
+
reject: () => undefined,
|
|
85
|
+
};
|
|
86
|
+
if (this.canStartFastRealtime(directEntry)) {
|
|
87
|
+
return this.startFastRealtime(directEntry);
|
|
88
|
+
}
|
|
89
|
+
if (this.active === null &&
|
|
90
|
+
this.activeFastRealtime === null &&
|
|
91
|
+
this.pending.length === 0) {
|
|
92
|
+
return this.start(directEntry);
|
|
93
|
+
}
|
|
94
|
+
return new Promise((resolve, reject) => {
|
|
95
|
+
const entry = { ...directEntry, enqueuedAt: this.now(), resolve, reject };
|
|
96
|
+
this.pending.push(entry);
|
|
97
|
+
this.drain();
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
stop() {
|
|
101
|
+
this.stopped = true;
|
|
102
|
+
while (this.pending.length > 0)
|
|
103
|
+
this.pending.shift()?.resolve(0);
|
|
104
|
+
}
|
|
105
|
+
drain() {
|
|
106
|
+
if (this.stopped || this.pending.length === 0)
|
|
107
|
+
return;
|
|
108
|
+
if (this.active !== null) {
|
|
109
|
+
const fastRealtime = this.nextFastRealtime();
|
|
110
|
+
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
111
|
+
this.pending.splice(this.pending.indexOf(fastRealtime), 1);
|
|
112
|
+
this.startFastRealtime(fastRealtime);
|
|
113
|
+
}
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
if (this.activeFastRealtime !== null)
|
|
117
|
+
return;
|
|
118
|
+
const next = this.takeNext();
|
|
119
|
+
if (next)
|
|
120
|
+
this.start(next);
|
|
121
|
+
}
|
|
122
|
+
takeNext() {
|
|
123
|
+
const now = this.now();
|
|
124
|
+
const starvedChore = this.pending.findIndex((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
125
|
+
if (starvedChore >= 0)
|
|
126
|
+
return this.pending.splice(starvedChore, 1)[0];
|
|
127
|
+
// A receiver's bulk drain is still realtime work, but its slow lane is the
|
|
128
|
+
// progress boundary for that delivery. Preserve its established turn
|
|
129
|
+
// before refilling later fast realtime arrivals; this does not let a
|
|
130
|
+
// chore overtake any realtime work.
|
|
131
|
+
const realtime = this.nextRealtimeIndex();
|
|
132
|
+
if (realtime >= 0)
|
|
133
|
+
return this.pending.splice(realtime, 1)[0];
|
|
134
|
+
return this.pending.shift();
|
|
135
|
+
}
|
|
136
|
+
nextRealtimeIndex() {
|
|
137
|
+
const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow");
|
|
138
|
+
return slowRealtime >= 0
|
|
139
|
+
? slowRealtime
|
|
140
|
+
: this.pending.findIndex((entry) => entry.workClass === "realtime");
|
|
141
|
+
}
|
|
142
|
+
nextFastRealtime() {
|
|
143
|
+
return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
|
|
144
|
+
}
|
|
145
|
+
hasStarvedChore() {
|
|
146
|
+
const now = this.now();
|
|
147
|
+
return this.pending.some((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
148
|
+
}
|
|
149
|
+
canStartFastRealtime(entry) {
|
|
150
|
+
return (!this._handoffActive &&
|
|
151
|
+
this.activeFastRealtime === null &&
|
|
152
|
+
entry.workClass === "realtime" &&
|
|
153
|
+
entry.lane === "fast" &&
|
|
154
|
+
entry.allowFastRealtimeOvertake &&
|
|
155
|
+
this.hasActiveSlowRealtime &&
|
|
156
|
+
!this.hasStarvedChore());
|
|
157
|
+
}
|
|
158
|
+
start(entry) {
|
|
159
|
+
this.active = entry;
|
|
160
|
+
let current;
|
|
161
|
+
try {
|
|
162
|
+
current = this.execute(entry);
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
current = Promise.reject(error);
|
|
166
|
+
}
|
|
167
|
+
void current.then((outcome) => {
|
|
168
|
+
entry.resolve(outcome);
|
|
169
|
+
this.complete(entry);
|
|
170
|
+
}, (error) => {
|
|
171
|
+
entry.reject(error);
|
|
172
|
+
this.complete(entry);
|
|
173
|
+
}).catch(() => undefined);
|
|
174
|
+
return current;
|
|
175
|
+
}
|
|
176
|
+
complete(entry) {
|
|
177
|
+
if (this.active === entry)
|
|
178
|
+
this.active = null;
|
|
179
|
+
this.drain();
|
|
180
|
+
}
|
|
181
|
+
startFastRealtime(entry) {
|
|
182
|
+
this.activeFastRealtime = entry;
|
|
183
|
+
let current;
|
|
184
|
+
try {
|
|
185
|
+
current = this.execute(entry);
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
current = Promise.reject(error);
|
|
189
|
+
}
|
|
190
|
+
void current.then((outcome) => {
|
|
191
|
+
entry.resolve(outcome);
|
|
192
|
+
this.completeFastRealtime(entry);
|
|
193
|
+
}, (error) => {
|
|
194
|
+
entry.reject(error);
|
|
195
|
+
this.completeFastRealtime(entry);
|
|
196
|
+
});
|
|
197
|
+
return current;
|
|
198
|
+
}
|
|
199
|
+
completeFastRealtime(entry) {
|
|
200
|
+
if (this.activeFastRealtime === entry)
|
|
201
|
+
this.activeFastRealtime = null;
|
|
202
|
+
this.drain();
|
|
203
|
+
}
|
|
204
|
+
execute(entry) {
|
|
205
|
+
return entry.task({ checkpoint: () => this.checkpoint(entry) });
|
|
206
|
+
}
|
|
207
|
+
async checkpoint(entry) {
|
|
208
|
+
if (this.stopped ||
|
|
209
|
+
entry.workClass !== "chore" ||
|
|
210
|
+
this.active !== entry ||
|
|
211
|
+
!this.pending.some((pending) => pending.workClass === "realtime")) {
|
|
212
|
+
return false;
|
|
213
|
+
}
|
|
214
|
+
// `entry` is suspended at a caller-declared durable boundary. It remains
|
|
215
|
+
// the sole full-pass continuation and will resume after this bounded burst.
|
|
216
|
+
const deferredAt = this.now();
|
|
217
|
+
this.active = null;
|
|
218
|
+
this._handoffActive = true;
|
|
219
|
+
let handedOff = false;
|
|
220
|
+
try {
|
|
221
|
+
while (!this.stopped && this.now() - deferredAt < this.starvationCapMs) {
|
|
222
|
+
const realtimeIndex = this.nextRealtimeIndex();
|
|
223
|
+
if (realtimeIndex < 0)
|
|
224
|
+
break;
|
|
225
|
+
const realtime = this.pending.splice(realtimeIndex, 1)[0];
|
|
226
|
+
handedOff = true;
|
|
227
|
+
this.active = realtime;
|
|
228
|
+
try {
|
|
229
|
+
realtime.resolve(await this.execute(realtime));
|
|
230
|
+
}
|
|
231
|
+
catch (error) {
|
|
232
|
+
realtime.reject(error);
|
|
233
|
+
}
|
|
234
|
+
finally {
|
|
235
|
+
if (this.active === realtime)
|
|
236
|
+
this.active = null;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
finally {
|
|
241
|
+
this._handoffActive = false;
|
|
242
|
+
if (!this.stopped)
|
|
243
|
+
this.active = entry;
|
|
244
|
+
}
|
|
245
|
+
return handedOff;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
33
248
|
const MAX_FULL_RECONCILE_REASONS = 7;
|
|
34
249
|
/**
|
|
35
250
|
* Area storage is AUTO by default and only applies after durable cutover.
|
|
@@ -62,7 +277,7 @@ function emitFullReconcileDecision({ mode, reasons, triggers, pollTick, configur
|
|
|
62
277
|
* is drained in bounded chunks across successive ticks instead of one giant
|
|
63
278
|
* scoped pass. Size is not a full-reconcile signal — unknown scope is
|
|
64
279
|
* (`overflowed` with no route hints, warm-up catch-up failure, first tick,
|
|
65
|
-
* scheduled interval, bare wake). The crossover for a *chunk* is the number
|
|
280
|
+
* scheduled interval, bare wake, vanished HQ-root coverage). The crossover for a *chunk* is the number
|
|
66
281
|
* of `--scope-path` arguments a targeted pass can carry without becoming a
|
|
67
282
|
* de-facto whole-tree walk. `HQ_SYNC_EVENT_BATCH_LIMIT` overrides; 0/invalid
|
|
68
283
|
* = default. Do not raise this to "fix" a large warm-up catch-up: that only
|
|
@@ -509,6 +724,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
509
724
|
const idleHeartbeatIntervalMs = deps.idleHeartbeatIntervalMs ?? WATCH_IDLE_HEARTBEAT_INTERVAL_MS;
|
|
510
725
|
const monotonicNow = deps.monotonicNow ?? (() => performance.now());
|
|
511
726
|
const runPass = deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
|
|
727
|
+
const runPassSupportsCooperativeCheckpoint = deps.runPass === undefined || deps.runPassSupportsCooperativeCheckpoint === true;
|
|
728
|
+
// Read the switch before the first pass, which is the largest presign user.
|
|
729
|
+
if (process.env.HQ_FLAGS_API_URL && process.env.VITEST !== "true") {
|
|
730
|
+
await refreshPresignDenialBackoffFlagWithin({
|
|
731
|
+
endpoint: process.env.HQ_FLAGS_API_URL,
|
|
732
|
+
getToken: runtime.defaultGetAccessToken,
|
|
733
|
+
});
|
|
734
|
+
}
|
|
512
735
|
const pollIdx = argv.indexOf("--poll-remote-ms");
|
|
513
736
|
// An explicit --poll-remote-ms pins a fixed interval (menubar/CLI still set
|
|
514
737
|
// one). When it is omitted, the interval is resolved fresh each cycle from
|
|
@@ -519,7 +742,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
519
742
|
const eventPush = argv.includes("--event-push");
|
|
520
743
|
const fullReconcileIntervalMs = resolveFullReconcileMs();
|
|
521
744
|
const companiesMode = argv.includes("--companies");
|
|
522
|
-
const
|
|
745
|
+
const companyIdx = argv.indexOf("--company");
|
|
746
|
+
const companyMode = companyIdx >= 0;
|
|
747
|
+
// The tenant a single-company run owns. Carried into route selection so a
|
|
748
|
+
// watcher or receiver event under another company cannot queue a pass for it.
|
|
749
|
+
const companySlug = companyMode ? argv[companyIdx + 1] : undefined;
|
|
523
750
|
const personalMode = argv.includes("--personal");
|
|
524
751
|
// Keep this at the watcher ingress rather than importing the runner's
|
|
525
752
|
// resolver: sync-runner imports this loop, and a reverse value import would
|
|
@@ -528,6 +755,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
528
755
|
const watchRouteSelection = {
|
|
529
756
|
companies: companiesMode,
|
|
530
757
|
company: companyMode,
|
|
758
|
+
...(companySlug !== undefined ? { companySlug } : {}),
|
|
531
759
|
personal: personalMode,
|
|
532
760
|
skipPersonal: parsed.skipPersonal === true ||
|
|
533
761
|
skipPersonalEnv === "1" ||
|
|
@@ -672,7 +900,27 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
672
900
|
resolveNoOperationLockUsers = resolve;
|
|
673
901
|
});
|
|
674
902
|
};
|
|
675
|
-
const
|
|
903
|
+
const passScheduler = new CooperativePassScheduler(monotonicNow);
|
|
904
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint) => {
|
|
905
|
+
const reportedAppliedUpserts = new Set();
|
|
906
|
+
const recordDurableAppliedUpserts = (upserts) => {
|
|
907
|
+
for (const applied of upserts) {
|
|
908
|
+
const key = `${applied.relativePath}\u0000${applied.contentHash}`;
|
|
909
|
+
if (reportedAppliedUpserts.has(key))
|
|
910
|
+
continue;
|
|
911
|
+
reportedAppliedUpserts.add(key);
|
|
912
|
+
rememberAppliedUpsert(applied.relativePath, applied.contentHash);
|
|
913
|
+
}
|
|
914
|
+
};
|
|
915
|
+
const invokePass = () => {
|
|
916
|
+
if (!runPassSupportsCooperativeCheckpoint)
|
|
917
|
+
return runPass(passArgvForRun);
|
|
918
|
+
// This callback is emitted by the production runner after journal
|
|
919
|
+
// repair/seed and before fanout. It starts ingress early, but never
|
|
920
|
+
// admits a second durable writer: queued realtime still waits for the
|
|
921
|
+
// existing scheduler's cooperative hand-off checkpoint.
|
|
922
|
+
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
|
|
923
|
+
};
|
|
676
924
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
677
925
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
678
926
|
// synchronous stretch starves the interval timer, so these boundary
|
|
@@ -684,9 +932,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
684
932
|
heapCheckpoint?.();
|
|
685
933
|
if (typeof outcome === "number")
|
|
686
934
|
return;
|
|
687
|
-
|
|
688
|
-
rememberAppliedUpsert(applied.relativePath, applied.contentHash);
|
|
689
|
-
}
|
|
935
|
+
recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
|
|
690
936
|
};
|
|
691
937
|
// Pre-admit boundary observation.
|
|
692
938
|
heapCheckpoint?.();
|
|
@@ -695,13 +941,31 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
695
941
|
// its keyed journal delta and per-object mutation are independent from the
|
|
696
942
|
// slow pass's bulk paths, and the slow pass already owns the root lock.
|
|
697
943
|
// Never bypass the lock when no slow pass is active.
|
|
944
|
+
if (
|
|
945
|
+
// A full chore is paused at a caller-declared durable checkpoint. Keep
|
|
946
|
+
// the root lock held by this process, but lend it to exactly one
|
|
947
|
+
// realtime pass; the suspended chore cannot mutate until this returns.
|
|
948
|
+
workClass === "realtime" &&
|
|
949
|
+
passScheduler.handoffActive &&
|
|
950
|
+
processOwnsOperationLock) {
|
|
951
|
+
acquireOperationLockUser();
|
|
952
|
+
try {
|
|
953
|
+
prepare?.();
|
|
954
|
+
const outcome = await invokePass();
|
|
955
|
+
recordAppliedUpserts(outcome);
|
|
956
|
+
return outcome;
|
|
957
|
+
}
|
|
958
|
+
finally {
|
|
959
|
+
releaseOperationLockUser();
|
|
960
|
+
}
|
|
961
|
+
}
|
|
698
962
|
if (lane === "fast" &&
|
|
699
963
|
processOwnsOperationLock &&
|
|
700
964
|
coordinatorWholeMutationChores === 0) {
|
|
701
965
|
acquireOperationLockUser();
|
|
702
966
|
try {
|
|
703
967
|
prepare?.();
|
|
704
|
-
const outcome = await
|
|
968
|
+
const outcome = await invokePass();
|
|
705
969
|
recordAppliedUpserts(outcome);
|
|
706
970
|
return outcome;
|
|
707
971
|
}
|
|
@@ -721,7 +985,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
721
985
|
acquireOperationLockUser();
|
|
722
986
|
}
|
|
723
987
|
prepare?.();
|
|
724
|
-
const outcome = await
|
|
988
|
+
const outcome = await invokePass();
|
|
725
989
|
recordAppliedUpserts(outcome);
|
|
726
990
|
return outcome;
|
|
727
991
|
}
|
|
@@ -755,7 +1019,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
755
1019
|
}
|
|
756
1020
|
try {
|
|
757
1021
|
prepare?.();
|
|
758
|
-
const outcome = await
|
|
1022
|
+
const outcome = await invokePass();
|
|
759
1023
|
recordAppliedUpserts(outcome);
|
|
760
1024
|
return outcome;
|
|
761
1025
|
}
|
|
@@ -778,7 +1042,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
778
1042
|
throw err;
|
|
779
1043
|
}
|
|
780
1044
|
};
|
|
781
|
-
const runTaskWithLock = async (task) => {
|
|
1045
|
+
const runTaskWithLock = async (task, workClass = "chore") => {
|
|
1046
|
+
if (workClass === "realtime" &&
|
|
1047
|
+
passScheduler.handoffActive &&
|
|
1048
|
+
processOwnsOperationLock) {
|
|
1049
|
+
acquireOperationLockUser();
|
|
1050
|
+
try {
|
|
1051
|
+
return await task();
|
|
1052
|
+
}
|
|
1053
|
+
finally {
|
|
1054
|
+
releaseOperationLockUser();
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
782
1057
|
try {
|
|
783
1058
|
const handle = acquireOperationLock(hqRoot, "sync", {
|
|
784
1059
|
timeoutSec: 0,
|
|
@@ -872,102 +1147,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
872
1147
|
heapCheckpoint?.();
|
|
873
1148
|
}
|
|
874
1149
|
};
|
|
875
|
-
let activeSlowPass = null;
|
|
876
|
-
let activeFastPass = null;
|
|
877
|
-
const pendingSlowPasses = [];
|
|
878
|
-
const pendingFastPasses = [];
|
|
879
1150
|
const resolveStoppedQueue = () => {
|
|
880
|
-
|
|
881
|
-
while (queue.length > 0)
|
|
882
|
-
queue.shift()?.resolve(0);
|
|
883
|
-
}
|
|
884
|
-
};
|
|
885
|
-
const drainQueuedPasses = () => {
|
|
886
|
-
if (stopped) {
|
|
887
|
-
resolveStoppedQueue();
|
|
888
|
-
return;
|
|
889
|
-
}
|
|
890
|
-
// A queued slow pass is the bulk-drain progress boundary. Admit it before
|
|
891
|
-
// refilling the realtime queue; otherwise a steady stream of fast work
|
|
892
|
-
// perpetually leapfrogs a receiver's bulk batch and its SQS poll loop
|
|
893
|
-
// cannot issue another receive.
|
|
894
|
-
if (activeSlowPass === null &&
|
|
895
|
-
activeFastPass === null &&
|
|
896
|
-
pendingSlowPasses.length > 0) {
|
|
897
|
-
const slow = pendingSlowPasses.shift();
|
|
898
|
-
if (slow) {
|
|
899
|
-
const current = startGuardedTask(slow.task, "slow");
|
|
900
|
-
void current.then(slow.resolve, slow.reject);
|
|
901
|
-
}
|
|
902
|
-
}
|
|
903
|
-
if (activeFastPass === null) {
|
|
904
|
-
const fast = pendingFastPasses.shift();
|
|
905
|
-
if (fast) {
|
|
906
|
-
const current = startGuardedTask(fast.task, "fast");
|
|
907
|
-
void current.then(fast.resolve, fast.reject);
|
|
908
|
-
}
|
|
909
|
-
}
|
|
910
|
-
// A fast pass that acquired the operation lock normally (rather than
|
|
911
|
-
// bypassing an in-process slow holder) still owns the cross-process
|
|
912
|
-
// boundary. Slow work must queue behind it; only fast work is allowed to
|
|
913
|
-
// overlap an already-lock-owning slow pass.
|
|
914
|
-
if (activeSlowPass === null && activeFastPass === null) {
|
|
915
|
-
const slow = pendingSlowPasses.shift();
|
|
916
|
-
if (slow) {
|
|
917
|
-
const current = startGuardedTask(slow.task, "slow");
|
|
918
|
-
void current.then(slow.resolve, slow.reject);
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
|
-
};
|
|
922
|
-
const startGuardedTask = (task, lane) => {
|
|
923
|
-
// Claim the lane before invoking `task`: a task can synchronously activate
|
|
924
|
-
// a watcher/scheduler callback before returning its promise. Without this
|
|
925
|
-
// placeholder that re-entrant callback sees an idle lane and starts a
|
|
926
|
-
// second lock contender.
|
|
927
|
-
const placeholder = Promise.resolve(0);
|
|
928
|
-
if (lane === "fast")
|
|
929
|
-
activeFastPass = placeholder;
|
|
930
|
-
else
|
|
931
|
-
activeSlowPass = placeholder;
|
|
932
|
-
let current;
|
|
933
|
-
try {
|
|
934
|
-
current = stopped ? placeholder : task();
|
|
935
|
-
}
|
|
936
|
-
catch (err) {
|
|
937
|
-
current = Promise.reject(err);
|
|
938
|
-
}
|
|
939
|
-
if (lane === "fast")
|
|
940
|
-
activeFastPass = current;
|
|
941
|
-
else
|
|
942
|
-
activeSlowPass = current;
|
|
943
|
-
void current
|
|
944
|
-
.finally(() => {
|
|
945
|
-
if (lane === "fast" && activeFastPass === current) {
|
|
946
|
-
activeFastPass = null;
|
|
947
|
-
drainQueuedPasses();
|
|
948
|
-
}
|
|
949
|
-
else if (lane === "slow" && activeSlowPass === current) {
|
|
950
|
-
activeSlowPass = null;
|
|
951
|
-
drainQueuedPasses();
|
|
952
|
-
}
|
|
953
|
-
})
|
|
954
|
-
.catch(() => undefined);
|
|
955
|
-
return current;
|
|
956
|
-
};
|
|
957
|
-
const runGuardedTask = (task, lane = "slow") => {
|
|
958
|
-
const active = lane === "fast"
|
|
959
|
-
? activeFastPass
|
|
960
|
-
: activeSlowPass ?? activeFastPass;
|
|
961
|
-
const queued = lane === "fast" ? pendingFastPasses : pendingSlowPasses;
|
|
962
|
-
if (active === null && queued.length === 0) {
|
|
963
|
-
return startGuardedTask(task, lane);
|
|
964
|
-
}
|
|
965
|
-
return new Promise((resolve, reject) => {
|
|
966
|
-
queued.push({ task, lane, resolve, reject });
|
|
967
|
-
drainQueuedPasses();
|
|
968
|
-
});
|
|
1151
|
+
passScheduler.stop();
|
|
969
1152
|
};
|
|
970
|
-
const
|
|
1153
|
+
const runGuardedTask = (task, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) =>
|
|
1154
|
+
// Lanes still choose the pass execution profile, but no longer decide
|
|
1155
|
+
// chore ownership. Realtime normally waits for a chore hand-off; the
|
|
1156
|
+
// watcher fast path explicitly opts into its established scoped exception.
|
|
1157
|
+
passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake);
|
|
1158
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) => runGuardedTask((context) => runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined), lane, workClass, allowFastRealtimeOvertake);
|
|
971
1159
|
/*
|
|
972
1160
|
* Area storage and the parallel coordinator default to AUTO. AUTO turns
|
|
973
1161
|
* them on after durable personal-vault cutover; explicit off is the kill
|
|
@@ -1159,6 +1347,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1159
1347
|
});
|
|
1160
1348
|
},
|
|
1161
1349
|
onCutOver: () => {
|
|
1350
|
+
releaseScopedJournalStore(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
1162
1351
|
parallelCoordinatorEnabled = refreshCoordinatorEnabled();
|
|
1163
1352
|
if (parallelCoordinatorEnabled && coordinator === null) {
|
|
1164
1353
|
coordinator = createCoordinator();
|
|
@@ -1167,6 +1356,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1167
1356
|
})
|
|
1168
1357
|
: { onTick() { } };
|
|
1169
1358
|
let watcher = null;
|
|
1359
|
+
let rootCoverageStale = false;
|
|
1170
1360
|
let eventPushSurfacesActivated = false;
|
|
1171
1361
|
let watcherUnavailableWarned = false;
|
|
1172
1362
|
let watcherRetryAt = null;
|
|
@@ -1240,11 +1430,36 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1240
1430
|
`watched-paths=${watchedPaths ?? "unknown"}, ` +
|
|
1241
1431
|
`dropped-untracked-deletes=${watcher.untrackedDeletesDropped?.() ?? "unknown"}, ` +
|
|
1242
1432
|
`drained-untracked-deletes=${drainedUntrackedDeletes})\n`);
|
|
1433
|
+
const probe = watcher.probeCoverage?.();
|
|
1434
|
+
if (probe && !probe.ok) {
|
|
1435
|
+
process.stderr.write("hq-sync-runner: WARN event-push watcher coverage stale; " +
|
|
1436
|
+
`reason=${probe.reason} stale-watches=${probe.staleWatches} ` +
|
|
1437
|
+
`watched-paths=${watchedPaths ?? "unknown"}; restoring from vault, not pushing deletes\n`);
|
|
1438
|
+
try {
|
|
1439
|
+
recoverHqRootLocalState(hqRoot);
|
|
1440
|
+
}
|
|
1441
|
+
catch (err) {
|
|
1442
|
+
process.stderr.write(`hq-sync-runner: WARN local HQ recovery failed: ${describeError(err)}\n`);
|
|
1443
|
+
}
|
|
1444
|
+
try {
|
|
1445
|
+
watcher.rebuildCoverage?.();
|
|
1446
|
+
}
|
|
1447
|
+
catch (err) {
|
|
1448
|
+
process.stderr.write(`hq-sync-runner: WARN watcher coverage rebuild failed: ${describeError(err)}\n`);
|
|
1449
|
+
}
|
|
1450
|
+
rootCoverageStale = true;
|
|
1451
|
+
interruptWait();
|
|
1452
|
+
}
|
|
1243
1453
|
};
|
|
1244
1454
|
/** Vanished never-journaled file deletes removed from batches at drain time. */
|
|
1245
1455
|
let drainedUntrackedDeletes = 0;
|
|
1246
1456
|
const pendingWatcherPaths = new Map();
|
|
1247
1457
|
const pendingWatcherChanges = new Map();
|
|
1458
|
+
// A warm-up scan observes filesystem state over time. Keep an ingress
|
|
1459
|
+
// sequence so its older observations cannot overwrite a live watcher event
|
|
1460
|
+
// that arrived while the scan was yielding.
|
|
1461
|
+
const pendingWatcherChangeSequences = new Map();
|
|
1462
|
+
let nextPendingWatcherChangeSequence = 0;
|
|
1248
1463
|
let pendingWatcherOriginalBatch = null;
|
|
1249
1464
|
// Every coalesced watcher batch gets one durable in-process generation.
|
|
1250
1465
|
// A wake can cause the scheduler to revisit a drained batch, but an event
|
|
@@ -1268,11 +1483,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1268
1483
|
const paths = new Map();
|
|
1269
1484
|
const changes = new Map();
|
|
1270
1485
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
1271
|
-
const
|
|
1486
|
+
const change = batch.changes?.get(absolutePath);
|
|
1487
|
+
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
1272
1488
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
1273
1489
|
continue;
|
|
1274
1490
|
paths.set(absolutePath, relativePath);
|
|
1275
|
-
const change = batch.changes?.get(absolutePath);
|
|
1276
1491
|
if (change)
|
|
1277
1492
|
changes.set(absolutePath, change);
|
|
1278
1493
|
}
|
|
@@ -1280,7 +1495,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1280
1495
|
return null;
|
|
1281
1496
|
return { ...batch, paths, changes };
|
|
1282
1497
|
};
|
|
1283
|
-
const addPendingWatcherChange = (changedRelPath, batch) => {
|
|
1498
|
+
const addPendingWatcherChange = (changedRelPath, batch, options) => {
|
|
1284
1499
|
if (batch) {
|
|
1285
1500
|
const selectedBatch = selectWatchBatch(batch);
|
|
1286
1501
|
if (!selectedBatch)
|
|
@@ -1293,20 +1508,30 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1293
1508
|
return false;
|
|
1294
1509
|
}
|
|
1295
1510
|
}
|
|
1296
|
-
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1297
1511
|
if (batch) {
|
|
1512
|
+
const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
|
|
1513
|
+
const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
|
|
1514
|
+
return previousSequence === undefined ||
|
|
1515
|
+
options?.preserveChangesAfter === undefined ||
|
|
1516
|
+
previousSequence <= options.preserveChangesAfter;
|
|
1517
|
+
});
|
|
1518
|
+
if (entries.length === 0)
|
|
1519
|
+
return false;
|
|
1520
|
+
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1298
1521
|
pendingWatcherOriginalBatch =
|
|
1299
1522
|
pendingWatcherPaths.size === 0 &&
|
|
1300
1523
|
!pendingWatcherBareChange &&
|
|
1301
1524
|
!pendingWatcherOverflowed &&
|
|
1302
|
-
!batch.overflowed
|
|
1525
|
+
!batch.overflowed &&
|
|
1526
|
+
entries.length === batch.paths.size
|
|
1303
1527
|
? batch
|
|
1304
1528
|
: null;
|
|
1305
|
-
for (const [absolutePath, relativePath] of
|
|
1529
|
+
for (const [absolutePath, relativePath] of entries) {
|
|
1306
1530
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
1307
1531
|
const change = batch.changes?.get(absolutePath);
|
|
1308
1532
|
if (change)
|
|
1309
1533
|
pendingWatcherChanges.set(absolutePath, change);
|
|
1534
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1310
1535
|
}
|
|
1311
1536
|
if (batch.overflowed) {
|
|
1312
1537
|
pendingWatcherOverflowed = true;
|
|
@@ -1323,6 +1548,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1323
1548
|
}
|
|
1324
1549
|
return true;
|
|
1325
1550
|
}
|
|
1551
|
+
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1326
1552
|
if (changedRelPath) {
|
|
1327
1553
|
pendingWatcherOriginalBatch = null;
|
|
1328
1554
|
// Codec boundary: `changedRelPath` is a canonical vault key (the
|
|
@@ -1331,6 +1557,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1331
1557
|
// point at the impossible colon path on Windows.
|
|
1332
1558
|
const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
|
|
1333
1559
|
pendingWatcherPaths.set(absolutePath, changedRelPath);
|
|
1560
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1334
1561
|
// A bare path signal carries no delete intent; drop any change record
|
|
1335
1562
|
// a prior batch left for this path so the stale intent cannot leak
|
|
1336
1563
|
// into the coalesced batch (#227).
|
|
@@ -1372,6 +1599,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1372
1599
|
const generation = pendingWatcherBatchGeneration;
|
|
1373
1600
|
pendingWatcherPaths.clear();
|
|
1374
1601
|
pendingWatcherChanges.clear();
|
|
1602
|
+
pendingWatcherChangeSequences.clear();
|
|
1375
1603
|
pendingWatcherOriginalBatch = null;
|
|
1376
1604
|
pendingWatcherBareChange = false;
|
|
1377
1605
|
pendingWatcherWarmupCatchup = false;
|
|
@@ -1390,6 +1618,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1390
1618
|
pendingWatcherOriginalBatch = null;
|
|
1391
1619
|
for (const [absolutePath, relativePath] of paths.entries()) {
|
|
1392
1620
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
1621
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1393
1622
|
}
|
|
1394
1623
|
};
|
|
1395
1624
|
/**
|
|
@@ -1470,8 +1699,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1470
1699
|
}
|
|
1471
1700
|
return null;
|
|
1472
1701
|
};
|
|
1473
|
-
const journalCoordinates = (relativePath) => {
|
|
1474
|
-
const route = routeChangeToTarget(relativePath);
|
|
1702
|
+
const journalCoordinates = (relativePath, kind) => {
|
|
1703
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1475
1704
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
1476
1705
|
return null;
|
|
1477
1706
|
if (route.kind === "personal") {
|
|
@@ -1506,7 +1735,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1506
1735
|
declaredSize >= 0) {
|
|
1507
1736
|
return declaredSize;
|
|
1508
1737
|
}
|
|
1509
|
-
const size = journalRows(
|
|
1738
|
+
const size = journalRows(readJournalScoped(coordinates.slug, [coordinates.key]))
|
|
1739
|
+
.get(coordinates.key)?.size;
|
|
1510
1740
|
return typeof size === "number" && Number.isSafeInteger(size) && size >= 0 ? size : null;
|
|
1511
1741
|
};
|
|
1512
1742
|
/**
|
|
@@ -1553,8 +1783,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1553
1783
|
return session.get(coordinates.key) === undefined;
|
|
1554
1784
|
};
|
|
1555
1785
|
const captureLocalDeleteSnapshots = (relativePath, kind) => {
|
|
1556
|
-
const coordinates = journalCoordinates(relativePath);
|
|
1557
|
-
const route = routeChangeToTarget(relativePath);
|
|
1786
|
+
const coordinates = journalCoordinates(relativePath, kind);
|
|
1787
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1558
1788
|
if (!coordinates || !route)
|
|
1559
1789
|
return [];
|
|
1560
1790
|
// Desktop workspace Off toggles are authoritative before journal I/O.
|
|
@@ -1675,8 +1905,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1675
1905
|
* it scopes a push or pull for the affected batch.
|
|
1676
1906
|
*/
|
|
1677
1907
|
const captureQueuedLocalDeleteSnapshots = async (relativePath, kind, isCurrent) => {
|
|
1678
|
-
const coordinates = journalCoordinates(relativePath);
|
|
1679
|
-
const route = routeChangeToTarget(relativePath);
|
|
1908
|
+
const coordinates = journalCoordinates(relativePath, kind);
|
|
1909
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1680
1910
|
if (!coordinates || !route)
|
|
1681
1911
|
return [];
|
|
1682
1912
|
if (route.kind === "company" &&
|
|
@@ -1770,8 +2000,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1770
2000
|
}
|
|
1771
2001
|
return snapshots;
|
|
1772
2002
|
};
|
|
1773
|
-
const warmupJournalPaths = ()
|
|
1774
|
-
const paths = [];
|
|
2003
|
+
const warmupJournalPaths = function* () {
|
|
1775
2004
|
const journals = listJournals();
|
|
1776
2005
|
const vaultJournalPresent = journals.some(({ slug }) => slug === PERSONAL_VAULT_JOURNAL_SLUG);
|
|
1777
2006
|
for (const { slug, journal } of journals) {
|
|
@@ -1804,10 +2033,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1804
2033
|
// ~12.8k of them per runner start after a transcript prune.
|
|
1805
2034
|
if (route.kind === "personal" && isPersonalVaultDeletionExcluded(key))
|
|
1806
2035
|
continue;
|
|
1807
|
-
|
|
2036
|
+
yield route.kind === "personal" ? key : `companies/${route.slug}/${key}`;
|
|
1808
2037
|
}
|
|
1809
2038
|
}
|
|
1810
|
-
return paths;
|
|
1811
2039
|
};
|
|
1812
2040
|
const publishedContentHashLookup = (relativePath) => {
|
|
1813
2041
|
const coordinates = journalCoordinates(relativePath);
|
|
@@ -1820,13 +2048,34 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1820
2048
|
return publishedContentHashFromJournalEntryBeforeSession(entry, serviceStartedAt);
|
|
1821
2049
|
};
|
|
1822
2050
|
const prepareWatcherChanges = (changes) => {
|
|
2051
|
+
const keysByJournal = new Map();
|
|
2052
|
+
const rememberKey = (slug, key) => {
|
|
2053
|
+
const keys = keysByJournal.get(slug) ?? new Set();
|
|
2054
|
+
keys.add(key);
|
|
2055
|
+
keysByJournal.set(slug, keys);
|
|
2056
|
+
};
|
|
2057
|
+
for (const [absolutePath, change] of changes.entries()) {
|
|
2058
|
+
if (change.kind === "unlink" || change.kind === "unlinkDir") {
|
|
2059
|
+
for (const snapshot of change.deleteSnapshots ?? []) {
|
|
2060
|
+
rememberKey(snapshot.journalSlug, snapshot.journalPath);
|
|
2061
|
+
}
|
|
2062
|
+
continue;
|
|
2063
|
+
}
|
|
2064
|
+
const relativePath = path
|
|
2065
|
+
.relative(hqRoot, absolutePath)
|
|
2066
|
+
.split(path.sep)
|
|
2067
|
+
.join("/");
|
|
2068
|
+
const coordinates = journalCoordinates(relativePath);
|
|
2069
|
+
if (coordinates)
|
|
2070
|
+
rememberKey(coordinates.slug, coordinates.key);
|
|
2071
|
+
}
|
|
1823
2072
|
const journals = new Map();
|
|
1824
2073
|
const dirty = new Set();
|
|
1825
2074
|
const journalFor = (slug) => {
|
|
1826
2075
|
const loaded = journals.get(slug);
|
|
1827
2076
|
if (loaded)
|
|
1828
2077
|
return loaded;
|
|
1829
|
-
const journal =
|
|
2078
|
+
const journal = readJournalScoped(slug, [...(keysByJournal.get(slug) ?? [])]);
|
|
1830
2079
|
journals.set(slug, journal);
|
|
1831
2080
|
return journal;
|
|
1832
2081
|
};
|
|
@@ -1884,7 +2133,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1884
2133
|
const groupBatchByRoute = (batch) => {
|
|
1885
2134
|
const grouped = new Map();
|
|
1886
2135
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
1887
|
-
const
|
|
2136
|
+
const change = batch.changes?.get(absolutePath);
|
|
2137
|
+
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
1888
2138
|
if (!route)
|
|
1889
2139
|
continue;
|
|
1890
2140
|
const key = routeKey(route);
|
|
@@ -1897,7 +2147,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1897
2147
|
deleteScopeRoots: [],
|
|
1898
2148
|
};
|
|
1899
2149
|
existing.paths.set(absolutePath, relativePath);
|
|
1900
|
-
const change = batch.changes?.get(absolutePath);
|
|
1901
2150
|
if (change)
|
|
1902
2151
|
existing.changes.set(absolutePath, change);
|
|
1903
2152
|
existing.relPaths = [...new Set([...existing.paths.values()])];
|
|
@@ -1981,7 +2230,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1981
2230
|
}
|
|
1982
2231
|
return false;
|
|
1983
2232
|
};
|
|
1984
|
-
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false) => {
|
|
2233
|
+
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
|
|
1985
2234
|
if (batch)
|
|
1986
2235
|
await resolveDeleteSnapshotCaptures(batch);
|
|
1987
2236
|
const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
|
|
@@ -2010,11 +2259,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2010
2259
|
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
2011
2260
|
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
2012
2261
|
? () => prepareWatcherChanges(admittedChanges)
|
|
2013
|
-
: undefined, lane);
|
|
2262
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake);
|
|
2014
2263
|
})
|
|
2015
2264
|
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
2016
2265
|
? () => prepareWatcherChanges(group.changes)
|
|
2017
|
-
: undefined, lane), accepted: true, coordinatorRefusedPaths: [] };
|
|
2266
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
|
|
2018
2267
|
const result = execution.result;
|
|
2019
2268
|
if (execution.accepted && passExitCode(result) === 0) {
|
|
2020
2269
|
const refusedPaths = new Set(typeof result === "number"
|
|
@@ -2084,8 +2333,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2084
2333
|
? "fast"
|
|
2085
2334
|
: "slow";
|
|
2086
2335
|
const execution = parallelCoordinatorEnabled
|
|
2087
|
-
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))
|
|
2088
|
-
: { result: await runGuarded(pullArgv, undefined, lane), accepted: true };
|
|
2336
|
+
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))
|
|
2337
|
+
: { result: await runGuarded(pullArgv, undefined, lane, "realtime"), accepted: true };
|
|
2089
2338
|
const exit = execution.accepted ? passExitCode(execution.result) : 1;
|
|
2090
2339
|
if (exit !== 0 && worstExit === 0)
|
|
2091
2340
|
worstExit = exit;
|
|
@@ -2290,13 +2539,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2290
2539
|
return;
|
|
2291
2540
|
}
|
|
2292
2541
|
if (lanes?.fast &&
|
|
2293
|
-
|
|
2542
|
+
(passScheduler.hasActiveChore || passScheduler.hasActiveSlowRealtime)) {
|
|
2294
2543
|
const generation = ++nextWatcherBatchGeneration;
|
|
2295
2544
|
// Fast dispatch bypasses the pending-batch ingress, so preserve both
|
|
2296
2545
|
// of its contracts here: only selected routes may run, and V2 sees
|
|
2297
2546
|
// the same dirty signal as the compatibility drain.
|
|
2298
2547
|
realtimeState.scheduler?.signal("watcher");
|
|
2299
|
-
void runScopedDrain(lanes.fast, generation, true).then((exit) => {
|
|
2548
|
+
void runScopedDrain(lanes.fast, generation, true, true).then((exit) => {
|
|
2300
2549
|
if (exit === 0)
|
|
2301
2550
|
return;
|
|
2302
2551
|
restorePendingWatcherPaths(lanes.fast.paths, generation);
|
|
@@ -2320,6 +2569,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2320
2569
|
interruptWait();
|
|
2321
2570
|
}
|
|
2322
2571
|
});
|
|
2572
|
+
try {
|
|
2573
|
+
recoverHqRootLocalState(hqRoot);
|
|
2574
|
+
}
|
|
2575
|
+
catch (err) {
|
|
2576
|
+
process.stderr.write(`hq-sync-runner: WARN local HQ recovery failed: ${describeError(err)}\n`);
|
|
2577
|
+
}
|
|
2323
2578
|
watcher.start();
|
|
2324
2579
|
const reportWatcherReady = () => {
|
|
2325
2580
|
if (stopped || watcher === null)
|
|
@@ -2345,29 +2600,36 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2345
2600
|
watcherRediscoveryRetries = 0;
|
|
2346
2601
|
process.stderr.write(`hq-sync-runner: event-push watcher active (watched-paths=${watchedPaths ?? "unknown"})\n`);
|
|
2347
2602
|
const startedAt = Date.now();
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2603
|
+
const liveChangeSequenceAtWarmupStart = nextPendingWatcherChangeSequence;
|
|
2604
|
+
void (async () => {
|
|
2605
|
+
try {
|
|
2606
|
+
const catchup = watcher?.collectWarmupCatchupAsync
|
|
2607
|
+
? await watcher.collectWarmupCatchupAsync(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths())
|
|
2608
|
+
: watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths());
|
|
2609
|
+
if (!catchup || catchup.cancelled || stopped || watcher === null)
|
|
2610
|
+
return;
|
|
2611
|
+
const queued = addPendingWatcherChange(undefined, catchup.batch, {
|
|
2612
|
+
preserveChangesAfter: liveChangeSequenceAtWarmupStart,
|
|
2613
|
+
});
|
|
2614
|
+
pendingWatcherWarmupCatchup ||= queued;
|
|
2615
|
+
process.stderr.write(JSON.stringify({
|
|
2616
|
+
type: "watcher-warmup-catchup",
|
|
2617
|
+
reason: "watch-set-ready",
|
|
2618
|
+
pathsFound: catchup.batch.paths.size,
|
|
2619
|
+
scannedPaths: catchup.scannedPaths,
|
|
2620
|
+
elapsedMs: Date.now() - startedAt,
|
|
2621
|
+
queued,
|
|
2622
|
+
}) + "\n");
|
|
2623
|
+
if (queued)
|
|
2624
|
+
interruptWait();
|
|
2625
|
+
}
|
|
2626
|
+
catch (err) {
|
|
2627
|
+
process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
|
|
2628
|
+
// A scan failure must not silently extend the warm-up blind spot.
|
|
2629
|
+
pendingWatcherWarmupCatchupFailed = true;
|
|
2363
2630
|
interruptWait();
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
|
|
2367
|
-
// A scan failure must not silently extend the warm-up blind spot.
|
|
2368
|
-
pendingWatcherWarmupCatchupFailed = true;
|
|
2369
|
-
interruptWait();
|
|
2370
|
-
}
|
|
2631
|
+
}
|
|
2632
|
+
})();
|
|
2371
2633
|
};
|
|
2372
2634
|
if (watcher.onReady)
|
|
2373
2635
|
watcher.onReady(reportWatcherReady);
|
|
@@ -2415,7 +2677,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2415
2677
|
// Submit every route in this lane before awaiting any result. In
|
|
2416
2678
|
// particular, a later route's fast receipt must never wait for an
|
|
2417
2679
|
// earlier route's slow pull to finish.
|
|
2418
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))));
|
|
2680
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))));
|
|
2419
2681
|
for (const execution of completed) {
|
|
2420
2682
|
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
2421
2683
|
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
@@ -2469,7 +2731,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2469
2731
|
}
|
|
2470
2732
|
for (const { route, paths } of pathsByRoute.values()) {
|
|
2471
2733
|
const targetedArgv = buildTargetedPullArgv(route, passArgv, paths);
|
|
2472
|
-
const result = await runGuarded(targetedArgv, undefined, lane);
|
|
2734
|
+
const result = await runGuarded(targetedArgv, undefined, lane, "realtime");
|
|
2473
2735
|
if (passExitCode(result) !== 0) {
|
|
2474
2736
|
throw new Error(`targeted pull failed with exit code ${passExitCode(result)}`);
|
|
2475
2737
|
}
|
|
@@ -2517,14 +2779,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2517
2779
|
const execution = await runRealtimePass("wake", "pull", [], "slow", async () => runGuardedTask(() => runTaskWithLock(async () => {
|
|
2518
2780
|
await drain();
|
|
2519
2781
|
return 0;
|
|
2520
|
-
})));
|
|
2782
|
+
}, "realtime"), "slow", "realtime"));
|
|
2521
2783
|
outcome = execution.accepted ? execution.result : 1;
|
|
2522
2784
|
}
|
|
2523
2785
|
else {
|
|
2524
2786
|
outcome = await runGuardedTask(() => runTaskWithLock(async () => {
|
|
2525
2787
|
await drain();
|
|
2526
2788
|
return 0;
|
|
2527
|
-
}));
|
|
2789
|
+
}, "realtime"), "slow", "realtime");
|
|
2528
2790
|
}
|
|
2529
2791
|
}
|
|
2530
2792
|
catch (err) {
|
|
@@ -2720,8 +2982,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2720
2982
|
targetedJournalSessions: () => targetedJournalSessionCensusSizes().targetedJournalSessions,
|
|
2721
2983
|
targetedJournalRows: () => targetedJournalSessionCensusSizes().targetedJournalRows,
|
|
2722
2984
|
cachedAreaJournalMigrations: () => journalCacheCensusSizes().cachedAreaJournalMigrations,
|
|
2723
|
-
pendingSlowPasses: () =>
|
|
2724
|
-
pendingFastPasses: () =>
|
|
2985
|
+
pendingSlowPasses: () => passScheduler.pendingLaneCount("slow"),
|
|
2986
|
+
pendingFastPasses: () => passScheduler.pendingLaneCount("fast"),
|
|
2725
2987
|
},
|
|
2726
2988
|
eventSyncSizes: () => eventSync?.censusSizes?.(),
|
|
2727
2989
|
watcherSizes: () => watcher?.censusSizes?.(),
|
|
@@ -2909,6 +3171,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2909
3171
|
const watcherWarmupCatchup = pending.warmupCatchup;
|
|
2910
3172
|
const watcherWarmupCatchupFailed = pending.warmupCatchupFailed;
|
|
2911
3173
|
const overflowRoutesUnknown = pending.batch?.overflowed === true && !overflowWithKnownRoutes;
|
|
3174
|
+
const coverageStale = rootCoverageStale;
|
|
3175
|
+
rootCoverageStale = false;
|
|
2912
3176
|
const scheduledInterval = fullReconcileIntervalMs === undefined
|
|
2913
3177
|
? pollTick % FULL_RECONCILE_EVERY_TICKS === 0
|
|
2914
3178
|
: lastFullReconcileAt !== null &&
|
|
@@ -2922,6 +3186,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2922
3186
|
"watcher-warmup-catch-up": watcherWarmupCatchup,
|
|
2923
3187
|
"watcher-warmup-catch-up-failed": watcherWarmupCatchupFailed,
|
|
2924
3188
|
"scheduled-interval": scheduledInterval,
|
|
3189
|
+
"root-coverage-stale": coverageStale,
|
|
2925
3190
|
};
|
|
2926
3191
|
const reasons = Object.entries(triggers)
|
|
2927
3192
|
.filter(([, matched]) => matched)
|
|
@@ -2931,7 +3196,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2931
3196
|
bareWake ||
|
|
2932
3197
|
overflowRoutesUnknown ||
|
|
2933
3198
|
watcherWarmupCatchupFailed ||
|
|
2934
|
-
scheduledInterval
|
|
3199
|
+
scheduledInterval ||
|
|
3200
|
+
coverageStale;
|
|
2935
3201
|
// Size is not unknown-scope. A fully-enumerated over-limit batch still
|
|
2936
3202
|
// has every path listed, so drain it in bounded chunks and keep the
|
|
2937
3203
|
// remainder queued. Skip this when a genuine full-reconcile trigger
|
|
@@ -3071,11 +3337,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3071
3337
|
quarantinedDeleteRoots.delete(root);
|
|
3072
3338
|
}
|
|
3073
3339
|
// Delete snapshot capture needs a fresh keyed view while an episode is
|
|
3074
|
-
// being assembled, but completed watch cycles do not.
|
|
3075
|
-
//
|
|
3076
|
-
//
|
|
3340
|
+
// being assembled, but completed watch cycles do not. Area-authoritative
|
|
3341
|
+
// journals are different: their bridge owns the packed aggregate that a
|
|
3342
|
+
// later watcher callback reuses. Dropping that bridge at every poll
|
|
3343
|
+
// makes the next scoped callback synchronously re-pack the whole vault.
|
|
3344
|
+
// Keep it for this runner's lifetime; layout/phase changes still close
|
|
3345
|
+
// it in openAreaMigrationIfAuthoritative(), and final shutdown releases
|
|
3346
|
+
// every bridge below.
|
|
3077
3347
|
releaseWatcherJournalStores();
|
|
3078
|
-
releaseCachedAreaJournalMigrations();
|
|
3079
3348
|
const exitCode = passExitCode(result);
|
|
3080
3349
|
if (exitCode === runtime.authRequiredPassExit) {
|
|
3081
3350
|
// The pass already emitted auth-error. Hand the sentinel to the
|
|
@@ -3110,8 +3379,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3110
3379
|
: "") +
|
|
3111
3380
|
"; staying alive, retrying next poll\n");
|
|
3112
3381
|
}
|
|
3382
|
+
else if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
3383
|
+
// HQ-SYNC-COMPANY-ROUTE-UID: routed paths are derived from local
|
|
3384
|
+
// company directory names, which may not be memberships. The runner
|
|
3385
|
+
// has already emitted the resolution error; terminating here would
|
|
3386
|
+
// restart the outpost watcher every 60 seconds for each such route.
|
|
3387
|
+
// Keep the watcher alive while preserving the one-shot non-zero code.
|
|
3388
|
+
process.stderr.write("hq-sync-runner: watch pass skipped — unresolved company target; retrying next poll\n");
|
|
3389
|
+
}
|
|
3113
3390
|
else if (exitCode !== 0) {
|
|
3114
|
-
|
|
3391
|
+
// A scoped company-resolution or per-pass failure is isolated to that
|
|
3392
|
+
// pass. The error was emitted by the runner; terminating the daemon
|
|
3393
|
+
// recreates the startup blind window and strands newer watcher work.
|
|
3394
|
+
process.stderr.write(`hq-sync-runner: watch pass failed (exit=${exitCode}); staying alive, retrying next poll\n`);
|
|
3115
3395
|
}
|
|
3116
3396
|
const completedPostRepairPass = !enqueuedFullChore && (typeof result === "number"
|
|
3117
3397
|
? exitCode === 0
|
|
@@ -3215,6 +3495,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3215
3495
|
releaseWatcherJournalStores();
|
|
3216
3496
|
releaseCachedAreaJournalMigrations();
|
|
3217
3497
|
await shutdown();
|
|
3498
|
+
// Coordinator shutdown waits for any active pass before the retained
|
|
3499
|
+
// projections are dropped. A shutdown must never evict a store that a
|
|
3500
|
+
// final scoped writer is still refreshing or committing through.
|
|
3501
|
+
disposeScopedJournalStores();
|
|
3218
3502
|
detachSignal?.();
|
|
3219
3503
|
}
|
|
3220
3504
|
}
|