@indigoai-us/hq-cloud 6.16.65 → 6.16.66-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +14 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +42 -8
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +62 -7
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +172 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +51 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +417 -165
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +19 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +40 -5
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +47 -0
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +59 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +48 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +1553 -192
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/push-scan-exclusions.test.js +4 -4
- package/dist/cli/push-scan-exclusions.test.js.map +1 -1
- package/dist/cli/share.d.ts +31 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +136 -6
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +295 -5
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +12 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +297 -125
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +255 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal-row-store.d.ts +40 -5
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +240 -21
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +54 -0
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +11 -6
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +324 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +51 -16
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +303 -79
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/object-io.d.ts +33 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +145 -8
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +248 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +20 -2
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/sync/area-ledger.d.ts +27 -3
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +149 -39
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +192 -5
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +29 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/presign-denial-backoff-flag.d.ts +22 -0
- package/dist/sync/presign-denial-backoff-flag.d.ts.map +1 -0
- package/dist/sync/presign-denial-backoff-flag.js +75 -0
- package/dist/sync/presign-denial-backoff-flag.js.map +1 -0
- package/dist/sync/presign-denial-backoff-flag.test.d.ts +2 -0
- package/dist/sync/presign-denial-backoff-flag.test.d.ts.map +1 -0
- package/dist/sync/presign-denial-backoff-flag.test.js +103 -0
- package/dist/sync/presign-denial-backoff-flag.test.js.map +1 -0
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +6 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +14 -2
- package/dist/sync/state-store.js.map +1 -1
- package/dist/watcher.d.ts +12 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +131 -0
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +63 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +3 -1
|
@@ -10,12 +10,12 @@ import { pickCanonicalPersonEntity } from "../vault-client.js";
|
|
|
10
10
|
import { acquireOperationLock, withOperationLock, withOperationLockSync, OperationLockedError, OPERATION_LOCKED_EXIT, } from "../operation-lock.js";
|
|
11
11
|
import { describeError } from "../lib/describe-error.js";
|
|
12
12
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
13
|
-
import { PARTIAL_SYNC_EXIT } from "../lib/exit-codes.js";
|
|
13
|
+
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
14
14
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
15
15
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
16
16
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
17
17
|
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
18
|
-
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice,
|
|
18
|
+
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, journalRows, readJournalScoped, writeJournal, } from "../journal.js";
|
|
19
19
|
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
20
20
|
import { NoopPushReceiver, } from "../sync/push-receiver.js";
|
|
21
21
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
@@ -25,12 +25,226 @@ import { AREA_AUTO_MIGRATION_SOURCE, createAreaAutoMigrationScheduler, openPerso
|
|
|
25
25
|
import { emitCloudTelemetry } from "../telemetry-events.js";
|
|
26
26
|
import { HeapCensus, startRunnerHeapSampling, heapSampleCloudProperties, heapPressureCloudProperties, heapPressureStderrLine, heapRecycleCloudProperties, heapRecycleStderrLine, performHeapHardExit, nodeHeapReaders, nodeMajorGcObserver, resolveDeclaredHeapCeilingBytes, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled, resolveHeapRecycleLiveFraction, resolveHeapRecycleSustainWindowMs, resolveHeapRecycleEmergencyFraction, resolveHeapRecycleMinUptimeMs, resolveHeapCheckpointMs, resolveHeapPinnedFraction, resolveHeapPinnedSustainMs, resolveHeapHardExitFraction, resolveHeapRecycleLatchHardExitMs, resolveHeapFootprintRecycleEnabled, resolveHeapFootprintRecycleBytes, resolveHeapFootprintHardExitBytes, HEAP_SAMPLE_INTERVAL_ENV, HEAP_PRESSURE_FRACTION_ENV, HEAP_RECYCLE_ENABLED_ENV, HEAP_RECYCLE_LIVE_FRACTION_ENV, HEAP_RECYCLE_SUSTAIN_WINDOW_MS_ENV, HEAP_RECYCLE_EMERGENCY_FRACTION_ENV, HEAP_RECYCLE_MIN_UPTIME_MS_ENV, HEAP_CHECKPOINT_MS_ENV, HEAP_PINNED_FRACTION_ENV, HEAP_PINNED_SUSTAIN_MS_ENV, HEAP_HARD_EXIT_FRACTION_ENV, HEAP_RECYCLE_LATCH_HARD_EXIT_MS_ENV, HEAP_FOOTPRINT_RECYCLE_ENABLED_ENV, HEAP_FOOTPRINT_RECYCLE_MB_ENV, HEAP_FOOTPRINT_HARD_EXIT_MB_ENV, } from "./sync-runner-heap-telemetry.js";
|
|
27
27
|
import { setRunnerEmitObserver } from "./sync-runner-events.js";
|
|
28
|
+
import { refreshPresignDenialBackoffFlagWithin } from "../sync/presign-denial-backoff-flag.js";
|
|
28
29
|
import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
|
|
29
30
|
import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr } from "../sync/realtime-telemetry.js";
|
|
30
31
|
import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
31
32
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
32
33
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
33
34
|
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
35
|
+
/** A deferred full chore must get the next admission after this long. */
|
|
36
|
+
export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
37
|
+
/**
|
|
38
|
+
* One durable-state owner at a time for a full chore. Realtime work never runs
|
|
39
|
+
* beside that chore: it explicitly awaits `checkpoint()` after a completed
|
|
40
|
+
* durable unit, then resumes once the hand-off is finished. A separately
|
|
41
|
+
* tracked, explicitly opted-in watcher fast pass may use the pre-existing
|
|
42
|
+
* scoped fast-over-slow exception, but never while a chore owns durable state.
|
|
43
|
+
*/
|
|
44
|
+
export class CooperativePassScheduler {
|
|
45
|
+
now;
|
|
46
|
+
starvationCapMs;
|
|
47
|
+
pending = [];
|
|
48
|
+
active = null;
|
|
49
|
+
// Scoped fast realtime work has the established in-process lock exception
|
|
50
|
+
// while a scoped slow realtime upload is in flight. Keep that slot separate
|
|
51
|
+
// from `active`: a chore still has exactly one durable-state owner.
|
|
52
|
+
activeFastRealtime = null;
|
|
53
|
+
stopped = false;
|
|
54
|
+
_handoffActive = false;
|
|
55
|
+
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS) {
|
|
56
|
+
this.now = now;
|
|
57
|
+
this.starvationCapMs = starvationCapMs;
|
|
58
|
+
}
|
|
59
|
+
get handoffActive() {
|
|
60
|
+
return this._handoffActive;
|
|
61
|
+
}
|
|
62
|
+
get hasActiveChore() {
|
|
63
|
+
return this.active?.workClass === "chore";
|
|
64
|
+
}
|
|
65
|
+
get hasActiveSlowRealtime() {
|
|
66
|
+
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
67
|
+
}
|
|
68
|
+
pendingLaneCount(lane) {
|
|
69
|
+
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
70
|
+
}
|
|
71
|
+
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false) {
|
|
72
|
+
if (this.stopped)
|
|
73
|
+
return Promise.resolve(0);
|
|
74
|
+
const directEntry = {
|
|
75
|
+
task,
|
|
76
|
+
workClass,
|
|
77
|
+
lane,
|
|
78
|
+
allowFastRealtimeOvertake,
|
|
79
|
+
// An immediately admitted task cannot starve, so avoid sampling the
|
|
80
|
+
// cadence clock merely to stamp it. This preserves the loop's deadline
|
|
81
|
+
// accounting and keeps the timestamp meaningful only for real queues.
|
|
82
|
+
enqueuedAt: 0,
|
|
83
|
+
resolve: () => undefined,
|
|
84
|
+
reject: () => undefined,
|
|
85
|
+
};
|
|
86
|
+
if (this.canStartFastRealtime(directEntry)) {
|
|
87
|
+
return this.startFastRealtime(directEntry);
|
|
88
|
+
}
|
|
89
|
+
if (this.active === null &&
|
|
90
|
+
this.activeFastRealtime === null &&
|
|
91
|
+
this.pending.length === 0) {
|
|
92
|
+
return this.start(directEntry);
|
|
93
|
+
}
|
|
94
|
+
return new Promise((resolve, reject) => {
|
|
95
|
+
const entry = { ...directEntry, enqueuedAt: this.now(), resolve, reject };
|
|
96
|
+
this.pending.push(entry);
|
|
97
|
+
this.drain();
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
stop() {
|
|
101
|
+
this.stopped = true;
|
|
102
|
+
while (this.pending.length > 0)
|
|
103
|
+
this.pending.shift()?.resolve(0);
|
|
104
|
+
}
|
|
105
|
+
drain() {
|
|
106
|
+
if (this.stopped || this.pending.length === 0)
|
|
107
|
+
return;
|
|
108
|
+
if (this.active !== null) {
|
|
109
|
+
const fastRealtime = this.nextFastRealtime();
|
|
110
|
+
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
111
|
+
this.pending.splice(this.pending.indexOf(fastRealtime), 1);
|
|
112
|
+
this.startFastRealtime(fastRealtime);
|
|
113
|
+
}
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
if (this.activeFastRealtime !== null)
|
|
117
|
+
return;
|
|
118
|
+
const next = this.takeNext();
|
|
119
|
+
if (next)
|
|
120
|
+
this.start(next);
|
|
121
|
+
}
|
|
122
|
+
takeNext() {
|
|
123
|
+
const now = this.now();
|
|
124
|
+
const starvedChore = this.pending.findIndex((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
125
|
+
if (starvedChore >= 0)
|
|
126
|
+
return this.pending.splice(starvedChore, 1)[0];
|
|
127
|
+
// A receiver's bulk drain is still realtime work, but its slow lane is the
|
|
128
|
+
// progress boundary for that delivery. Preserve its established turn
|
|
129
|
+
// before refilling later fast realtime arrivals; this does not let a
|
|
130
|
+
// chore overtake any realtime work.
|
|
131
|
+
const realtime = this.nextRealtimeIndex();
|
|
132
|
+
if (realtime >= 0)
|
|
133
|
+
return this.pending.splice(realtime, 1)[0];
|
|
134
|
+
return this.pending.shift();
|
|
135
|
+
}
|
|
136
|
+
nextRealtimeIndex() {
|
|
137
|
+
const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow");
|
|
138
|
+
return slowRealtime >= 0
|
|
139
|
+
? slowRealtime
|
|
140
|
+
: this.pending.findIndex((entry) => entry.workClass === "realtime");
|
|
141
|
+
}
|
|
142
|
+
nextFastRealtime() {
|
|
143
|
+
return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
|
|
144
|
+
}
|
|
145
|
+
hasStarvedChore() {
|
|
146
|
+
const now = this.now();
|
|
147
|
+
return this.pending.some((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
148
|
+
}
|
|
149
|
+
canStartFastRealtime(entry) {
|
|
150
|
+
return (!this._handoffActive &&
|
|
151
|
+
this.activeFastRealtime === null &&
|
|
152
|
+
entry.workClass === "realtime" &&
|
|
153
|
+
entry.lane === "fast" &&
|
|
154
|
+
entry.allowFastRealtimeOvertake &&
|
|
155
|
+
this.hasActiveSlowRealtime &&
|
|
156
|
+
!this.hasStarvedChore());
|
|
157
|
+
}
|
|
158
|
+
start(entry) {
|
|
159
|
+
this.active = entry;
|
|
160
|
+
let current;
|
|
161
|
+
try {
|
|
162
|
+
current = this.execute(entry);
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
current = Promise.reject(error);
|
|
166
|
+
}
|
|
167
|
+
void current.then((outcome) => {
|
|
168
|
+
entry.resolve(outcome);
|
|
169
|
+
this.complete(entry);
|
|
170
|
+
}, (error) => {
|
|
171
|
+
entry.reject(error);
|
|
172
|
+
this.complete(entry);
|
|
173
|
+
}).catch(() => undefined);
|
|
174
|
+
return current;
|
|
175
|
+
}
|
|
176
|
+
complete(entry) {
|
|
177
|
+
if (this.active === entry)
|
|
178
|
+
this.active = null;
|
|
179
|
+
this.drain();
|
|
180
|
+
}
|
|
181
|
+
startFastRealtime(entry) {
|
|
182
|
+
this.activeFastRealtime = entry;
|
|
183
|
+
let current;
|
|
184
|
+
try {
|
|
185
|
+
current = this.execute(entry);
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
current = Promise.reject(error);
|
|
189
|
+
}
|
|
190
|
+
void current.then((outcome) => {
|
|
191
|
+
entry.resolve(outcome);
|
|
192
|
+
this.completeFastRealtime(entry);
|
|
193
|
+
}, (error) => {
|
|
194
|
+
entry.reject(error);
|
|
195
|
+
this.completeFastRealtime(entry);
|
|
196
|
+
});
|
|
197
|
+
return current;
|
|
198
|
+
}
|
|
199
|
+
completeFastRealtime(entry) {
|
|
200
|
+
if (this.activeFastRealtime === entry)
|
|
201
|
+
this.activeFastRealtime = null;
|
|
202
|
+
this.drain();
|
|
203
|
+
}
|
|
204
|
+
execute(entry) {
|
|
205
|
+
return entry.task({ checkpoint: () => this.checkpoint(entry) });
|
|
206
|
+
}
|
|
207
|
+
async checkpoint(entry) {
|
|
208
|
+
if (this.stopped ||
|
|
209
|
+
entry.workClass !== "chore" ||
|
|
210
|
+
this.active !== entry ||
|
|
211
|
+
!this.pending.some((pending) => pending.workClass === "realtime")) {
|
|
212
|
+
return false;
|
|
213
|
+
}
|
|
214
|
+
// `entry` is suspended at a caller-declared durable boundary. It remains
|
|
215
|
+
// the sole full-pass continuation and will resume after this bounded burst.
|
|
216
|
+
const deferredAt = this.now();
|
|
217
|
+
this.active = null;
|
|
218
|
+
this._handoffActive = true;
|
|
219
|
+
let handedOff = false;
|
|
220
|
+
try {
|
|
221
|
+
while (!this.stopped && this.now() - deferredAt < this.starvationCapMs) {
|
|
222
|
+
const realtimeIndex = this.nextRealtimeIndex();
|
|
223
|
+
if (realtimeIndex < 0)
|
|
224
|
+
break;
|
|
225
|
+
const realtime = this.pending.splice(realtimeIndex, 1)[0];
|
|
226
|
+
handedOff = true;
|
|
227
|
+
this.active = realtime;
|
|
228
|
+
try {
|
|
229
|
+
realtime.resolve(await this.execute(realtime));
|
|
230
|
+
}
|
|
231
|
+
catch (error) {
|
|
232
|
+
realtime.reject(error);
|
|
233
|
+
}
|
|
234
|
+
finally {
|
|
235
|
+
if (this.active === realtime)
|
|
236
|
+
this.active = null;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
finally {
|
|
241
|
+
this._handoffActive = false;
|
|
242
|
+
if (!this.stopped)
|
|
243
|
+
this.active = entry;
|
|
244
|
+
}
|
|
245
|
+
return handedOff;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
34
248
|
const MAX_FULL_RECONCILE_REASONS = 7;
|
|
35
249
|
/**
|
|
36
250
|
* Area storage is AUTO by default and only applies after durable cutover.
|
|
@@ -510,6 +724,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
510
724
|
const idleHeartbeatIntervalMs = deps.idleHeartbeatIntervalMs ?? WATCH_IDLE_HEARTBEAT_INTERVAL_MS;
|
|
511
725
|
const monotonicNow = deps.monotonicNow ?? (() => performance.now());
|
|
512
726
|
const runPass = deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
|
|
727
|
+
const runPassSupportsCooperativeCheckpoint = deps.runPass === undefined || deps.runPassSupportsCooperativeCheckpoint === true;
|
|
728
|
+
// Read the switch before the first pass, which is the largest presign user.
|
|
729
|
+
if (process.env.HQ_FLAGS_API_URL && process.env.VITEST !== "true") {
|
|
730
|
+
await refreshPresignDenialBackoffFlagWithin({
|
|
731
|
+
endpoint: process.env.HQ_FLAGS_API_URL,
|
|
732
|
+
getToken: runtime.defaultGetAccessToken,
|
|
733
|
+
});
|
|
734
|
+
}
|
|
513
735
|
const pollIdx = argv.indexOf("--poll-remote-ms");
|
|
514
736
|
// An explicit --poll-remote-ms pins a fixed interval (menubar/CLI still set
|
|
515
737
|
// one). When it is omitted, the interval is resolved fresh each cycle from
|
|
@@ -520,7 +742,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
520
742
|
const eventPush = argv.includes("--event-push");
|
|
521
743
|
const fullReconcileIntervalMs = resolveFullReconcileMs();
|
|
522
744
|
const companiesMode = argv.includes("--companies");
|
|
523
|
-
const
|
|
745
|
+
const companyIdx = argv.indexOf("--company");
|
|
746
|
+
const companyMode = companyIdx >= 0;
|
|
747
|
+
// The tenant a single-company run owns. Carried into route selection so a
|
|
748
|
+
// watcher or receiver event under another company cannot queue a pass for it.
|
|
749
|
+
const companySlug = companyMode ? argv[companyIdx + 1] : undefined;
|
|
524
750
|
const personalMode = argv.includes("--personal");
|
|
525
751
|
// Keep this at the watcher ingress rather than importing the runner's
|
|
526
752
|
// resolver: sync-runner imports this loop, and a reverse value import would
|
|
@@ -529,6 +755,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
529
755
|
const watchRouteSelection = {
|
|
530
756
|
companies: companiesMode,
|
|
531
757
|
company: companyMode,
|
|
758
|
+
...(companySlug !== undefined ? { companySlug } : {}),
|
|
532
759
|
personal: personalMode,
|
|
533
760
|
skipPersonal: parsed.skipPersonal === true ||
|
|
534
761
|
skipPersonalEnv === "1" ||
|
|
@@ -673,7 +900,27 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
673
900
|
resolveNoOperationLockUsers = resolve;
|
|
674
901
|
});
|
|
675
902
|
};
|
|
676
|
-
const
|
|
903
|
+
const passScheduler = new CooperativePassScheduler(monotonicNow);
|
|
904
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint) => {
|
|
905
|
+
const reportedAppliedUpserts = new Set();
|
|
906
|
+
const recordDurableAppliedUpserts = (upserts) => {
|
|
907
|
+
for (const applied of upserts) {
|
|
908
|
+
const key = `${applied.relativePath}\u0000${applied.contentHash}`;
|
|
909
|
+
if (reportedAppliedUpserts.has(key))
|
|
910
|
+
continue;
|
|
911
|
+
reportedAppliedUpserts.add(key);
|
|
912
|
+
rememberAppliedUpsert(applied.relativePath, applied.contentHash);
|
|
913
|
+
}
|
|
914
|
+
};
|
|
915
|
+
const invokePass = () => {
|
|
916
|
+
if (!runPassSupportsCooperativeCheckpoint)
|
|
917
|
+
return runPass(passArgvForRun);
|
|
918
|
+
// This callback is emitted by the production runner after journal
|
|
919
|
+
// repair/seed and before fanout. It starts ingress early, but never
|
|
920
|
+
// admits a second durable writer: queued realtime still waits for the
|
|
921
|
+
// existing scheduler's cooperative hand-off checkpoint.
|
|
922
|
+
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
|
|
923
|
+
};
|
|
677
924
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
678
925
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
679
926
|
// synchronous stretch starves the interval timer, so these boundary
|
|
@@ -685,9 +932,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
685
932
|
heapCheckpoint?.();
|
|
686
933
|
if (typeof outcome === "number")
|
|
687
934
|
return;
|
|
688
|
-
|
|
689
|
-
rememberAppliedUpsert(applied.relativePath, applied.contentHash);
|
|
690
|
-
}
|
|
935
|
+
recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
|
|
691
936
|
};
|
|
692
937
|
// Pre-admit boundary observation.
|
|
693
938
|
heapCheckpoint?.();
|
|
@@ -696,13 +941,31 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
696
941
|
// its keyed journal delta and per-object mutation are independent from the
|
|
697
942
|
// slow pass's bulk paths, and the slow pass already owns the root lock.
|
|
698
943
|
// Never bypass the lock when no slow pass is active.
|
|
944
|
+
if (
|
|
945
|
+
// A full chore is paused at a caller-declared durable checkpoint. Keep
|
|
946
|
+
// the root lock held by this process, but lend it to exactly one
|
|
947
|
+
// realtime pass; the suspended chore cannot mutate until this returns.
|
|
948
|
+
workClass === "realtime" &&
|
|
949
|
+
passScheduler.handoffActive &&
|
|
950
|
+
processOwnsOperationLock) {
|
|
951
|
+
acquireOperationLockUser();
|
|
952
|
+
try {
|
|
953
|
+
prepare?.();
|
|
954
|
+
const outcome = await invokePass();
|
|
955
|
+
recordAppliedUpserts(outcome);
|
|
956
|
+
return outcome;
|
|
957
|
+
}
|
|
958
|
+
finally {
|
|
959
|
+
releaseOperationLockUser();
|
|
960
|
+
}
|
|
961
|
+
}
|
|
699
962
|
if (lane === "fast" &&
|
|
700
963
|
processOwnsOperationLock &&
|
|
701
964
|
coordinatorWholeMutationChores === 0) {
|
|
702
965
|
acquireOperationLockUser();
|
|
703
966
|
try {
|
|
704
967
|
prepare?.();
|
|
705
|
-
const outcome = await
|
|
968
|
+
const outcome = await invokePass();
|
|
706
969
|
recordAppliedUpserts(outcome);
|
|
707
970
|
return outcome;
|
|
708
971
|
}
|
|
@@ -722,7 +985,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
722
985
|
acquireOperationLockUser();
|
|
723
986
|
}
|
|
724
987
|
prepare?.();
|
|
725
|
-
const outcome = await
|
|
988
|
+
const outcome = await invokePass();
|
|
726
989
|
recordAppliedUpserts(outcome);
|
|
727
990
|
return outcome;
|
|
728
991
|
}
|
|
@@ -756,7 +1019,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
756
1019
|
}
|
|
757
1020
|
try {
|
|
758
1021
|
prepare?.();
|
|
759
|
-
const outcome = await
|
|
1022
|
+
const outcome = await invokePass();
|
|
760
1023
|
recordAppliedUpserts(outcome);
|
|
761
1024
|
return outcome;
|
|
762
1025
|
}
|
|
@@ -779,7 +1042,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
779
1042
|
throw err;
|
|
780
1043
|
}
|
|
781
1044
|
};
|
|
782
|
-
const runTaskWithLock = async (task) => {
|
|
1045
|
+
const runTaskWithLock = async (task, workClass = "chore") => {
|
|
1046
|
+
if (workClass === "realtime" &&
|
|
1047
|
+
passScheduler.handoffActive &&
|
|
1048
|
+
processOwnsOperationLock) {
|
|
1049
|
+
acquireOperationLockUser();
|
|
1050
|
+
try {
|
|
1051
|
+
return await task();
|
|
1052
|
+
}
|
|
1053
|
+
finally {
|
|
1054
|
+
releaseOperationLockUser();
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
783
1057
|
try {
|
|
784
1058
|
const handle = acquireOperationLock(hqRoot, "sync", {
|
|
785
1059
|
timeoutSec: 0,
|
|
@@ -873,102 +1147,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
873
1147
|
heapCheckpoint?.();
|
|
874
1148
|
}
|
|
875
1149
|
};
|
|
876
|
-
let activeSlowPass = null;
|
|
877
|
-
let activeFastPass = null;
|
|
878
|
-
const pendingSlowPasses = [];
|
|
879
|
-
const pendingFastPasses = [];
|
|
880
1150
|
const resolveStoppedQueue = () => {
|
|
881
|
-
|
|
882
|
-
while (queue.length > 0)
|
|
883
|
-
queue.shift()?.resolve(0);
|
|
884
|
-
}
|
|
1151
|
+
passScheduler.stop();
|
|
885
1152
|
};
|
|
886
|
-
const
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
// refilling the realtime queue; otherwise a steady stream of fast work
|
|
893
|
-
// perpetually leapfrogs a receiver's bulk batch and its SQS poll loop
|
|
894
|
-
// cannot issue another receive.
|
|
895
|
-
if (activeSlowPass === null &&
|
|
896
|
-
activeFastPass === null &&
|
|
897
|
-
pendingSlowPasses.length > 0) {
|
|
898
|
-
const slow = pendingSlowPasses.shift();
|
|
899
|
-
if (slow) {
|
|
900
|
-
const current = startGuardedTask(slow.task, "slow");
|
|
901
|
-
void current.then(slow.resolve, slow.reject);
|
|
902
|
-
}
|
|
903
|
-
}
|
|
904
|
-
if (activeFastPass === null) {
|
|
905
|
-
const fast = pendingFastPasses.shift();
|
|
906
|
-
if (fast) {
|
|
907
|
-
const current = startGuardedTask(fast.task, "fast");
|
|
908
|
-
void current.then(fast.resolve, fast.reject);
|
|
909
|
-
}
|
|
910
|
-
}
|
|
911
|
-
// A fast pass that acquired the operation lock normally (rather than
|
|
912
|
-
// bypassing an in-process slow holder) still owns the cross-process
|
|
913
|
-
// boundary. Slow work must queue behind it; only fast work is allowed to
|
|
914
|
-
// overlap an already-lock-owning slow pass.
|
|
915
|
-
if (activeSlowPass === null && activeFastPass === null) {
|
|
916
|
-
const slow = pendingSlowPasses.shift();
|
|
917
|
-
if (slow) {
|
|
918
|
-
const current = startGuardedTask(slow.task, "slow");
|
|
919
|
-
void current.then(slow.resolve, slow.reject);
|
|
920
|
-
}
|
|
921
|
-
}
|
|
922
|
-
};
|
|
923
|
-
const startGuardedTask = (task, lane) => {
|
|
924
|
-
// Claim the lane before invoking `task`: a task can synchronously activate
|
|
925
|
-
// a watcher/scheduler callback before returning its promise. Without this
|
|
926
|
-
// placeholder that re-entrant callback sees an idle lane and starts a
|
|
927
|
-
// second lock contender.
|
|
928
|
-
const placeholder = Promise.resolve(0);
|
|
929
|
-
if (lane === "fast")
|
|
930
|
-
activeFastPass = placeholder;
|
|
931
|
-
else
|
|
932
|
-
activeSlowPass = placeholder;
|
|
933
|
-
let current;
|
|
934
|
-
try {
|
|
935
|
-
current = stopped ? placeholder : task();
|
|
936
|
-
}
|
|
937
|
-
catch (err) {
|
|
938
|
-
current = Promise.reject(err);
|
|
939
|
-
}
|
|
940
|
-
if (lane === "fast")
|
|
941
|
-
activeFastPass = current;
|
|
942
|
-
else
|
|
943
|
-
activeSlowPass = current;
|
|
944
|
-
void current
|
|
945
|
-
.finally(() => {
|
|
946
|
-
if (lane === "fast" && activeFastPass === current) {
|
|
947
|
-
activeFastPass = null;
|
|
948
|
-
drainQueuedPasses();
|
|
949
|
-
}
|
|
950
|
-
else if (lane === "slow" && activeSlowPass === current) {
|
|
951
|
-
activeSlowPass = null;
|
|
952
|
-
drainQueuedPasses();
|
|
953
|
-
}
|
|
954
|
-
})
|
|
955
|
-
.catch(() => undefined);
|
|
956
|
-
return current;
|
|
957
|
-
};
|
|
958
|
-
const runGuardedTask = (task, lane = "slow") => {
|
|
959
|
-
const active = lane === "fast"
|
|
960
|
-
? activeFastPass
|
|
961
|
-
: activeSlowPass ?? activeFastPass;
|
|
962
|
-
const queued = lane === "fast" ? pendingFastPasses : pendingSlowPasses;
|
|
963
|
-
if (active === null && queued.length === 0) {
|
|
964
|
-
return startGuardedTask(task, lane);
|
|
965
|
-
}
|
|
966
|
-
return new Promise((resolve, reject) => {
|
|
967
|
-
queued.push({ task, lane, resolve, reject });
|
|
968
|
-
drainQueuedPasses();
|
|
969
|
-
});
|
|
970
|
-
};
|
|
971
|
-
const runGuarded = (passArgvForRun, prepare, lane = "slow") => runGuardedTask(() => runPassWithLock(passArgvForRun, prepare, lane), lane);
|
|
1153
|
+
const runGuardedTask = (task, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) =>
|
|
1154
|
+
// Lanes still choose the pass execution profile, but no longer decide
|
|
1155
|
+
// chore ownership. Realtime normally waits for a chore hand-off; the
|
|
1156
|
+
// watcher fast path explicitly opts into its established scoped exception.
|
|
1157
|
+
passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake);
|
|
1158
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) => runGuardedTask((context) => runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined), lane, workClass, allowFastRealtimeOvertake);
|
|
972
1159
|
/*
|
|
973
1160
|
* Area storage and the parallel coordinator default to AUTO. AUTO turns
|
|
974
1161
|
* them on after durable personal-vault cutover; explicit off is the kill
|
|
@@ -1160,6 +1347,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1160
1347
|
});
|
|
1161
1348
|
},
|
|
1162
1349
|
onCutOver: () => {
|
|
1350
|
+
releaseScopedJournalStore(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
1163
1351
|
parallelCoordinatorEnabled = refreshCoordinatorEnabled();
|
|
1164
1352
|
if (parallelCoordinatorEnabled && coordinator === null) {
|
|
1165
1353
|
coordinator = createCoordinator();
|
|
@@ -1267,6 +1455,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1267
1455
|
let drainedUntrackedDeletes = 0;
|
|
1268
1456
|
const pendingWatcherPaths = new Map();
|
|
1269
1457
|
const pendingWatcherChanges = new Map();
|
|
1458
|
+
// A warm-up scan observes filesystem state over time. Keep an ingress
|
|
1459
|
+
// sequence so its older observations cannot overwrite a live watcher event
|
|
1460
|
+
// that arrived while the scan was yielding.
|
|
1461
|
+
const pendingWatcherChangeSequences = new Map();
|
|
1462
|
+
let nextPendingWatcherChangeSequence = 0;
|
|
1270
1463
|
let pendingWatcherOriginalBatch = null;
|
|
1271
1464
|
// Every coalesced watcher batch gets one durable in-process generation.
|
|
1272
1465
|
// A wake can cause the scheduler to revisit a drained batch, but an event
|
|
@@ -1290,11 +1483,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1290
1483
|
const paths = new Map();
|
|
1291
1484
|
const changes = new Map();
|
|
1292
1485
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
1293
|
-
const
|
|
1486
|
+
const change = batch.changes?.get(absolutePath);
|
|
1487
|
+
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
1294
1488
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
1295
1489
|
continue;
|
|
1296
1490
|
paths.set(absolutePath, relativePath);
|
|
1297
|
-
const change = batch.changes?.get(absolutePath);
|
|
1298
1491
|
if (change)
|
|
1299
1492
|
changes.set(absolutePath, change);
|
|
1300
1493
|
}
|
|
@@ -1302,7 +1495,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1302
1495
|
return null;
|
|
1303
1496
|
return { ...batch, paths, changes };
|
|
1304
1497
|
};
|
|
1305
|
-
const addPendingWatcherChange = (changedRelPath, batch) => {
|
|
1498
|
+
const addPendingWatcherChange = (changedRelPath, batch, options) => {
|
|
1306
1499
|
if (batch) {
|
|
1307
1500
|
const selectedBatch = selectWatchBatch(batch);
|
|
1308
1501
|
if (!selectedBatch)
|
|
@@ -1315,20 +1508,30 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1315
1508
|
return false;
|
|
1316
1509
|
}
|
|
1317
1510
|
}
|
|
1318
|
-
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1319
1511
|
if (batch) {
|
|
1512
|
+
const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
|
|
1513
|
+
const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
|
|
1514
|
+
return previousSequence === undefined ||
|
|
1515
|
+
options?.preserveChangesAfter === undefined ||
|
|
1516
|
+
previousSequence <= options.preserveChangesAfter;
|
|
1517
|
+
});
|
|
1518
|
+
if (entries.length === 0)
|
|
1519
|
+
return false;
|
|
1520
|
+
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1320
1521
|
pendingWatcherOriginalBatch =
|
|
1321
1522
|
pendingWatcherPaths.size === 0 &&
|
|
1322
1523
|
!pendingWatcherBareChange &&
|
|
1323
1524
|
!pendingWatcherOverflowed &&
|
|
1324
|
-
!batch.overflowed
|
|
1525
|
+
!batch.overflowed &&
|
|
1526
|
+
entries.length === batch.paths.size
|
|
1325
1527
|
? batch
|
|
1326
1528
|
: null;
|
|
1327
|
-
for (const [absolutePath, relativePath] of
|
|
1529
|
+
for (const [absolutePath, relativePath] of entries) {
|
|
1328
1530
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
1329
1531
|
const change = batch.changes?.get(absolutePath);
|
|
1330
1532
|
if (change)
|
|
1331
1533
|
pendingWatcherChanges.set(absolutePath, change);
|
|
1534
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1332
1535
|
}
|
|
1333
1536
|
if (batch.overflowed) {
|
|
1334
1537
|
pendingWatcherOverflowed = true;
|
|
@@ -1345,6 +1548,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1345
1548
|
}
|
|
1346
1549
|
return true;
|
|
1347
1550
|
}
|
|
1551
|
+
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1348
1552
|
if (changedRelPath) {
|
|
1349
1553
|
pendingWatcherOriginalBatch = null;
|
|
1350
1554
|
// Codec boundary: `changedRelPath` is a canonical vault key (the
|
|
@@ -1353,6 +1557,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1353
1557
|
// point at the impossible colon path on Windows.
|
|
1354
1558
|
const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
|
|
1355
1559
|
pendingWatcherPaths.set(absolutePath, changedRelPath);
|
|
1560
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1356
1561
|
// A bare path signal carries no delete intent; drop any change record
|
|
1357
1562
|
// a prior batch left for this path so the stale intent cannot leak
|
|
1358
1563
|
// into the coalesced batch (#227).
|
|
@@ -1394,6 +1599,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1394
1599
|
const generation = pendingWatcherBatchGeneration;
|
|
1395
1600
|
pendingWatcherPaths.clear();
|
|
1396
1601
|
pendingWatcherChanges.clear();
|
|
1602
|
+
pendingWatcherChangeSequences.clear();
|
|
1397
1603
|
pendingWatcherOriginalBatch = null;
|
|
1398
1604
|
pendingWatcherBareChange = false;
|
|
1399
1605
|
pendingWatcherWarmupCatchup = false;
|
|
@@ -1412,6 +1618,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1412
1618
|
pendingWatcherOriginalBatch = null;
|
|
1413
1619
|
for (const [absolutePath, relativePath] of paths.entries()) {
|
|
1414
1620
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
1621
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1415
1622
|
}
|
|
1416
1623
|
};
|
|
1417
1624
|
/**
|
|
@@ -1492,8 +1699,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1492
1699
|
}
|
|
1493
1700
|
return null;
|
|
1494
1701
|
};
|
|
1495
|
-
const journalCoordinates = (relativePath) => {
|
|
1496
|
-
const route = routeChangeToTarget(relativePath);
|
|
1702
|
+
const journalCoordinates = (relativePath, kind) => {
|
|
1703
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1497
1704
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
1498
1705
|
return null;
|
|
1499
1706
|
if (route.kind === "personal") {
|
|
@@ -1528,7 +1735,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1528
1735
|
declaredSize >= 0) {
|
|
1529
1736
|
return declaredSize;
|
|
1530
1737
|
}
|
|
1531
|
-
const size = journalRows(
|
|
1738
|
+
const size = journalRows(readJournalScoped(coordinates.slug, [coordinates.key]))
|
|
1739
|
+
.get(coordinates.key)?.size;
|
|
1532
1740
|
return typeof size === "number" && Number.isSafeInteger(size) && size >= 0 ? size : null;
|
|
1533
1741
|
};
|
|
1534
1742
|
/**
|
|
@@ -1575,8 +1783,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1575
1783
|
return session.get(coordinates.key) === undefined;
|
|
1576
1784
|
};
|
|
1577
1785
|
const captureLocalDeleteSnapshots = (relativePath, kind) => {
|
|
1578
|
-
const coordinates = journalCoordinates(relativePath);
|
|
1579
|
-
const route = routeChangeToTarget(relativePath);
|
|
1786
|
+
const coordinates = journalCoordinates(relativePath, kind);
|
|
1787
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1580
1788
|
if (!coordinates || !route)
|
|
1581
1789
|
return [];
|
|
1582
1790
|
// Desktop workspace Off toggles are authoritative before journal I/O.
|
|
@@ -1697,8 +1905,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1697
1905
|
* it scopes a push or pull for the affected batch.
|
|
1698
1906
|
*/
|
|
1699
1907
|
const captureQueuedLocalDeleteSnapshots = async (relativePath, kind, isCurrent) => {
|
|
1700
|
-
const coordinates = journalCoordinates(relativePath);
|
|
1701
|
-
const route = routeChangeToTarget(relativePath);
|
|
1908
|
+
const coordinates = journalCoordinates(relativePath, kind);
|
|
1909
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1702
1910
|
if (!coordinates || !route)
|
|
1703
1911
|
return [];
|
|
1704
1912
|
if (route.kind === "company" &&
|
|
@@ -1792,8 +2000,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1792
2000
|
}
|
|
1793
2001
|
return snapshots;
|
|
1794
2002
|
};
|
|
1795
|
-
const warmupJournalPaths = ()
|
|
1796
|
-
const paths = [];
|
|
2003
|
+
const warmupJournalPaths = function* () {
|
|
1797
2004
|
const journals = listJournals();
|
|
1798
2005
|
const vaultJournalPresent = journals.some(({ slug }) => slug === PERSONAL_VAULT_JOURNAL_SLUG);
|
|
1799
2006
|
for (const { slug, journal } of journals) {
|
|
@@ -1826,10 +2033,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1826
2033
|
// ~12.8k of them per runner start after a transcript prune.
|
|
1827
2034
|
if (route.kind === "personal" && isPersonalVaultDeletionExcluded(key))
|
|
1828
2035
|
continue;
|
|
1829
|
-
|
|
2036
|
+
yield route.kind === "personal" ? key : `companies/${route.slug}/${key}`;
|
|
1830
2037
|
}
|
|
1831
2038
|
}
|
|
1832
|
-
return paths;
|
|
1833
2039
|
};
|
|
1834
2040
|
const publishedContentHashLookup = (relativePath) => {
|
|
1835
2041
|
const coordinates = journalCoordinates(relativePath);
|
|
@@ -1842,13 +2048,34 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1842
2048
|
return publishedContentHashFromJournalEntryBeforeSession(entry, serviceStartedAt);
|
|
1843
2049
|
};
|
|
1844
2050
|
const prepareWatcherChanges = (changes) => {
|
|
2051
|
+
const keysByJournal = new Map();
|
|
2052
|
+
const rememberKey = (slug, key) => {
|
|
2053
|
+
const keys = keysByJournal.get(slug) ?? new Set();
|
|
2054
|
+
keys.add(key);
|
|
2055
|
+
keysByJournal.set(slug, keys);
|
|
2056
|
+
};
|
|
2057
|
+
for (const [absolutePath, change] of changes.entries()) {
|
|
2058
|
+
if (change.kind === "unlink" || change.kind === "unlinkDir") {
|
|
2059
|
+
for (const snapshot of change.deleteSnapshots ?? []) {
|
|
2060
|
+
rememberKey(snapshot.journalSlug, snapshot.journalPath);
|
|
2061
|
+
}
|
|
2062
|
+
continue;
|
|
2063
|
+
}
|
|
2064
|
+
const relativePath = path
|
|
2065
|
+
.relative(hqRoot, absolutePath)
|
|
2066
|
+
.split(path.sep)
|
|
2067
|
+
.join("/");
|
|
2068
|
+
const coordinates = journalCoordinates(relativePath);
|
|
2069
|
+
if (coordinates)
|
|
2070
|
+
rememberKey(coordinates.slug, coordinates.key);
|
|
2071
|
+
}
|
|
1845
2072
|
const journals = new Map();
|
|
1846
2073
|
const dirty = new Set();
|
|
1847
2074
|
const journalFor = (slug) => {
|
|
1848
2075
|
const loaded = journals.get(slug);
|
|
1849
2076
|
if (loaded)
|
|
1850
2077
|
return loaded;
|
|
1851
|
-
const journal =
|
|
2078
|
+
const journal = readJournalScoped(slug, [...(keysByJournal.get(slug) ?? [])]);
|
|
1852
2079
|
journals.set(slug, journal);
|
|
1853
2080
|
return journal;
|
|
1854
2081
|
};
|
|
@@ -1906,7 +2133,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1906
2133
|
const groupBatchByRoute = (batch) => {
|
|
1907
2134
|
const grouped = new Map();
|
|
1908
2135
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
1909
|
-
const
|
|
2136
|
+
const change = batch.changes?.get(absolutePath);
|
|
2137
|
+
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
1910
2138
|
if (!route)
|
|
1911
2139
|
continue;
|
|
1912
2140
|
const key = routeKey(route);
|
|
@@ -1919,7 +2147,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1919
2147
|
deleteScopeRoots: [],
|
|
1920
2148
|
};
|
|
1921
2149
|
existing.paths.set(absolutePath, relativePath);
|
|
1922
|
-
const change = batch.changes?.get(absolutePath);
|
|
1923
2150
|
if (change)
|
|
1924
2151
|
existing.changes.set(absolutePath, change);
|
|
1925
2152
|
existing.relPaths = [...new Set([...existing.paths.values()])];
|
|
@@ -2003,7 +2230,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2003
2230
|
}
|
|
2004
2231
|
return false;
|
|
2005
2232
|
};
|
|
2006
|
-
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false) => {
|
|
2233
|
+
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
|
|
2007
2234
|
if (batch)
|
|
2008
2235
|
await resolveDeleteSnapshotCaptures(batch);
|
|
2009
2236
|
const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
|
|
@@ -2032,11 +2259,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2032
2259
|
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
2033
2260
|
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
2034
2261
|
? () => prepareWatcherChanges(admittedChanges)
|
|
2035
|
-
: undefined, lane);
|
|
2262
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake);
|
|
2036
2263
|
})
|
|
2037
2264
|
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
2038
2265
|
? () => prepareWatcherChanges(group.changes)
|
|
2039
|
-
: undefined, lane), accepted: true, coordinatorRefusedPaths: [] };
|
|
2266
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
|
|
2040
2267
|
const result = execution.result;
|
|
2041
2268
|
if (execution.accepted && passExitCode(result) === 0) {
|
|
2042
2269
|
const refusedPaths = new Set(typeof result === "number"
|
|
@@ -2106,8 +2333,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2106
2333
|
? "fast"
|
|
2107
2334
|
: "slow";
|
|
2108
2335
|
const execution = parallelCoordinatorEnabled
|
|
2109
|
-
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))
|
|
2110
|
-
: { result: await runGuarded(pullArgv, undefined, lane), accepted: true };
|
|
2336
|
+
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))
|
|
2337
|
+
: { result: await runGuarded(pullArgv, undefined, lane, "realtime"), accepted: true };
|
|
2111
2338
|
const exit = execution.accepted ? passExitCode(execution.result) : 1;
|
|
2112
2339
|
if (exit !== 0 && worstExit === 0)
|
|
2113
2340
|
worstExit = exit;
|
|
@@ -2312,13 +2539,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2312
2539
|
return;
|
|
2313
2540
|
}
|
|
2314
2541
|
if (lanes?.fast &&
|
|
2315
|
-
|
|
2542
|
+
(passScheduler.hasActiveChore || passScheduler.hasActiveSlowRealtime)) {
|
|
2316
2543
|
const generation = ++nextWatcherBatchGeneration;
|
|
2317
2544
|
// Fast dispatch bypasses the pending-batch ingress, so preserve both
|
|
2318
2545
|
// of its contracts here: only selected routes may run, and V2 sees
|
|
2319
2546
|
// the same dirty signal as the compatibility drain.
|
|
2320
2547
|
realtimeState.scheduler?.signal("watcher");
|
|
2321
|
-
void runScopedDrain(lanes.fast, generation, true).then((exit) => {
|
|
2548
|
+
void runScopedDrain(lanes.fast, generation, true, true).then((exit) => {
|
|
2322
2549
|
if (exit === 0)
|
|
2323
2550
|
return;
|
|
2324
2551
|
restorePendingWatcherPaths(lanes.fast.paths, generation);
|
|
@@ -2373,29 +2600,36 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2373
2600
|
watcherRediscoveryRetries = 0;
|
|
2374
2601
|
process.stderr.write(`hq-sync-runner: event-push watcher active (watched-paths=${watchedPaths ?? "unknown"})\n`);
|
|
2375
2602
|
const startedAt = Date.now();
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2603
|
+
const liveChangeSequenceAtWarmupStart = nextPendingWatcherChangeSequence;
|
|
2604
|
+
void (async () => {
|
|
2605
|
+
try {
|
|
2606
|
+
const catchup = watcher?.collectWarmupCatchupAsync
|
|
2607
|
+
? await watcher.collectWarmupCatchupAsync(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths())
|
|
2608
|
+
: watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths());
|
|
2609
|
+
if (!catchup || catchup.cancelled || stopped || watcher === null)
|
|
2610
|
+
return;
|
|
2611
|
+
const queued = addPendingWatcherChange(undefined, catchup.batch, {
|
|
2612
|
+
preserveChangesAfter: liveChangeSequenceAtWarmupStart,
|
|
2613
|
+
});
|
|
2614
|
+
pendingWatcherWarmupCatchup ||= queued;
|
|
2615
|
+
process.stderr.write(JSON.stringify({
|
|
2616
|
+
type: "watcher-warmup-catchup",
|
|
2617
|
+
reason: "watch-set-ready",
|
|
2618
|
+
pathsFound: catchup.batch.paths.size,
|
|
2619
|
+
scannedPaths: catchup.scannedPaths,
|
|
2620
|
+
elapsedMs: Date.now() - startedAt,
|
|
2621
|
+
queued,
|
|
2622
|
+
}) + "\n");
|
|
2623
|
+
if (queued)
|
|
2624
|
+
interruptWait();
|
|
2625
|
+
}
|
|
2626
|
+
catch (err) {
|
|
2627
|
+
process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
|
|
2628
|
+
// A scan failure must not silently extend the warm-up blind spot.
|
|
2629
|
+
pendingWatcherWarmupCatchupFailed = true;
|
|
2391
2630
|
interruptWait();
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
|
|
2395
|
-
// A scan failure must not silently extend the warm-up blind spot.
|
|
2396
|
-
pendingWatcherWarmupCatchupFailed = true;
|
|
2397
|
-
interruptWait();
|
|
2398
|
-
}
|
|
2631
|
+
}
|
|
2632
|
+
})();
|
|
2399
2633
|
};
|
|
2400
2634
|
if (watcher.onReady)
|
|
2401
2635
|
watcher.onReady(reportWatcherReady);
|
|
@@ -2443,7 +2677,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2443
2677
|
// Submit every route in this lane before awaiting any result. In
|
|
2444
2678
|
// particular, a later route's fast receipt must never wait for an
|
|
2445
2679
|
// earlier route's slow pull to finish.
|
|
2446
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))));
|
|
2680
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))));
|
|
2447
2681
|
for (const execution of completed) {
|
|
2448
2682
|
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
2449
2683
|
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
@@ -2497,7 +2731,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2497
2731
|
}
|
|
2498
2732
|
for (const { route, paths } of pathsByRoute.values()) {
|
|
2499
2733
|
const targetedArgv = buildTargetedPullArgv(route, passArgv, paths);
|
|
2500
|
-
const result = await runGuarded(targetedArgv, undefined, lane);
|
|
2734
|
+
const result = await runGuarded(targetedArgv, undefined, lane, "realtime");
|
|
2501
2735
|
if (passExitCode(result) !== 0) {
|
|
2502
2736
|
throw new Error(`targeted pull failed with exit code ${passExitCode(result)}`);
|
|
2503
2737
|
}
|
|
@@ -2545,14 +2779,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2545
2779
|
const execution = await runRealtimePass("wake", "pull", [], "slow", async () => runGuardedTask(() => runTaskWithLock(async () => {
|
|
2546
2780
|
await drain();
|
|
2547
2781
|
return 0;
|
|
2548
|
-
})));
|
|
2782
|
+
}, "realtime"), "slow", "realtime"));
|
|
2549
2783
|
outcome = execution.accepted ? execution.result : 1;
|
|
2550
2784
|
}
|
|
2551
2785
|
else {
|
|
2552
2786
|
outcome = await runGuardedTask(() => runTaskWithLock(async () => {
|
|
2553
2787
|
await drain();
|
|
2554
2788
|
return 0;
|
|
2555
|
-
}));
|
|
2789
|
+
}, "realtime"), "slow", "realtime");
|
|
2556
2790
|
}
|
|
2557
2791
|
}
|
|
2558
2792
|
catch (err) {
|
|
@@ -2748,8 +2982,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2748
2982
|
targetedJournalSessions: () => targetedJournalSessionCensusSizes().targetedJournalSessions,
|
|
2749
2983
|
targetedJournalRows: () => targetedJournalSessionCensusSizes().targetedJournalRows,
|
|
2750
2984
|
cachedAreaJournalMigrations: () => journalCacheCensusSizes().cachedAreaJournalMigrations,
|
|
2751
|
-
pendingSlowPasses: () =>
|
|
2752
|
-
pendingFastPasses: () =>
|
|
2985
|
+
pendingSlowPasses: () => passScheduler.pendingLaneCount("slow"),
|
|
2986
|
+
pendingFastPasses: () => passScheduler.pendingLaneCount("fast"),
|
|
2753
2987
|
},
|
|
2754
2988
|
eventSyncSizes: () => eventSync?.censusSizes?.(),
|
|
2755
2989
|
watcherSizes: () => watcher?.censusSizes?.(),
|
|
@@ -3103,11 +3337,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3103
3337
|
quarantinedDeleteRoots.delete(root);
|
|
3104
3338
|
}
|
|
3105
3339
|
// Delete snapshot capture needs a fresh keyed view while an episode is
|
|
3106
|
-
// being assembled, but completed watch cycles do not.
|
|
3107
|
-
//
|
|
3108
|
-
//
|
|
3340
|
+
// being assembled, but completed watch cycles do not. Area-authoritative
|
|
3341
|
+
// journals are different: their bridge owns the packed aggregate that a
|
|
3342
|
+
// later watcher callback reuses. Dropping that bridge at every poll
|
|
3343
|
+
// makes the next scoped callback synchronously re-pack the whole vault.
|
|
3344
|
+
// Keep it for this runner's lifetime; layout/phase changes still close
|
|
3345
|
+
// it in openAreaMigrationIfAuthoritative(), and final shutdown releases
|
|
3346
|
+
// every bridge below.
|
|
3109
3347
|
releaseWatcherJournalStores();
|
|
3110
|
-
releaseCachedAreaJournalMigrations();
|
|
3111
3348
|
const exitCode = passExitCode(result);
|
|
3112
3349
|
if (exitCode === runtime.authRequiredPassExit) {
|
|
3113
3350
|
// The pass already emitted auth-error. Hand the sentinel to the
|
|
@@ -3142,8 +3379,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3142
3379
|
: "") +
|
|
3143
3380
|
"; staying alive, retrying next poll\n");
|
|
3144
3381
|
}
|
|
3382
|
+
else if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
3383
|
+
// HQ-SYNC-COMPANY-ROUTE-UID: routed paths are derived from local
|
|
3384
|
+
// company directory names, which may not be memberships. The runner
|
|
3385
|
+
// has already emitted the resolution error; terminating here would
|
|
3386
|
+
// restart the outpost watcher every 60 seconds for each such route.
|
|
3387
|
+
// Keep the watcher alive while preserving the one-shot non-zero code.
|
|
3388
|
+
process.stderr.write("hq-sync-runner: watch pass skipped — unresolved company target; retrying next poll\n");
|
|
3389
|
+
}
|
|
3145
3390
|
else if (exitCode !== 0) {
|
|
3146
|
-
|
|
3391
|
+
// A scoped company-resolution or per-pass failure is isolated to that
|
|
3392
|
+
// pass. The error was emitted by the runner; terminating the daemon
|
|
3393
|
+
// recreates the startup blind window and strands newer watcher work.
|
|
3394
|
+
process.stderr.write(`hq-sync-runner: watch pass failed (exit=${exitCode}); staying alive, retrying next poll\n`);
|
|
3147
3395
|
}
|
|
3148
3396
|
const completedPostRepairPass = !enqueuedFullChore && (typeof result === "number"
|
|
3149
3397
|
? exitCode === 0
|
|
@@ -3247,6 +3495,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3247
3495
|
releaseWatcherJournalStores();
|
|
3248
3496
|
releaseCachedAreaJournalMigrations();
|
|
3249
3497
|
await shutdown();
|
|
3498
|
+
// Coordinator shutdown waits for any active pass before the retained
|
|
3499
|
+
// projections are dropped. A shutdown must never evict a store that a
|
|
3500
|
+
// final scoped writer is still refreshing or committing through.
|
|
3501
|
+
disposeScopedJournalStores();
|
|
3250
3502
|
detachSignal?.();
|
|
3251
3503
|
}
|
|
3252
3504
|
}
|