@indigoai-us/hq-cloud 6.16.57 → 6.16.58-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +14 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +42 -8
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +55 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +172 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +51 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +409 -165
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +19 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +40 -5
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +47 -0
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +59 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +48 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +1553 -192
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/share.d.ts +19 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +55 -0
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +87 -1
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +12 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +297 -125
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +255 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal-row-store.d.ts +40 -5
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +240 -21
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +54 -0
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +11 -6
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +324 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +51 -16
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +303 -79
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +20 -2
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/sync/area-ledger.d.ts +27 -3
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +149 -39
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +192 -5
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +29 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +6 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +14 -2
- package/dist/sync/state-store.js.map +1 -1
- package/dist/watcher.d.ts +12 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +131 -0
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +63 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +2 -1
|
@@ -10,11 +10,11 @@ import { pickCanonicalPersonEntity } from "../vault-client.js";
|
|
|
10
10
|
import { acquireOperationLock, withOperationLock, withOperationLockSync, OperationLockedError, OPERATION_LOCKED_EXIT, } from "../operation-lock.js";
|
|
11
11
|
import { describeError } from "../lib/describe-error.js";
|
|
12
12
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
13
|
-
import { PARTIAL_SYNC_EXIT } from "../lib/exit-codes.js";
|
|
13
|
+
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
14
14
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
15
15
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
16
16
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
17
|
-
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice,
|
|
17
|
+
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, journalRows, readJournalScoped, writeJournal, } from "../journal.js";
|
|
18
18
|
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
19
19
|
import { NoopPushReceiver, } from "../sync/push-receiver.js";
|
|
20
20
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
@@ -30,6 +30,219 @@ import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
|
30
30
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
31
31
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
32
32
|
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
33
|
+
/** A deferred full chore must get the next admission after this long. */
|
|
34
|
+
export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
35
|
+
/**
|
|
36
|
+
* One durable-state owner at a time for a full chore. Realtime work never runs
|
|
37
|
+
* beside that chore: it explicitly awaits `checkpoint()` after a completed
|
|
38
|
+
* durable unit, then resumes once the hand-off is finished. A separately
|
|
39
|
+
* tracked, explicitly opted-in watcher fast pass may use the pre-existing
|
|
40
|
+
* scoped fast-over-slow exception, but never while a chore owns durable state.
|
|
41
|
+
*/
|
|
42
|
+
export class CooperativePassScheduler {
|
|
43
|
+
now;
|
|
44
|
+
starvationCapMs;
|
|
45
|
+
pending = [];
|
|
46
|
+
active = null;
|
|
47
|
+
// Scoped fast realtime work has the established in-process lock exception
|
|
48
|
+
// while a scoped slow realtime upload is in flight. Keep that slot separate
|
|
49
|
+
// from `active`: a chore still has exactly one durable-state owner.
|
|
50
|
+
activeFastRealtime = null;
|
|
51
|
+
stopped = false;
|
|
52
|
+
_handoffActive = false;
|
|
53
|
+
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS) {
|
|
54
|
+
this.now = now;
|
|
55
|
+
this.starvationCapMs = starvationCapMs;
|
|
56
|
+
}
|
|
57
|
+
get handoffActive() {
|
|
58
|
+
return this._handoffActive;
|
|
59
|
+
}
|
|
60
|
+
get hasActiveChore() {
|
|
61
|
+
return this.active?.workClass === "chore";
|
|
62
|
+
}
|
|
63
|
+
get hasActiveSlowRealtime() {
|
|
64
|
+
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
65
|
+
}
|
|
66
|
+
pendingLaneCount(lane) {
|
|
67
|
+
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
68
|
+
}
|
|
69
|
+
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false) {
|
|
70
|
+
if (this.stopped)
|
|
71
|
+
return Promise.resolve(0);
|
|
72
|
+
const directEntry = {
|
|
73
|
+
task,
|
|
74
|
+
workClass,
|
|
75
|
+
lane,
|
|
76
|
+
allowFastRealtimeOvertake,
|
|
77
|
+
// An immediately admitted task cannot starve, so avoid sampling the
|
|
78
|
+
// cadence clock merely to stamp it. This preserves the loop's deadline
|
|
79
|
+
// accounting and keeps the timestamp meaningful only for real queues.
|
|
80
|
+
enqueuedAt: 0,
|
|
81
|
+
resolve: () => undefined,
|
|
82
|
+
reject: () => undefined,
|
|
83
|
+
};
|
|
84
|
+
if (this.canStartFastRealtime(directEntry)) {
|
|
85
|
+
return this.startFastRealtime(directEntry);
|
|
86
|
+
}
|
|
87
|
+
if (this.active === null &&
|
|
88
|
+
this.activeFastRealtime === null &&
|
|
89
|
+
this.pending.length === 0) {
|
|
90
|
+
return this.start(directEntry);
|
|
91
|
+
}
|
|
92
|
+
return new Promise((resolve, reject) => {
|
|
93
|
+
const entry = { ...directEntry, enqueuedAt: this.now(), resolve, reject };
|
|
94
|
+
this.pending.push(entry);
|
|
95
|
+
this.drain();
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
stop() {
|
|
99
|
+
this.stopped = true;
|
|
100
|
+
while (this.pending.length > 0)
|
|
101
|
+
this.pending.shift()?.resolve(0);
|
|
102
|
+
}
|
|
103
|
+
drain() {
|
|
104
|
+
if (this.stopped || this.pending.length === 0)
|
|
105
|
+
return;
|
|
106
|
+
if (this.active !== null) {
|
|
107
|
+
const fastRealtime = this.nextFastRealtime();
|
|
108
|
+
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
109
|
+
this.pending.splice(this.pending.indexOf(fastRealtime), 1);
|
|
110
|
+
this.startFastRealtime(fastRealtime);
|
|
111
|
+
}
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
if (this.activeFastRealtime !== null)
|
|
115
|
+
return;
|
|
116
|
+
const next = this.takeNext();
|
|
117
|
+
if (next)
|
|
118
|
+
this.start(next);
|
|
119
|
+
}
|
|
120
|
+
takeNext() {
|
|
121
|
+
const now = this.now();
|
|
122
|
+
const starvedChore = this.pending.findIndex((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
123
|
+
if (starvedChore >= 0)
|
|
124
|
+
return this.pending.splice(starvedChore, 1)[0];
|
|
125
|
+
// A receiver's bulk drain is still realtime work, but its slow lane is the
|
|
126
|
+
// progress boundary for that delivery. Preserve its established turn
|
|
127
|
+
// before refilling later fast realtime arrivals; this does not let a
|
|
128
|
+
// chore overtake any realtime work.
|
|
129
|
+
const realtime = this.nextRealtimeIndex();
|
|
130
|
+
if (realtime >= 0)
|
|
131
|
+
return this.pending.splice(realtime, 1)[0];
|
|
132
|
+
return this.pending.shift();
|
|
133
|
+
}
|
|
134
|
+
nextRealtimeIndex() {
|
|
135
|
+
const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow");
|
|
136
|
+
return slowRealtime >= 0
|
|
137
|
+
? slowRealtime
|
|
138
|
+
: this.pending.findIndex((entry) => entry.workClass === "realtime");
|
|
139
|
+
}
|
|
140
|
+
nextFastRealtime() {
|
|
141
|
+
return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
|
|
142
|
+
}
|
|
143
|
+
hasStarvedChore() {
|
|
144
|
+
const now = this.now();
|
|
145
|
+
return this.pending.some((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
146
|
+
}
|
|
147
|
+
canStartFastRealtime(entry) {
|
|
148
|
+
return (!this._handoffActive &&
|
|
149
|
+
this.activeFastRealtime === null &&
|
|
150
|
+
entry.workClass === "realtime" &&
|
|
151
|
+
entry.lane === "fast" &&
|
|
152
|
+
entry.allowFastRealtimeOvertake &&
|
|
153
|
+
this.hasActiveSlowRealtime &&
|
|
154
|
+
!this.hasStarvedChore());
|
|
155
|
+
}
|
|
156
|
+
start(entry) {
|
|
157
|
+
this.active = entry;
|
|
158
|
+
let current;
|
|
159
|
+
try {
|
|
160
|
+
current = this.execute(entry);
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
current = Promise.reject(error);
|
|
164
|
+
}
|
|
165
|
+
void current.then((outcome) => {
|
|
166
|
+
entry.resolve(outcome);
|
|
167
|
+
this.complete(entry);
|
|
168
|
+
}, (error) => {
|
|
169
|
+
entry.reject(error);
|
|
170
|
+
this.complete(entry);
|
|
171
|
+
}).catch(() => undefined);
|
|
172
|
+
return current;
|
|
173
|
+
}
|
|
174
|
+
complete(entry) {
|
|
175
|
+
if (this.active === entry)
|
|
176
|
+
this.active = null;
|
|
177
|
+
this.drain();
|
|
178
|
+
}
|
|
179
|
+
startFastRealtime(entry) {
|
|
180
|
+
this.activeFastRealtime = entry;
|
|
181
|
+
let current;
|
|
182
|
+
try {
|
|
183
|
+
current = this.execute(entry);
|
|
184
|
+
}
|
|
185
|
+
catch (error) {
|
|
186
|
+
current = Promise.reject(error);
|
|
187
|
+
}
|
|
188
|
+
void current.then((outcome) => {
|
|
189
|
+
entry.resolve(outcome);
|
|
190
|
+
this.completeFastRealtime(entry);
|
|
191
|
+
}, (error) => {
|
|
192
|
+
entry.reject(error);
|
|
193
|
+
this.completeFastRealtime(entry);
|
|
194
|
+
});
|
|
195
|
+
return current;
|
|
196
|
+
}
|
|
197
|
+
completeFastRealtime(entry) {
|
|
198
|
+
if (this.activeFastRealtime === entry)
|
|
199
|
+
this.activeFastRealtime = null;
|
|
200
|
+
this.drain();
|
|
201
|
+
}
|
|
202
|
+
execute(entry) {
|
|
203
|
+
return entry.task({ checkpoint: () => this.checkpoint(entry) });
|
|
204
|
+
}
|
|
205
|
+
async checkpoint(entry) {
|
|
206
|
+
if (this.stopped ||
|
|
207
|
+
entry.workClass !== "chore" ||
|
|
208
|
+
this.active !== entry ||
|
|
209
|
+
!this.pending.some((pending) => pending.workClass === "realtime")) {
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
// `entry` is suspended at a caller-declared durable boundary. It remains
|
|
213
|
+
// the sole full-pass continuation and will resume after this bounded burst.
|
|
214
|
+
const deferredAt = this.now();
|
|
215
|
+
this.active = null;
|
|
216
|
+
this._handoffActive = true;
|
|
217
|
+
let handedOff = false;
|
|
218
|
+
try {
|
|
219
|
+
while (!this.stopped && this.now() - deferredAt < this.starvationCapMs) {
|
|
220
|
+
const realtimeIndex = this.nextRealtimeIndex();
|
|
221
|
+
if (realtimeIndex < 0)
|
|
222
|
+
break;
|
|
223
|
+
const realtime = this.pending.splice(realtimeIndex, 1)[0];
|
|
224
|
+
handedOff = true;
|
|
225
|
+
this.active = realtime;
|
|
226
|
+
try {
|
|
227
|
+
realtime.resolve(await this.execute(realtime));
|
|
228
|
+
}
|
|
229
|
+
catch (error) {
|
|
230
|
+
realtime.reject(error);
|
|
231
|
+
}
|
|
232
|
+
finally {
|
|
233
|
+
if (this.active === realtime)
|
|
234
|
+
this.active = null;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
finally {
|
|
239
|
+
this._handoffActive = false;
|
|
240
|
+
if (!this.stopped)
|
|
241
|
+
this.active = entry;
|
|
242
|
+
}
|
|
243
|
+
return handedOff;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
33
246
|
const MAX_FULL_RECONCILE_REASONS = 7;
|
|
34
247
|
/**
|
|
35
248
|
* Area storage is AUTO by default and only applies after durable cutover.
|
|
@@ -509,6 +722,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
509
722
|
const idleHeartbeatIntervalMs = deps.idleHeartbeatIntervalMs ?? WATCH_IDLE_HEARTBEAT_INTERVAL_MS;
|
|
510
723
|
const monotonicNow = deps.monotonicNow ?? (() => performance.now());
|
|
511
724
|
const runPass = deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
|
|
725
|
+
const runPassSupportsCooperativeCheckpoint = deps.runPass === undefined || deps.runPassSupportsCooperativeCheckpoint === true;
|
|
512
726
|
const pollIdx = argv.indexOf("--poll-remote-ms");
|
|
513
727
|
// An explicit --poll-remote-ms pins a fixed interval (menubar/CLI still set
|
|
514
728
|
// one). When it is omitted, the interval is resolved fresh each cycle from
|
|
@@ -519,7 +733,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
519
733
|
const eventPush = argv.includes("--event-push");
|
|
520
734
|
const fullReconcileIntervalMs = resolveFullReconcileMs();
|
|
521
735
|
const companiesMode = argv.includes("--companies");
|
|
522
|
-
const
|
|
736
|
+
const companyIdx = argv.indexOf("--company");
|
|
737
|
+
const companyMode = companyIdx >= 0;
|
|
738
|
+
// The tenant a single-company run owns. Carried into route selection so a
|
|
739
|
+
// watcher or receiver event under another company cannot queue a pass for it.
|
|
740
|
+
const companySlug = companyMode ? argv[companyIdx + 1] : undefined;
|
|
523
741
|
const personalMode = argv.includes("--personal");
|
|
524
742
|
// Keep this at the watcher ingress rather than importing the runner's
|
|
525
743
|
// resolver: sync-runner imports this loop, and a reverse value import would
|
|
@@ -528,6 +746,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
528
746
|
const watchRouteSelection = {
|
|
529
747
|
companies: companiesMode,
|
|
530
748
|
company: companyMode,
|
|
749
|
+
...(companySlug !== undefined ? { companySlug } : {}),
|
|
531
750
|
personal: personalMode,
|
|
532
751
|
skipPersonal: parsed.skipPersonal === true ||
|
|
533
752
|
skipPersonalEnv === "1" ||
|
|
@@ -672,7 +891,27 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
672
891
|
resolveNoOperationLockUsers = resolve;
|
|
673
892
|
});
|
|
674
893
|
};
|
|
675
|
-
const
|
|
894
|
+
const passScheduler = new CooperativePassScheduler(monotonicNow);
|
|
895
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint) => {
|
|
896
|
+
const reportedAppliedUpserts = new Set();
|
|
897
|
+
const recordDurableAppliedUpserts = (upserts) => {
|
|
898
|
+
for (const applied of upserts) {
|
|
899
|
+
const key = `${applied.relativePath}\u0000${applied.contentHash}`;
|
|
900
|
+
if (reportedAppliedUpserts.has(key))
|
|
901
|
+
continue;
|
|
902
|
+
reportedAppliedUpserts.add(key);
|
|
903
|
+
rememberAppliedUpsert(applied.relativePath, applied.contentHash);
|
|
904
|
+
}
|
|
905
|
+
};
|
|
906
|
+
const invokePass = () => {
|
|
907
|
+
if (!runPassSupportsCooperativeCheckpoint)
|
|
908
|
+
return runPass(passArgvForRun);
|
|
909
|
+
// This callback is emitted by the production runner after journal
|
|
910
|
+
// repair/seed and before fanout. It starts ingress early, but never
|
|
911
|
+
// admits a second durable writer: queued realtime still waits for the
|
|
912
|
+
// existing scheduler's cooperative hand-off checkpoint.
|
|
913
|
+
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
|
|
914
|
+
};
|
|
676
915
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
677
916
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
678
917
|
// synchronous stretch starves the interval timer, so these boundary
|
|
@@ -684,9 +923,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
684
923
|
heapCheckpoint?.();
|
|
685
924
|
if (typeof outcome === "number")
|
|
686
925
|
return;
|
|
687
|
-
|
|
688
|
-
rememberAppliedUpsert(applied.relativePath, applied.contentHash);
|
|
689
|
-
}
|
|
926
|
+
recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
|
|
690
927
|
};
|
|
691
928
|
// Pre-admit boundary observation.
|
|
692
929
|
heapCheckpoint?.();
|
|
@@ -695,13 +932,31 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
695
932
|
// its keyed journal delta and per-object mutation are independent from the
|
|
696
933
|
// slow pass's bulk paths, and the slow pass already owns the root lock.
|
|
697
934
|
// Never bypass the lock when no slow pass is active.
|
|
935
|
+
if (
|
|
936
|
+
// A full chore is paused at a caller-declared durable checkpoint. Keep
|
|
937
|
+
// the root lock held by this process, but lend it to exactly one
|
|
938
|
+
// realtime pass; the suspended chore cannot mutate until this returns.
|
|
939
|
+
workClass === "realtime" &&
|
|
940
|
+
passScheduler.handoffActive &&
|
|
941
|
+
processOwnsOperationLock) {
|
|
942
|
+
acquireOperationLockUser();
|
|
943
|
+
try {
|
|
944
|
+
prepare?.();
|
|
945
|
+
const outcome = await invokePass();
|
|
946
|
+
recordAppliedUpserts(outcome);
|
|
947
|
+
return outcome;
|
|
948
|
+
}
|
|
949
|
+
finally {
|
|
950
|
+
releaseOperationLockUser();
|
|
951
|
+
}
|
|
952
|
+
}
|
|
698
953
|
if (lane === "fast" &&
|
|
699
954
|
processOwnsOperationLock &&
|
|
700
955
|
coordinatorWholeMutationChores === 0) {
|
|
701
956
|
acquireOperationLockUser();
|
|
702
957
|
try {
|
|
703
958
|
prepare?.();
|
|
704
|
-
const outcome = await
|
|
959
|
+
const outcome = await invokePass();
|
|
705
960
|
recordAppliedUpserts(outcome);
|
|
706
961
|
return outcome;
|
|
707
962
|
}
|
|
@@ -721,7 +976,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
721
976
|
acquireOperationLockUser();
|
|
722
977
|
}
|
|
723
978
|
prepare?.();
|
|
724
|
-
const outcome = await
|
|
979
|
+
const outcome = await invokePass();
|
|
725
980
|
recordAppliedUpserts(outcome);
|
|
726
981
|
return outcome;
|
|
727
982
|
}
|
|
@@ -755,7 +1010,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
755
1010
|
}
|
|
756
1011
|
try {
|
|
757
1012
|
prepare?.();
|
|
758
|
-
const outcome = await
|
|
1013
|
+
const outcome = await invokePass();
|
|
759
1014
|
recordAppliedUpserts(outcome);
|
|
760
1015
|
return outcome;
|
|
761
1016
|
}
|
|
@@ -778,7 +1033,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
778
1033
|
throw err;
|
|
779
1034
|
}
|
|
780
1035
|
};
|
|
781
|
-
const runTaskWithLock = async (task) => {
|
|
1036
|
+
const runTaskWithLock = async (task, workClass = "chore") => {
|
|
1037
|
+
if (workClass === "realtime" &&
|
|
1038
|
+
passScheduler.handoffActive &&
|
|
1039
|
+
processOwnsOperationLock) {
|
|
1040
|
+
acquireOperationLockUser();
|
|
1041
|
+
try {
|
|
1042
|
+
return await task();
|
|
1043
|
+
}
|
|
1044
|
+
finally {
|
|
1045
|
+
releaseOperationLockUser();
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
782
1048
|
try {
|
|
783
1049
|
const handle = acquireOperationLock(hqRoot, "sync", {
|
|
784
1050
|
timeoutSec: 0,
|
|
@@ -872,102 +1138,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
872
1138
|
heapCheckpoint?.();
|
|
873
1139
|
}
|
|
874
1140
|
};
|
|
875
|
-
let activeSlowPass = null;
|
|
876
|
-
let activeFastPass = null;
|
|
877
|
-
const pendingSlowPasses = [];
|
|
878
|
-
const pendingFastPasses = [];
|
|
879
1141
|
const resolveStoppedQueue = () => {
|
|
880
|
-
|
|
881
|
-
while (queue.length > 0)
|
|
882
|
-
queue.shift()?.resolve(0);
|
|
883
|
-
}
|
|
1142
|
+
passScheduler.stop();
|
|
884
1143
|
};
|
|
885
|
-
const
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
// refilling the realtime queue; otherwise a steady stream of fast work
|
|
892
|
-
// perpetually leapfrogs a receiver's bulk batch and its SQS poll loop
|
|
893
|
-
// cannot issue another receive.
|
|
894
|
-
if (activeSlowPass === null &&
|
|
895
|
-
activeFastPass === null &&
|
|
896
|
-
pendingSlowPasses.length > 0) {
|
|
897
|
-
const slow = pendingSlowPasses.shift();
|
|
898
|
-
if (slow) {
|
|
899
|
-
const current = startGuardedTask(slow.task, "slow");
|
|
900
|
-
void current.then(slow.resolve, slow.reject);
|
|
901
|
-
}
|
|
902
|
-
}
|
|
903
|
-
if (activeFastPass === null) {
|
|
904
|
-
const fast = pendingFastPasses.shift();
|
|
905
|
-
if (fast) {
|
|
906
|
-
const current = startGuardedTask(fast.task, "fast");
|
|
907
|
-
void current.then(fast.resolve, fast.reject);
|
|
908
|
-
}
|
|
909
|
-
}
|
|
910
|
-
// A fast pass that acquired the operation lock normally (rather than
|
|
911
|
-
// bypassing an in-process slow holder) still owns the cross-process
|
|
912
|
-
// boundary. Slow work must queue behind it; only fast work is allowed to
|
|
913
|
-
// overlap an already-lock-owning slow pass.
|
|
914
|
-
if (activeSlowPass === null && activeFastPass === null) {
|
|
915
|
-
const slow = pendingSlowPasses.shift();
|
|
916
|
-
if (slow) {
|
|
917
|
-
const current = startGuardedTask(slow.task, "slow");
|
|
918
|
-
void current.then(slow.resolve, slow.reject);
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
|
-
};
|
|
922
|
-
const startGuardedTask = (task, lane) => {
|
|
923
|
-
// Claim the lane before invoking `task`: a task can synchronously activate
|
|
924
|
-
// a watcher/scheduler callback before returning its promise. Without this
|
|
925
|
-
// placeholder that re-entrant callback sees an idle lane and starts a
|
|
926
|
-
// second lock contender.
|
|
927
|
-
const placeholder = Promise.resolve(0);
|
|
928
|
-
if (lane === "fast")
|
|
929
|
-
activeFastPass = placeholder;
|
|
930
|
-
else
|
|
931
|
-
activeSlowPass = placeholder;
|
|
932
|
-
let current;
|
|
933
|
-
try {
|
|
934
|
-
current = stopped ? placeholder : task();
|
|
935
|
-
}
|
|
936
|
-
catch (err) {
|
|
937
|
-
current = Promise.reject(err);
|
|
938
|
-
}
|
|
939
|
-
if (lane === "fast")
|
|
940
|
-
activeFastPass = current;
|
|
941
|
-
else
|
|
942
|
-
activeSlowPass = current;
|
|
943
|
-
void current
|
|
944
|
-
.finally(() => {
|
|
945
|
-
if (lane === "fast" && activeFastPass === current) {
|
|
946
|
-
activeFastPass = null;
|
|
947
|
-
drainQueuedPasses();
|
|
948
|
-
}
|
|
949
|
-
else if (lane === "slow" && activeSlowPass === current) {
|
|
950
|
-
activeSlowPass = null;
|
|
951
|
-
drainQueuedPasses();
|
|
952
|
-
}
|
|
953
|
-
})
|
|
954
|
-
.catch(() => undefined);
|
|
955
|
-
return current;
|
|
956
|
-
};
|
|
957
|
-
const runGuardedTask = (task, lane = "slow") => {
|
|
958
|
-
const active = lane === "fast"
|
|
959
|
-
? activeFastPass
|
|
960
|
-
: activeSlowPass ?? activeFastPass;
|
|
961
|
-
const queued = lane === "fast" ? pendingFastPasses : pendingSlowPasses;
|
|
962
|
-
if (active === null && queued.length === 0) {
|
|
963
|
-
return startGuardedTask(task, lane);
|
|
964
|
-
}
|
|
965
|
-
return new Promise((resolve, reject) => {
|
|
966
|
-
queued.push({ task, lane, resolve, reject });
|
|
967
|
-
drainQueuedPasses();
|
|
968
|
-
});
|
|
969
|
-
};
|
|
970
|
-
const runGuarded = (passArgvForRun, prepare, lane = "slow") => runGuardedTask(() => runPassWithLock(passArgvForRun, prepare, lane), lane);
|
|
1144
|
+
const runGuardedTask = (task, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) =>
|
|
1145
|
+
// Lanes still choose the pass execution profile, but no longer decide
|
|
1146
|
+
// chore ownership. Realtime normally waits for a chore hand-off; the
|
|
1147
|
+
// watcher fast path explicitly opts into its established scoped exception.
|
|
1148
|
+
passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake);
|
|
1149
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) => runGuardedTask((context) => runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined), lane, workClass, allowFastRealtimeOvertake);
|
|
971
1150
|
/*
|
|
972
1151
|
* Area storage and the parallel coordinator default to AUTO. AUTO turns
|
|
973
1152
|
* them on after durable personal-vault cutover; explicit off is the kill
|
|
@@ -1159,6 +1338,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1159
1338
|
});
|
|
1160
1339
|
},
|
|
1161
1340
|
onCutOver: () => {
|
|
1341
|
+
releaseScopedJournalStore(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
1162
1342
|
parallelCoordinatorEnabled = refreshCoordinatorEnabled();
|
|
1163
1343
|
if (parallelCoordinatorEnabled && coordinator === null) {
|
|
1164
1344
|
coordinator = createCoordinator();
|
|
@@ -1245,6 +1425,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1245
1425
|
let drainedUntrackedDeletes = 0;
|
|
1246
1426
|
const pendingWatcherPaths = new Map();
|
|
1247
1427
|
const pendingWatcherChanges = new Map();
|
|
1428
|
+
// A warm-up scan observes filesystem state over time. Keep an ingress
|
|
1429
|
+
// sequence so its older observations cannot overwrite a live watcher event
|
|
1430
|
+
// that arrived while the scan was yielding.
|
|
1431
|
+
const pendingWatcherChangeSequences = new Map();
|
|
1432
|
+
let nextPendingWatcherChangeSequence = 0;
|
|
1248
1433
|
let pendingWatcherOriginalBatch = null;
|
|
1249
1434
|
// Every coalesced watcher batch gets one durable in-process generation.
|
|
1250
1435
|
// A wake can cause the scheduler to revisit a drained batch, but an event
|
|
@@ -1268,11 +1453,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1268
1453
|
const paths = new Map();
|
|
1269
1454
|
const changes = new Map();
|
|
1270
1455
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
1271
|
-
const
|
|
1456
|
+
const change = batch.changes?.get(absolutePath);
|
|
1457
|
+
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
1272
1458
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
1273
1459
|
continue;
|
|
1274
1460
|
paths.set(absolutePath, relativePath);
|
|
1275
|
-
const change = batch.changes?.get(absolutePath);
|
|
1276
1461
|
if (change)
|
|
1277
1462
|
changes.set(absolutePath, change);
|
|
1278
1463
|
}
|
|
@@ -1280,7 +1465,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1280
1465
|
return null;
|
|
1281
1466
|
return { ...batch, paths, changes };
|
|
1282
1467
|
};
|
|
1283
|
-
const addPendingWatcherChange = (changedRelPath, batch) => {
|
|
1468
|
+
const addPendingWatcherChange = (changedRelPath, batch, options) => {
|
|
1284
1469
|
if (batch) {
|
|
1285
1470
|
const selectedBatch = selectWatchBatch(batch);
|
|
1286
1471
|
if (!selectedBatch)
|
|
@@ -1293,20 +1478,30 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1293
1478
|
return false;
|
|
1294
1479
|
}
|
|
1295
1480
|
}
|
|
1296
|
-
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1297
1481
|
if (batch) {
|
|
1482
|
+
const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
|
|
1483
|
+
const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
|
|
1484
|
+
return previousSequence === undefined ||
|
|
1485
|
+
options?.preserveChangesAfter === undefined ||
|
|
1486
|
+
previousSequence <= options.preserveChangesAfter;
|
|
1487
|
+
});
|
|
1488
|
+
if (entries.length === 0)
|
|
1489
|
+
return false;
|
|
1490
|
+
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1298
1491
|
pendingWatcherOriginalBatch =
|
|
1299
1492
|
pendingWatcherPaths.size === 0 &&
|
|
1300
1493
|
!pendingWatcherBareChange &&
|
|
1301
1494
|
!pendingWatcherOverflowed &&
|
|
1302
|
-
!batch.overflowed
|
|
1495
|
+
!batch.overflowed &&
|
|
1496
|
+
entries.length === batch.paths.size
|
|
1303
1497
|
? batch
|
|
1304
1498
|
: null;
|
|
1305
|
-
for (const [absolutePath, relativePath] of
|
|
1499
|
+
for (const [absolutePath, relativePath] of entries) {
|
|
1306
1500
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
1307
1501
|
const change = batch.changes?.get(absolutePath);
|
|
1308
1502
|
if (change)
|
|
1309
1503
|
pendingWatcherChanges.set(absolutePath, change);
|
|
1504
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1310
1505
|
}
|
|
1311
1506
|
if (batch.overflowed) {
|
|
1312
1507
|
pendingWatcherOverflowed = true;
|
|
@@ -1323,6 +1518,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1323
1518
|
}
|
|
1324
1519
|
return true;
|
|
1325
1520
|
}
|
|
1521
|
+
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1326
1522
|
if (changedRelPath) {
|
|
1327
1523
|
pendingWatcherOriginalBatch = null;
|
|
1328
1524
|
// Codec boundary: `changedRelPath` is a canonical vault key (the
|
|
@@ -1331,6 +1527,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1331
1527
|
// point at the impossible colon path on Windows.
|
|
1332
1528
|
const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
|
|
1333
1529
|
pendingWatcherPaths.set(absolutePath, changedRelPath);
|
|
1530
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1334
1531
|
// A bare path signal carries no delete intent; drop any change record
|
|
1335
1532
|
// a prior batch left for this path so the stale intent cannot leak
|
|
1336
1533
|
// into the coalesced batch (#227).
|
|
@@ -1372,6 +1569,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1372
1569
|
const generation = pendingWatcherBatchGeneration;
|
|
1373
1570
|
pendingWatcherPaths.clear();
|
|
1374
1571
|
pendingWatcherChanges.clear();
|
|
1572
|
+
pendingWatcherChangeSequences.clear();
|
|
1375
1573
|
pendingWatcherOriginalBatch = null;
|
|
1376
1574
|
pendingWatcherBareChange = false;
|
|
1377
1575
|
pendingWatcherWarmupCatchup = false;
|
|
@@ -1390,6 +1588,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1390
1588
|
pendingWatcherOriginalBatch = null;
|
|
1391
1589
|
for (const [absolutePath, relativePath] of paths.entries()) {
|
|
1392
1590
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
1591
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1393
1592
|
}
|
|
1394
1593
|
};
|
|
1395
1594
|
/**
|
|
@@ -1470,8 +1669,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1470
1669
|
}
|
|
1471
1670
|
return null;
|
|
1472
1671
|
};
|
|
1473
|
-
const journalCoordinates = (relativePath) => {
|
|
1474
|
-
const route = routeChangeToTarget(relativePath);
|
|
1672
|
+
const journalCoordinates = (relativePath, kind) => {
|
|
1673
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1475
1674
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
1476
1675
|
return null;
|
|
1477
1676
|
if (route.kind === "personal") {
|
|
@@ -1506,7 +1705,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1506
1705
|
declaredSize >= 0) {
|
|
1507
1706
|
return declaredSize;
|
|
1508
1707
|
}
|
|
1509
|
-
const size = journalRows(
|
|
1708
|
+
const size = journalRows(readJournalScoped(coordinates.slug, [coordinates.key]))
|
|
1709
|
+
.get(coordinates.key)?.size;
|
|
1510
1710
|
return typeof size === "number" && Number.isSafeInteger(size) && size >= 0 ? size : null;
|
|
1511
1711
|
};
|
|
1512
1712
|
/**
|
|
@@ -1553,8 +1753,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1553
1753
|
return session.get(coordinates.key) === undefined;
|
|
1554
1754
|
};
|
|
1555
1755
|
const captureLocalDeleteSnapshots = (relativePath, kind) => {
|
|
1556
|
-
const coordinates = journalCoordinates(relativePath);
|
|
1557
|
-
const route = routeChangeToTarget(relativePath);
|
|
1756
|
+
const coordinates = journalCoordinates(relativePath, kind);
|
|
1757
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1558
1758
|
if (!coordinates || !route)
|
|
1559
1759
|
return [];
|
|
1560
1760
|
// Desktop workspace Off toggles are authoritative before journal I/O.
|
|
@@ -1675,8 +1875,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1675
1875
|
* it scopes a push or pull for the affected batch.
|
|
1676
1876
|
*/
|
|
1677
1877
|
const captureQueuedLocalDeleteSnapshots = async (relativePath, kind, isCurrent) => {
|
|
1678
|
-
const coordinates = journalCoordinates(relativePath);
|
|
1679
|
-
const route = routeChangeToTarget(relativePath);
|
|
1878
|
+
const coordinates = journalCoordinates(relativePath, kind);
|
|
1879
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1680
1880
|
if (!coordinates || !route)
|
|
1681
1881
|
return [];
|
|
1682
1882
|
if (route.kind === "company" &&
|
|
@@ -1770,8 +1970,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1770
1970
|
}
|
|
1771
1971
|
return snapshots;
|
|
1772
1972
|
};
|
|
1773
|
-
const warmupJournalPaths = ()
|
|
1774
|
-
const paths = [];
|
|
1973
|
+
const warmupJournalPaths = function* () {
|
|
1775
1974
|
const journals = listJournals();
|
|
1776
1975
|
const vaultJournalPresent = journals.some(({ slug }) => slug === PERSONAL_VAULT_JOURNAL_SLUG);
|
|
1777
1976
|
for (const { slug, journal } of journals) {
|
|
@@ -1804,10 +2003,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1804
2003
|
// ~12.8k of them per runner start after a transcript prune.
|
|
1805
2004
|
if (route.kind === "personal" && isPersonalVaultDeletionExcluded(key))
|
|
1806
2005
|
continue;
|
|
1807
|
-
|
|
2006
|
+
yield route.kind === "personal" ? key : `companies/${route.slug}/${key}`;
|
|
1808
2007
|
}
|
|
1809
2008
|
}
|
|
1810
|
-
return paths;
|
|
1811
2009
|
};
|
|
1812
2010
|
const publishedContentHashLookup = (relativePath) => {
|
|
1813
2011
|
const coordinates = journalCoordinates(relativePath);
|
|
@@ -1820,13 +2018,34 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1820
2018
|
return publishedContentHashFromJournalEntryBeforeSession(entry, serviceStartedAt);
|
|
1821
2019
|
};
|
|
1822
2020
|
const prepareWatcherChanges = (changes) => {
|
|
2021
|
+
const keysByJournal = new Map();
|
|
2022
|
+
const rememberKey = (slug, key) => {
|
|
2023
|
+
const keys = keysByJournal.get(slug) ?? new Set();
|
|
2024
|
+
keys.add(key);
|
|
2025
|
+
keysByJournal.set(slug, keys);
|
|
2026
|
+
};
|
|
2027
|
+
for (const [absolutePath, change] of changes.entries()) {
|
|
2028
|
+
if (change.kind === "unlink" || change.kind === "unlinkDir") {
|
|
2029
|
+
for (const snapshot of change.deleteSnapshots ?? []) {
|
|
2030
|
+
rememberKey(snapshot.journalSlug, snapshot.journalPath);
|
|
2031
|
+
}
|
|
2032
|
+
continue;
|
|
2033
|
+
}
|
|
2034
|
+
const relativePath = path
|
|
2035
|
+
.relative(hqRoot, absolutePath)
|
|
2036
|
+
.split(path.sep)
|
|
2037
|
+
.join("/");
|
|
2038
|
+
const coordinates = journalCoordinates(relativePath);
|
|
2039
|
+
if (coordinates)
|
|
2040
|
+
rememberKey(coordinates.slug, coordinates.key);
|
|
2041
|
+
}
|
|
1823
2042
|
const journals = new Map();
|
|
1824
2043
|
const dirty = new Set();
|
|
1825
2044
|
const journalFor = (slug) => {
|
|
1826
2045
|
const loaded = journals.get(slug);
|
|
1827
2046
|
if (loaded)
|
|
1828
2047
|
return loaded;
|
|
1829
|
-
const journal =
|
|
2048
|
+
const journal = readJournalScoped(slug, [...(keysByJournal.get(slug) ?? [])]);
|
|
1830
2049
|
journals.set(slug, journal);
|
|
1831
2050
|
return journal;
|
|
1832
2051
|
};
|
|
@@ -1884,7 +2103,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1884
2103
|
const groupBatchByRoute = (batch) => {
|
|
1885
2104
|
const grouped = new Map();
|
|
1886
2105
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
1887
|
-
const
|
|
2106
|
+
const change = batch.changes?.get(absolutePath);
|
|
2107
|
+
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
1888
2108
|
if (!route)
|
|
1889
2109
|
continue;
|
|
1890
2110
|
const key = routeKey(route);
|
|
@@ -1897,7 +2117,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1897
2117
|
deleteScopeRoots: [],
|
|
1898
2118
|
};
|
|
1899
2119
|
existing.paths.set(absolutePath, relativePath);
|
|
1900
|
-
const change = batch.changes?.get(absolutePath);
|
|
1901
2120
|
if (change)
|
|
1902
2121
|
existing.changes.set(absolutePath, change);
|
|
1903
2122
|
existing.relPaths = [...new Set([...existing.paths.values()])];
|
|
@@ -1981,7 +2200,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1981
2200
|
}
|
|
1982
2201
|
return false;
|
|
1983
2202
|
};
|
|
1984
|
-
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false) => {
|
|
2203
|
+
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
|
|
1985
2204
|
if (batch)
|
|
1986
2205
|
await resolveDeleteSnapshotCaptures(batch);
|
|
1987
2206
|
const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
|
|
@@ -2010,11 +2229,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2010
2229
|
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
2011
2230
|
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
2012
2231
|
? () => prepareWatcherChanges(admittedChanges)
|
|
2013
|
-
: undefined, lane);
|
|
2232
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake);
|
|
2014
2233
|
})
|
|
2015
2234
|
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
2016
2235
|
? () => prepareWatcherChanges(group.changes)
|
|
2017
|
-
: undefined, lane), accepted: true, coordinatorRefusedPaths: [] };
|
|
2236
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
|
|
2018
2237
|
const result = execution.result;
|
|
2019
2238
|
if (execution.accepted && passExitCode(result) === 0) {
|
|
2020
2239
|
const refusedPaths = new Set(typeof result === "number"
|
|
@@ -2084,8 +2303,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2084
2303
|
? "fast"
|
|
2085
2304
|
: "slow";
|
|
2086
2305
|
const execution = parallelCoordinatorEnabled
|
|
2087
|
-
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))
|
|
2088
|
-
: { result: await runGuarded(pullArgv, undefined, lane), accepted: true };
|
|
2306
|
+
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))
|
|
2307
|
+
: { result: await runGuarded(pullArgv, undefined, lane, "realtime"), accepted: true };
|
|
2089
2308
|
const exit = execution.accepted ? passExitCode(execution.result) : 1;
|
|
2090
2309
|
if (exit !== 0 && worstExit === 0)
|
|
2091
2310
|
worstExit = exit;
|
|
@@ -2290,13 +2509,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2290
2509
|
return;
|
|
2291
2510
|
}
|
|
2292
2511
|
if (lanes?.fast &&
|
|
2293
|
-
|
|
2512
|
+
(passScheduler.hasActiveChore || passScheduler.hasActiveSlowRealtime)) {
|
|
2294
2513
|
const generation = ++nextWatcherBatchGeneration;
|
|
2295
2514
|
// Fast dispatch bypasses the pending-batch ingress, so preserve both
|
|
2296
2515
|
// of its contracts here: only selected routes may run, and V2 sees
|
|
2297
2516
|
// the same dirty signal as the compatibility drain.
|
|
2298
2517
|
realtimeState.scheduler?.signal("watcher");
|
|
2299
|
-
void runScopedDrain(lanes.fast, generation, true).then((exit) => {
|
|
2518
|
+
void runScopedDrain(lanes.fast, generation, true, true).then((exit) => {
|
|
2300
2519
|
if (exit === 0)
|
|
2301
2520
|
return;
|
|
2302
2521
|
restorePendingWatcherPaths(lanes.fast.paths, generation);
|
|
@@ -2345,29 +2564,36 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2345
2564
|
watcherRediscoveryRetries = 0;
|
|
2346
2565
|
process.stderr.write(`hq-sync-runner: event-push watcher active (watched-paths=${watchedPaths ?? "unknown"})\n`);
|
|
2347
2566
|
const startedAt = Date.now();
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2567
|
+
const liveChangeSequenceAtWarmupStart = nextPendingWatcherChangeSequence;
|
|
2568
|
+
void (async () => {
|
|
2569
|
+
try {
|
|
2570
|
+
const catchup = watcher?.collectWarmupCatchupAsync
|
|
2571
|
+
? await watcher.collectWarmupCatchupAsync(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths())
|
|
2572
|
+
: watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths());
|
|
2573
|
+
if (!catchup || catchup.cancelled || stopped || watcher === null)
|
|
2574
|
+
return;
|
|
2575
|
+
const queued = addPendingWatcherChange(undefined, catchup.batch, {
|
|
2576
|
+
preserveChangesAfter: liveChangeSequenceAtWarmupStart,
|
|
2577
|
+
});
|
|
2578
|
+
pendingWatcherWarmupCatchup ||= queued;
|
|
2579
|
+
process.stderr.write(JSON.stringify({
|
|
2580
|
+
type: "watcher-warmup-catchup",
|
|
2581
|
+
reason: "watch-set-ready",
|
|
2582
|
+
pathsFound: catchup.batch.paths.size,
|
|
2583
|
+
scannedPaths: catchup.scannedPaths,
|
|
2584
|
+
elapsedMs: Date.now() - startedAt,
|
|
2585
|
+
queued,
|
|
2586
|
+
}) + "\n");
|
|
2587
|
+
if (queued)
|
|
2588
|
+
interruptWait();
|
|
2589
|
+
}
|
|
2590
|
+
catch (err) {
|
|
2591
|
+
process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
|
|
2592
|
+
// A scan failure must not silently extend the warm-up blind spot.
|
|
2593
|
+
pendingWatcherWarmupCatchupFailed = true;
|
|
2363
2594
|
interruptWait();
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
|
|
2367
|
-
// A scan failure must not silently extend the warm-up blind spot.
|
|
2368
|
-
pendingWatcherWarmupCatchupFailed = true;
|
|
2369
|
-
interruptWait();
|
|
2370
|
-
}
|
|
2595
|
+
}
|
|
2596
|
+
})();
|
|
2371
2597
|
};
|
|
2372
2598
|
if (watcher.onReady)
|
|
2373
2599
|
watcher.onReady(reportWatcherReady);
|
|
@@ -2415,7 +2641,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2415
2641
|
// Submit every route in this lane before awaiting any result. In
|
|
2416
2642
|
// particular, a later route's fast receipt must never wait for an
|
|
2417
2643
|
// earlier route's slow pull to finish.
|
|
2418
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))));
|
|
2644
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))));
|
|
2419
2645
|
for (const execution of completed) {
|
|
2420
2646
|
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
2421
2647
|
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
@@ -2469,7 +2695,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2469
2695
|
}
|
|
2470
2696
|
for (const { route, paths } of pathsByRoute.values()) {
|
|
2471
2697
|
const targetedArgv = buildTargetedPullArgv(route, passArgv, paths);
|
|
2472
|
-
const result = await runGuarded(targetedArgv, undefined, lane);
|
|
2698
|
+
const result = await runGuarded(targetedArgv, undefined, lane, "realtime");
|
|
2473
2699
|
if (passExitCode(result) !== 0) {
|
|
2474
2700
|
throw new Error(`targeted pull failed with exit code ${passExitCode(result)}`);
|
|
2475
2701
|
}
|
|
@@ -2517,14 +2743,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2517
2743
|
const execution = await runRealtimePass("wake", "pull", [], "slow", async () => runGuardedTask(() => runTaskWithLock(async () => {
|
|
2518
2744
|
await drain();
|
|
2519
2745
|
return 0;
|
|
2520
|
-
})));
|
|
2746
|
+
}, "realtime"), "slow", "realtime"));
|
|
2521
2747
|
outcome = execution.accepted ? execution.result : 1;
|
|
2522
2748
|
}
|
|
2523
2749
|
else {
|
|
2524
2750
|
outcome = await runGuardedTask(() => runTaskWithLock(async () => {
|
|
2525
2751
|
await drain();
|
|
2526
2752
|
return 0;
|
|
2527
|
-
}));
|
|
2753
|
+
}, "realtime"), "slow", "realtime");
|
|
2528
2754
|
}
|
|
2529
2755
|
}
|
|
2530
2756
|
catch (err) {
|
|
@@ -2720,8 +2946,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2720
2946
|
targetedJournalSessions: () => targetedJournalSessionCensusSizes().targetedJournalSessions,
|
|
2721
2947
|
targetedJournalRows: () => targetedJournalSessionCensusSizes().targetedJournalRows,
|
|
2722
2948
|
cachedAreaJournalMigrations: () => journalCacheCensusSizes().cachedAreaJournalMigrations,
|
|
2723
|
-
pendingSlowPasses: () =>
|
|
2724
|
-
pendingFastPasses: () =>
|
|
2949
|
+
pendingSlowPasses: () => passScheduler.pendingLaneCount("slow"),
|
|
2950
|
+
pendingFastPasses: () => passScheduler.pendingLaneCount("fast"),
|
|
2725
2951
|
},
|
|
2726
2952
|
eventSyncSizes: () => eventSync?.censusSizes?.(),
|
|
2727
2953
|
watcherSizes: () => watcher?.censusSizes?.(),
|
|
@@ -3071,11 +3297,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3071
3297
|
quarantinedDeleteRoots.delete(root);
|
|
3072
3298
|
}
|
|
3073
3299
|
// Delete snapshot capture needs a fresh keyed view while an episode is
|
|
3074
|
-
// being assembled, but completed watch cycles do not.
|
|
3075
|
-
//
|
|
3076
|
-
//
|
|
3300
|
+
// being assembled, but completed watch cycles do not. Area-authoritative
|
|
3301
|
+
// journals are different: their bridge owns the packed aggregate that a
|
|
3302
|
+
// later watcher callback reuses. Dropping that bridge at every poll
|
|
3303
|
+
// makes the next scoped callback synchronously re-pack the whole vault.
|
|
3304
|
+
// Keep it for this runner's lifetime; layout/phase changes still close
|
|
3305
|
+
// it in openAreaMigrationIfAuthoritative(), and final shutdown releases
|
|
3306
|
+
// every bridge below.
|
|
3077
3307
|
releaseWatcherJournalStores();
|
|
3078
|
-
releaseCachedAreaJournalMigrations();
|
|
3079
3308
|
const exitCode = passExitCode(result);
|
|
3080
3309
|
if (exitCode === runtime.authRequiredPassExit) {
|
|
3081
3310
|
// The pass already emitted auth-error. Hand the sentinel to the
|
|
@@ -3110,8 +3339,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3110
3339
|
: "") +
|
|
3111
3340
|
"; staying alive, retrying next poll\n");
|
|
3112
3341
|
}
|
|
3342
|
+
else if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
3343
|
+
// HQ-SYNC-COMPANY-ROUTE-UID: routed paths are derived from local
|
|
3344
|
+
// company directory names, which may not be memberships. The runner
|
|
3345
|
+
// has already emitted the resolution error; terminating here would
|
|
3346
|
+
// restart the outpost watcher every 60 seconds for each such route.
|
|
3347
|
+
// Keep the watcher alive while preserving the one-shot non-zero code.
|
|
3348
|
+
process.stderr.write("hq-sync-runner: watch pass skipped — unresolved company target; retrying next poll\n");
|
|
3349
|
+
}
|
|
3113
3350
|
else if (exitCode !== 0) {
|
|
3114
|
-
|
|
3351
|
+
// A scoped company-resolution or per-pass failure is isolated to that
|
|
3352
|
+
// pass. The error was emitted by the runner; terminating the daemon
|
|
3353
|
+
// recreates the startup blind window and strands newer watcher work.
|
|
3354
|
+
process.stderr.write(`hq-sync-runner: watch pass failed (exit=${exitCode}); staying alive, retrying next poll\n`);
|
|
3115
3355
|
}
|
|
3116
3356
|
const completedPostRepairPass = !enqueuedFullChore && (typeof result === "number"
|
|
3117
3357
|
? exitCode === 0
|
|
@@ -3215,6 +3455,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3215
3455
|
releaseWatcherJournalStores();
|
|
3216
3456
|
releaseCachedAreaJournalMigrations();
|
|
3217
3457
|
await shutdown();
|
|
3458
|
+
// Coordinator shutdown waits for any active pass before the retained
|
|
3459
|
+
// projections are dropped. A shutdown must never evict a store that a
|
|
3460
|
+
// final scoped writer is still refreshing or committing through.
|
|
3461
|
+
disposeScopedJournalStores();
|
|
3218
3462
|
detachSignal?.();
|
|
3219
3463
|
}
|
|
3220
3464
|
}
|