@indigoai-us/hq-cloud 6.16.68 → 6.16.70-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +24 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +64 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-events.test.js +19 -0
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +25 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +171 -21
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +9 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +193 -1
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +18 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +143 -127
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +181 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +98 -49
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +105 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal-decode-window.d.ts +45 -0
- package/dist/journal-decode-window.d.ts.map +1 -0
- package/dist/journal-decode-window.js +70 -0
- package/dist/journal-decode-window.js.map +1 -0
- package/dist/journal-decode-window.test.d.ts +2 -0
- package/dist/journal-decode-window.test.d.ts.map +1 -0
- package/dist/journal-decode-window.test.js +99 -0
- package/dist/journal-decode-window.test.js.map +1 -0
- package/dist/journal-row-store.d.ts +4 -0
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +67 -2
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-streaming.heap.test.d.ts +2 -0
- package/dist/journal-streaming.heap.test.d.ts.map +1 -0
- package/dist/journal-streaming.heap.test.js +190 -0
- package/dist/journal-streaming.heap.test.js.map +1 -0
- package/dist/journal.d.ts +1 -11
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +4 -53
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +13 -7
- package/dist/journal.test.js.map +1 -1
- package/dist/lib/net-errors.d.ts +12 -0
- package/dist/lib/net-errors.d.ts.map +1 -1
- package/dist/lib/net-errors.js +22 -0
- package/dist/lib/net-errors.js.map +1 -1
- package/dist/lib/net-errors.test.js +35 -1
- package/dist/lib/net-errors.test.js.map +1 -1
- package/dist/lib/s3-content-checksum.d.ts +16 -0
- package/dist/lib/s3-content-checksum.d.ts.map +1 -0
- package/dist/lib/s3-content-checksum.js +91 -0
- package/dist/lib/s3-content-checksum.js.map +1 -0
- package/dist/lib/s3-content-checksum.test.d.ts +2 -0
- package/dist/lib/s3-content-checksum.test.d.ts.map +1 -0
- package/dist/lib/s3-content-checksum.test.js +26 -0
- package/dist/lib/s3-content-checksum.test.js.map +1 -0
- package/dist/object-io.d.ts +2 -0
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +37 -19
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +89 -2
- package/dist/object-io.test.js.map +1 -1
- package/dist/sync/candidate-uploader.d.ts.map +1 -1
- package/dist/sync/candidate-uploader.js +6 -1
- package/dist/sync/candidate-uploader.js.map +1 -1
- package/dist/sync/full-pass-caps.d.ts +113 -0
- package/dist/sync/full-pass-caps.d.ts.map +1 -0
- package/dist/sync/full-pass-caps.js +383 -0
- package/dist/sync/full-pass-caps.js.map +1 -0
- package/dist/sync/full-pass-caps.test.d.ts +2 -0
- package/dist/sync/full-pass-caps.test.d.ts.map +1 -0
- package/dist/sync/full-pass-caps.test.js +126 -0
- package/dist/sync/full-pass-caps.test.js.map +1 -0
- package/dist/sync/multipart-uploader.d.ts.map +1 -1
- package/dist/sync/multipart-uploader.js +2 -1
- package/dist/sync/multipart-uploader.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +5 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -2
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +10 -0
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/package.json +2 -1
- package/dist/journal-row-cache.d.ts +0 -102
- package/dist/journal-row-cache.d.ts.map +0 -1
- package/dist/journal-row-cache.heap.test.d.ts +0 -2
- package/dist/journal-row-cache.heap.test.d.ts.map +0 -1
- package/dist/journal-row-cache.heap.test.js +0 -102
- package/dist/journal-row-cache.heap.test.js.map +0 -1
- package/dist/journal-row-cache.js +0 -549
- package/dist/journal-row-cache.js.map +0 -1
- package/dist/journal-row-cache.test.d.ts +0 -2
- package/dist/journal-row-cache.test.d.ts.map +0 -1
- package/dist/journal-row-cache.test.js +0 -155
- package/dist/journal-row-cache.test.js.map +0 -1
|
@@ -26,13 +26,54 @@ import { emitCloudTelemetry } from "../telemetry-events.js";
|
|
|
26
26
|
import { HeapCensus, startRunnerHeapSampling, heapSampleCloudProperties, heapPressureCloudProperties, heapPressureStderrLine, heapRecycleCloudProperties, heapRecycleStderrLine, performHeapHardExit, nodeHeapReaders, nodeMajorGcObserver, resolveDeclaredHeapCeilingBytes, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled, resolveHeapRecycleLiveFraction, resolveHeapRecycleSustainWindowMs, resolveHeapRecycleEmergencyFraction, resolveHeapRecycleMinUptimeMs, resolveHeapCheckpointMs, resolveHeapPinnedFraction, resolveHeapPinnedSustainMs, resolveHeapHardExitFraction, resolveHeapRecycleLatchHardExitMs, resolveHeapFootprintRecycleEnabled, resolveHeapFootprintRecycleBytes, resolveHeapFootprintHardExitBytes, HEAP_SAMPLE_INTERVAL_ENV, HEAP_PRESSURE_FRACTION_ENV, HEAP_RECYCLE_ENABLED_ENV, HEAP_RECYCLE_LIVE_FRACTION_ENV, HEAP_RECYCLE_SUSTAIN_WINDOW_MS_ENV, HEAP_RECYCLE_EMERGENCY_FRACTION_ENV, HEAP_RECYCLE_MIN_UPTIME_MS_ENV, HEAP_CHECKPOINT_MS_ENV, HEAP_PINNED_FRACTION_ENV, HEAP_PINNED_SUSTAIN_MS_ENV, HEAP_HARD_EXIT_FRACTION_ENV, HEAP_RECYCLE_LATCH_HARD_EXIT_MS_ENV, HEAP_FOOTPRINT_RECYCLE_ENABLED_ENV, HEAP_FOOTPRINT_RECYCLE_MB_ENV, HEAP_FOOTPRINT_HARD_EXIT_MB_ENV, } from "./sync-runner-heap-telemetry.js";
|
|
27
27
|
import { setRunnerEmitObserver } from "./sync-runner-events.js";
|
|
28
28
|
import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
|
|
29
|
-
import {
|
|
29
|
+
import { createHqFlagsCapSource, FullPassCapController, } from "../sync/full-pass-caps.js";
|
|
30
|
+
import { DEFAULT_REALTIME_AUTHORIZATION_TIMEOUT_MS } from "../sync/realtime-rollout.js";
|
|
31
|
+
import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr, } from "../sync/realtime-telemetry.js";
|
|
30
32
|
import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
31
33
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
32
34
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
33
35
|
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
34
36
|
/** A deferred full chore must get the next admission after this long. */
|
|
35
37
|
export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
38
|
+
/**
|
|
39
|
+
* How long a realtime pass may sit behind an active full chore before it is
|
|
40
|
+
* handed back as retryable. Twice the authorization timeout so a single hung
|
|
41
|
+
* ObjectIO request (60s) cannot stretch "commit" into minutes, while a chore
|
|
42
|
+
* still has one 15s yield window to admit the item.
|
|
43
|
+
*/
|
|
44
|
+
export const REALTIME_ADMISSION_DEADLINE_MS = 2 * DEFAULT_REALTIME_AUTHORIZATION_TIMEOUT_MS;
|
|
45
|
+
let realtimeAdmissionDeadlineMsForTest;
|
|
46
|
+
export function setRealtimeAdmissionDeadlineMsForTesting(ms) {
|
|
47
|
+
realtimeAdmissionDeadlineMsForTest = ms;
|
|
48
|
+
}
|
|
49
|
+
function realtimeAdmissionDeadlineMs() {
|
|
50
|
+
return realtimeAdmissionDeadlineMsForTest ?? REALTIME_ADMISSION_DEADLINE_MS;
|
|
51
|
+
}
|
|
52
|
+
export class RealtimeAdmissionTimeoutError extends Error {
|
|
53
|
+
retryable = true;
|
|
54
|
+
constructor(deadlineMs) {
|
|
55
|
+
super(`realtime admission timeout after ${deadlineMs}ms`);
|
|
56
|
+
this.name = "RealtimeAdmissionTimeout";
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function realtimeTelemetryReason(paths) {
|
|
60
|
+
for (const path of paths) {
|
|
61
|
+
const reason = path.reason;
|
|
62
|
+
if (!reason)
|
|
63
|
+
continue;
|
|
64
|
+
if (reason === "superseded by newer realtime work")
|
|
65
|
+
return "superseded";
|
|
66
|
+
if (reason === "stale path epoch")
|
|
67
|
+
return "stale-epoch";
|
|
68
|
+
if (reason.startsWith("realtime admission timeout"))
|
|
69
|
+
return "admission-timeout";
|
|
70
|
+
if (reason.startsWith("sync pass exited"))
|
|
71
|
+
return "pass-exit";
|
|
72
|
+
if (reason === "coordinator is shutting down")
|
|
73
|
+
return "shutdown";
|
|
74
|
+
}
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
36
77
|
/**
|
|
37
78
|
* One durable-state owner at a time for a full chore. Realtime work never runs
|
|
38
79
|
* beside that chore: it explicitly awaits `checkpoint()` after a completed
|
|
@@ -43,6 +84,7 @@ export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
|
43
84
|
export class CooperativePassScheduler {
|
|
44
85
|
now;
|
|
45
86
|
starvationCapMs;
|
|
87
|
+
options;
|
|
46
88
|
pending = [];
|
|
47
89
|
active = null;
|
|
48
90
|
// Scoped fast realtime work has the established in-process lock exception
|
|
@@ -51,9 +93,10 @@ export class CooperativePassScheduler {
|
|
|
51
93
|
activeFastRealtime = null;
|
|
52
94
|
stopped = false;
|
|
53
95
|
_handoffActive = false;
|
|
54
|
-
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS) {
|
|
96
|
+
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
|
|
55
97
|
this.now = now;
|
|
56
98
|
this.starvationCapMs = starvationCapMs;
|
|
99
|
+
this.options = options;
|
|
57
100
|
}
|
|
58
101
|
get handoffActive() {
|
|
59
102
|
return this._handoffActive;
|
|
@@ -67,14 +110,18 @@ export class CooperativePassScheduler {
|
|
|
67
110
|
pendingLaneCount(lane) {
|
|
68
111
|
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
69
112
|
}
|
|
70
|
-
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false) {
|
|
113
|
+
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false, signal, resourceCap = workClass === "chore") {
|
|
71
114
|
if (this.stopped)
|
|
72
115
|
return Promise.resolve(0);
|
|
116
|
+
if (signal?.aborted) {
|
|
117
|
+
return Promise.reject(this.abortedError());
|
|
118
|
+
}
|
|
73
119
|
const directEntry = {
|
|
74
120
|
task,
|
|
75
121
|
workClass,
|
|
76
122
|
lane,
|
|
77
123
|
allowFastRealtimeOvertake,
|
|
124
|
+
resourceCap: workClass === "chore" && resourceCap,
|
|
78
125
|
// An immediately admitted task cannot starve, so avoid sampling the
|
|
79
126
|
// cadence clock merely to stamp it. This preserves the loop's deadline
|
|
80
127
|
// accounting and keeps the timestamp meaningful only for real queues.
|
|
@@ -91,15 +138,74 @@ export class CooperativePassScheduler {
|
|
|
91
138
|
return this.start(directEntry);
|
|
92
139
|
}
|
|
93
140
|
return new Promise((resolve, reject) => {
|
|
94
|
-
const entry = {
|
|
95
|
-
|
|
141
|
+
const entry = {
|
|
142
|
+
...directEntry,
|
|
143
|
+
enqueuedAt: this.now(),
|
|
144
|
+
resolve,
|
|
145
|
+
reject,
|
|
146
|
+
};
|
|
147
|
+
if (workClass === "realtime" && this.hasActiveChore) {
|
|
148
|
+
const deadlineMs = realtimeAdmissionDeadlineMs();
|
|
149
|
+
entry.admissionTimer = setTimeout(() => {
|
|
150
|
+
this.expireAdmission(entry, deadlineMs);
|
|
151
|
+
}, deadlineMs);
|
|
152
|
+
entry.admissionTimer.unref?.();
|
|
153
|
+
}
|
|
154
|
+
if (signal) {
|
|
155
|
+
const onAbort = () => {
|
|
156
|
+
this.dropPending(entry, this.abortedError());
|
|
157
|
+
};
|
|
158
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
159
|
+
entry.abort = () => signal.removeEventListener("abort", onAbort);
|
|
160
|
+
}
|
|
161
|
+
this.enqueuePending(entry);
|
|
96
162
|
this.drain();
|
|
97
163
|
});
|
|
98
164
|
}
|
|
99
165
|
stop() {
|
|
100
166
|
this.stopped = true;
|
|
101
|
-
while (this.pending.length > 0)
|
|
102
|
-
this.pending.shift()
|
|
167
|
+
while (this.pending.length > 0) {
|
|
168
|
+
const entry = this.pending.shift();
|
|
169
|
+
if (!entry)
|
|
170
|
+
break;
|
|
171
|
+
this.clearAdmission(entry);
|
|
172
|
+
entry.resolve(0);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
abortedError() {
|
|
176
|
+
return Object.assign(new Error("realtime work was aborted"), {
|
|
177
|
+
name: "AbortError",
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
/** Realtime stays ahead of a queued full chore so retries are not buried. */
|
|
181
|
+
enqueuePending(entry) {
|
|
182
|
+
if (entry.workClass === "realtime") {
|
|
183
|
+
const choreIdx = this.pending.findIndex((pending) => pending.workClass === "chore");
|
|
184
|
+
if (choreIdx >= 0) {
|
|
185
|
+
this.pending.splice(choreIdx, 0, entry);
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
this.pending.push(entry);
|
|
190
|
+
}
|
|
191
|
+
clearAdmission(entry) {
|
|
192
|
+
if (entry.admissionTimer !== undefined) {
|
|
193
|
+
clearTimeout(entry.admissionTimer);
|
|
194
|
+
entry.admissionTimer = undefined;
|
|
195
|
+
}
|
|
196
|
+
entry.abort?.();
|
|
197
|
+
entry.abort = undefined;
|
|
198
|
+
}
|
|
199
|
+
dropPending(entry, error) {
|
|
200
|
+
const idx = this.pending.indexOf(entry);
|
|
201
|
+
if (idx < 0)
|
|
202
|
+
return;
|
|
203
|
+
this.pending.splice(idx, 1);
|
|
204
|
+
this.clearAdmission(entry);
|
|
205
|
+
entry.reject(error);
|
|
206
|
+
}
|
|
207
|
+
expireAdmission(entry, deadlineMs) {
|
|
208
|
+
this.dropPending(entry, new RealtimeAdmissionTimeoutError(deadlineMs));
|
|
103
209
|
}
|
|
104
210
|
drain() {
|
|
105
211
|
if (this.stopped || this.pending.length === 0)
|
|
@@ -155,18 +261,25 @@ export class CooperativePassScheduler {
|
|
|
155
261
|
!this.hasStarvedChore());
|
|
156
262
|
}
|
|
157
263
|
start(entry) {
|
|
264
|
+
this.clearAdmission(entry);
|
|
158
265
|
this.active = entry;
|
|
159
266
|
let current;
|
|
160
267
|
try {
|
|
161
268
|
current = this.execute(entry);
|
|
162
269
|
}
|
|
163
270
|
catch (error) {
|
|
271
|
+
if (entry.resourceCap)
|
|
272
|
+
this.options.caps?.endChore();
|
|
164
273
|
current = Promise.reject(error);
|
|
165
274
|
}
|
|
166
275
|
void current.then((outcome) => {
|
|
276
|
+
if (entry.resourceCap)
|
|
277
|
+
this.options.caps?.endChore();
|
|
167
278
|
entry.resolve(outcome);
|
|
168
279
|
this.complete(entry);
|
|
169
280
|
}, (error) => {
|
|
281
|
+
if (entry.resourceCap)
|
|
282
|
+
this.options.caps?.endChore();
|
|
170
283
|
entry.reject(error);
|
|
171
284
|
this.complete(entry);
|
|
172
285
|
}).catch(() => undefined);
|
|
@@ -178,6 +291,7 @@ export class CooperativePassScheduler {
|
|
|
178
291
|
this.drain();
|
|
179
292
|
}
|
|
180
293
|
startFastRealtime(entry) {
|
|
294
|
+
this.clearAdmission(entry);
|
|
181
295
|
this.activeFastRealtime = entry;
|
|
182
296
|
let current;
|
|
183
297
|
try {
|
|
@@ -201,9 +315,11 @@ export class CooperativePassScheduler {
|
|
|
201
315
|
this.drain();
|
|
202
316
|
}
|
|
203
317
|
execute(entry) {
|
|
318
|
+
if (entry.resourceCap)
|
|
319
|
+
this.options.caps?.beginChore();
|
|
204
320
|
return entry.task({ checkpoint: () => this.checkpoint(entry) });
|
|
205
321
|
}
|
|
206
|
-
async
|
|
322
|
+
async drainRealtime(entry) {
|
|
207
323
|
if (this.stopped ||
|
|
208
324
|
entry.workClass !== "chore" ||
|
|
209
325
|
this.active !== entry ||
|
|
@@ -215,6 +331,7 @@ export class CooperativePassScheduler {
|
|
|
215
331
|
const deferredAt = this.now();
|
|
216
332
|
this.active = null;
|
|
217
333
|
this._handoffActive = true;
|
|
334
|
+
this.options.caps?.suspendForHandoff();
|
|
218
335
|
let handedOff = false;
|
|
219
336
|
try {
|
|
220
337
|
while (!this.stopped && this.now() - deferredAt < this.starvationCapMs) {
|
|
@@ -222,6 +339,7 @@ export class CooperativePassScheduler {
|
|
|
222
339
|
if (realtimeIndex < 0)
|
|
223
340
|
break;
|
|
224
341
|
const realtime = this.pending.splice(realtimeIndex, 1)[0];
|
|
342
|
+
this.clearAdmission(realtime);
|
|
225
343
|
handedOff = true;
|
|
226
344
|
this.active = realtime;
|
|
227
345
|
try {
|
|
@@ -240,6 +358,18 @@ export class CooperativePassScheduler {
|
|
|
240
358
|
this._handoffActive = false;
|
|
241
359
|
if (!this.stopped)
|
|
242
360
|
this.active = entry;
|
|
361
|
+
this.options.caps?.resumeAfterHandoff();
|
|
362
|
+
}
|
|
363
|
+
return handedOff;
|
|
364
|
+
}
|
|
365
|
+
async checkpoint(entry) {
|
|
366
|
+
if (this.stopped || entry.workClass !== "chore" || this.active !== entry) {
|
|
367
|
+
return false;
|
|
368
|
+
}
|
|
369
|
+
const handedOff = await this.drainRealtime(entry);
|
|
370
|
+
const caps = this.options.caps;
|
|
371
|
+
if (caps && entry.resourceCap) {
|
|
372
|
+
await caps.applyAtCheckpoint(() => this.drainRealtime(entry));
|
|
243
373
|
}
|
|
244
374
|
return handedOff;
|
|
245
375
|
}
|
|
@@ -906,7 +1036,24 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
906
1036
|
resolveNoOperationLockUsers = resolve;
|
|
907
1037
|
});
|
|
908
1038
|
};
|
|
909
|
-
const
|
|
1039
|
+
const flagSource = createHqFlagsCapSource({
|
|
1040
|
+
endpoint: process.env.HQ_FLAGS_API_URL,
|
|
1041
|
+
getToken: runtime.defaultGetAccessToken,
|
|
1042
|
+
});
|
|
1043
|
+
const fullPassCaps = deps.fullPassCaps ??
|
|
1044
|
+
new FullPassCapController({
|
|
1045
|
+
flags: flagSource,
|
|
1046
|
+
});
|
|
1047
|
+
if (!deps.fullPassCaps && process.env.HQ_FLAGS_API_URL && process.env.VITEST !== "true") {
|
|
1048
|
+
const refresh = setTimeout(() => {
|
|
1049
|
+
void flagSource.refresh?.().then((value) => {
|
|
1050
|
+
if (value)
|
|
1051
|
+
fullPassCaps.applyFlags(value);
|
|
1052
|
+
});
|
|
1053
|
+
}, 0);
|
|
1054
|
+
refresh.unref?.();
|
|
1055
|
+
}
|
|
1056
|
+
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, { caps: fullPassCaps });
|
|
910
1057
|
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint) => {
|
|
911
1058
|
const reportedAppliedUpserts = new Set();
|
|
912
1059
|
const recordDurableAppliedUpserts = (upserts) => {
|
|
@@ -1156,12 +1303,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1156
1303
|
const resolveStoppedQueue = () => {
|
|
1157
1304
|
passScheduler.stop();
|
|
1158
1305
|
};
|
|
1159
|
-
const runGuardedTask = (task, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) =>
|
|
1306
|
+
const runGuardedTask = (task, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false) =>
|
|
1160
1307
|
// Lanes still choose the pass execution profile, but no longer decide
|
|
1161
1308
|
// chore ownership. Realtime normally waits for a chore hand-off; the
|
|
1162
1309
|
// watcher fast path explicitly opts into its established scoped exception.
|
|
1163
|
-
passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake);
|
|
1164
|
-
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) => runGuardedTask((context) => runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined), lane, workClass, allowFastRealtimeOvertake);
|
|
1310
|
+
passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap);
|
|
1311
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false) => runGuardedTask((context) => runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined), lane, workClass, allowFastRealtimeOvertake, signal, resourceCap);
|
|
1165
1312
|
/*
|
|
1166
1313
|
* Area storage and the parallel coordinator default to AUTO. AUTO turns
|
|
1167
1314
|
* them on after durable personal-vault cutover; explicit off is the kill
|
|
@@ -1260,11 +1407,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1260
1407
|
? outcome.paths.every((path) => path.status === "durable" || path.status === "refused")
|
|
1261
1408
|
: allRefused;
|
|
1262
1409
|
if (telemetry) {
|
|
1263
|
-
|
|
1410
|
+
const outcomeLabel = executorSucceeded && outcome.paths.every((path) => path.status === "durable")
|
|
1264
1411
|
? "durable"
|
|
1265
1412
|
: outcome.paths.some((path) => path.status === "refused")
|
|
1266
1413
|
? "refused"
|
|
1267
|
-
: "retryable"
|
|
1414
|
+
: "retryable";
|
|
1415
|
+
telemetry.finish(outcomeLabel, realtimeTelemetryReason(outcome.paths));
|
|
1268
1416
|
}
|
|
1269
1417
|
return {
|
|
1270
1418
|
result: execution.result ?? (allRefused ? 0 : 1),
|
|
@@ -1275,7 +1423,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1275
1423
|
};
|
|
1276
1424
|
}
|
|
1277
1425
|
catch (error) {
|
|
1278
|
-
telemetry?.finish("retryable"
|
|
1426
|
+
telemetry?.finish("retryable", error instanceof RealtimeAdmissionTimeoutError
|
|
1427
|
+
? "admission-timeout"
|
|
1428
|
+
: undefined);
|
|
1279
1429
|
throw error;
|
|
1280
1430
|
}
|
|
1281
1431
|
finally {
|
|
@@ -2265,7 +2415,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2265
2415
|
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
2266
2416
|
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
2267
2417
|
? () => prepareWatcherChanges(admittedChanges)
|
|
2268
|
-
: undefined, lane, "realtime", allowFastRealtimeOvertake);
|
|
2418
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal);
|
|
2269
2419
|
})
|
|
2270
2420
|
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
2271
2421
|
? () => prepareWatcherChanges(group.changes)
|
|
@@ -2339,7 +2489,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2339
2489
|
? "fast"
|
|
2340
2490
|
: "slow";
|
|
2341
2491
|
const execution = parallelCoordinatorEnabled
|
|
2342
|
-
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))
|
|
2492
|
+
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", false, slice.signal))
|
|
2343
2493
|
: { result: await runGuarded(pullArgv, undefined, lane, "realtime"), accepted: true };
|
|
2344
2494
|
const exit = scopedRouteDrainExit(execution);
|
|
2345
2495
|
if (exit !== 0 && worstExit === 0)
|
|
@@ -2693,7 +2843,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2693
2843
|
// Submit every route in this lane before awaiting any result. In
|
|
2694
2844
|
// particular, a later route's fast receipt must never wait for an
|
|
2695
2845
|
// earlier route's slow pull to finish.
|
|
2696
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))));
|
|
2846
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", false, slice.signal))));
|
|
2697
2847
|
for (const execution of completed) {
|
|
2698
2848
|
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
2699
2849
|
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
@@ -3275,7 +3425,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3275
3425
|
if (!fullChorePending) {
|
|
3276
3426
|
fullChorePending = true;
|
|
3277
3427
|
lastFullReconcileAt = monotonicNow();
|
|
3278
|
-
enqueueChore("full-reconcile", () => runGuarded(passArgv), (execution) => {
|
|
3428
|
+
enqueueChore("full-reconcile", () => runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true), (execution) => {
|
|
3279
3429
|
fullChorePending = false;
|
|
3280
3430
|
const exit = passExitCode(execution.result);
|
|
3281
3431
|
if (exit === runtime.authRequiredPassExit) {
|
|
@@ -3327,7 +3477,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3327
3477
|
}
|
|
3328
3478
|
else {
|
|
3329
3479
|
ranFullReconcile = true;
|
|
3330
|
-
result = await runGuarded(passArgv);
|
|
3480
|
+
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3331
3481
|
}
|
|
3332
3482
|
}
|
|
3333
3483
|
else if (workingBatch === null) {
|
|
@@ -3360,7 +3510,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3360
3510
|
`falling back to a full reconcile\n`);
|
|
3361
3511
|
ranFullReconcile = true;
|
|
3362
3512
|
emitDecision("full", ["targeted-pass-failed"]);
|
|
3363
|
-
result = await runGuarded(passArgv);
|
|
3513
|
+
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3364
3514
|
}
|
|
3365
3515
|
}
|
|
3366
3516
|
// This batch's episode has now been prepared and pushed, so a later
|