@indigoai-us/hq-cloud 6.16.73 → 6.16.74-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +38 -6
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +196 -49
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.test.js +492 -15
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +6 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +13 -2
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +31 -4
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -24,6 +24,14 @@ export declare const REALTIME_ADMISSION_DEADLINE_MS: number;
|
|
|
24
24
|
* that value only promotes a deferred chore to the front of the queue.
|
|
25
25
|
*/
|
|
26
26
|
export declare const REALTIME_DRAIN_BURST_MS: number;
|
|
27
|
+
/**
|
|
28
|
+
* How many realtime executes may run during one chore hand-off.
|
|
29
|
+
* HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
|
|
30
|
+
* workers. One long transfer keeps its worker; a realtime item that arrives
|
|
31
|
+
* while that execute() is in flight starts on the other slot instead of
|
|
32
|
+
* waiting for the whole batch. In-flight work is never aborted.
|
|
33
|
+
*/
|
|
34
|
+
export declare const REALTIME_WORKER_OCCUPANCY_CAP = 2;
|
|
27
35
|
export declare function setRealtimeAdmissionDeadlineMsForTesting(ms: number | undefined): void;
|
|
28
36
|
export declare function setRealtimeDrainBurstMsForTesting(ms: number | undefined): void;
|
|
29
37
|
export declare class RealtimeAdmissionTimeoutError extends Error {
|
|
@@ -56,11 +64,13 @@ export interface GuardedPassContext {
|
|
|
56
64
|
signal: AbortSignal;
|
|
57
65
|
}
|
|
58
66
|
/**
|
|
59
|
-
* One durable-state owner at a time for a full chore. Realtime work
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
67
|
+
* One durable-state owner at a time for a full chore. Realtime work runs only
|
|
68
|
+
* while that chore is suspended at `checkpoint()`. Only coordinator-fenced
|
|
69
|
+
* realtime executes may use up to {@link REALTIME_WORKER_OCCUPANCY_CAP}
|
|
70
|
+
* occupants during that hand-off; legacy execution remains single-flight.
|
|
71
|
+
* A separately tracked, explicitly opted-in watcher fast pass may use the
|
|
72
|
+
* pre-existing scoped fast-over-slow exception, but never while a chore owns
|
|
73
|
+
* durable state.
|
|
64
74
|
*/
|
|
65
75
|
export declare class CooperativePassScheduler<T> {
|
|
66
76
|
private readonly now;
|
|
@@ -68,6 +78,13 @@ export declare class CooperativePassScheduler<T> {
|
|
|
68
78
|
private readonly options;
|
|
69
79
|
private readonly pending;
|
|
70
80
|
private active;
|
|
81
|
+
/** Realtime executes started by the current hand-off. Not the chore slot. */
|
|
82
|
+
private readonly realtimeOccupants;
|
|
83
|
+
private handoffBacklog;
|
|
84
|
+
private handoffDeferredAt;
|
|
85
|
+
private handoffAdmitted;
|
|
86
|
+
private occupantEpoch;
|
|
87
|
+
private occupantWaiters;
|
|
71
88
|
private activeFastRealtime;
|
|
72
89
|
private stopped;
|
|
73
90
|
private _handoffActive;
|
|
@@ -77,8 +94,10 @@ export declare class CooperativePassScheduler<T> {
|
|
|
77
94
|
get handoffActive(): boolean;
|
|
78
95
|
get hasActiveChore(): boolean;
|
|
79
96
|
get hasActiveSlowRealtime(): boolean;
|
|
97
|
+
/** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
|
|
98
|
+
get activeRealtimeWorkerCount(): number;
|
|
80
99
|
pendingLaneCount(lane: "fast" | "slow"): number;
|
|
81
|
-
submit(task: (context: GuardedPassContext) => Promise<T>, workClass: GuardedWorkClass, lane?: "fast" | "slow", allowFastRealtimeOvertake?: boolean, signal?: AbortSignal, resourceCap?: boolean): Promise<T>;
|
|
100
|
+
submit(task: (context: GuardedPassContext) => Promise<T>, workClass: GuardedWorkClass, lane?: "fast" | "slow", allowFastRealtimeOvertake?: boolean, signal?: AbortSignal, resourceCap?: boolean, allowConcurrentHandoff?: boolean): Promise<T>;
|
|
82
101
|
stop(): void;
|
|
83
102
|
private abortedError;
|
|
84
103
|
/** Realtime stays ahead of a queued full chore so retries are not buried. */
|
|
@@ -97,6 +116,19 @@ export declare class CooperativePassScheduler<T> {
|
|
|
97
116
|
private startFastRealtime;
|
|
98
117
|
private completeFastRealtime;
|
|
99
118
|
private execute;
|
|
119
|
+
private withinHandoffAdmission;
|
|
120
|
+
private waitOccupantChange;
|
|
121
|
+
private notifyOccupantChange;
|
|
122
|
+
/** Run one realtime execute without aborting it and without taking the chore slot. */
|
|
123
|
+
private startRealtimeOccupant;
|
|
124
|
+
/** Eligible arrivals may take a free slot before hand-off admission closes. */
|
|
125
|
+
private promoteOccupancyArrivals;
|
|
126
|
+
/**
|
|
127
|
+
* Pull at most one already-queued realtime item while the admission window
|
|
128
|
+
* is open and a worker slot is free. Further backlog waits for that
|
|
129
|
+
* execute() so pre-queued hand-offs stay one durable writer at a time.
|
|
130
|
+
*/
|
|
131
|
+
private admitHandoffBacklog;
|
|
100
132
|
private drainRealtime;
|
|
101
133
|
private checkpoint;
|
|
102
134
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAG1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAG1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAqBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,EACnC,sBAAsB,UAAQ,GAC7B,OAAO,CAAC,CAAC,CAAC;IAsFb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,+EAA+E;IAC/E,OAAO,CAAC,wBAAwB;IAehC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AACjE,eAAO,MAAM,yBAAyB,EAAE,MAAM,GAAG,SAAuB,CAAC;AAEzE,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,GAAG,SAAS,CAOpB;AAiBD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAg1GjB"}
|
|
@@ -49,6 +49,14 @@ export const REALTIME_ADMISSION_DEADLINE_MS = 2 * DEFAULT_REALTIME_AUTHORIZATION
|
|
|
49
49
|
* that value only promotes a deferred chore to the front of the queue.
|
|
50
50
|
*/
|
|
51
51
|
export const REALTIME_DRAIN_BURST_MS = REALTIME_ADMISSION_DEADLINE_MS;
|
|
52
|
+
/**
|
|
53
|
+
* How many realtime executes may run during one chore hand-off.
|
|
54
|
+
* HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
|
|
55
|
+
* workers. One long transfer keeps its worker; a realtime item that arrives
|
|
56
|
+
* while that execute() is in flight starts on the other slot instead of
|
|
57
|
+
* waiting for the whole batch. In-flight work is never aborted.
|
|
58
|
+
*/
|
|
59
|
+
export const REALTIME_WORKER_OCCUPANCY_CAP = 2;
|
|
52
60
|
let realtimeAdmissionDeadlineMsForTest;
|
|
53
61
|
let realtimeDrainBurstMsForTest;
|
|
54
62
|
export function setRealtimeAdmissionDeadlineMsForTesting(ms) {
|
|
@@ -118,11 +126,13 @@ export function realtimeTelemetryReason(paths) {
|
|
|
118
126
|
return undefined;
|
|
119
127
|
}
|
|
120
128
|
/**
|
|
121
|
-
* One durable-state owner at a time for a full chore. Realtime work
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
129
|
+
* One durable-state owner at a time for a full chore. Realtime work runs only
|
|
130
|
+
* while that chore is suspended at `checkpoint()`. Only coordinator-fenced
|
|
131
|
+
* realtime executes may use up to {@link REALTIME_WORKER_OCCUPANCY_CAP}
|
|
132
|
+
* occupants during that hand-off; legacy execution remains single-flight.
|
|
133
|
+
* A separately tracked, explicitly opted-in watcher fast pass may use the
|
|
134
|
+
* pre-existing scoped fast-over-slow exception, but never while a chore owns
|
|
135
|
+
* durable state.
|
|
126
136
|
*/
|
|
127
137
|
export class CooperativePassScheduler {
|
|
128
138
|
now;
|
|
@@ -130,6 +140,13 @@ export class CooperativePassScheduler {
|
|
|
130
140
|
options;
|
|
131
141
|
pending = [];
|
|
132
142
|
active = null;
|
|
143
|
+
/** Realtime executes started by the current hand-off. Not the chore slot. */
|
|
144
|
+
realtimeOccupants = new Set();
|
|
145
|
+
handoffBacklog = null;
|
|
146
|
+
handoffDeferredAt = null;
|
|
147
|
+
handoffAdmitted = false;
|
|
148
|
+
occupantEpoch = 0;
|
|
149
|
+
occupantWaiters = [];
|
|
133
150
|
// Scoped fast realtime work has the established in-process lock exception
|
|
134
151
|
// while a scoped slow realtime upload is in flight. Keep that slot separate
|
|
135
152
|
// from `active`: a chore still has exactly one durable-state owner.
|
|
@@ -150,10 +167,16 @@ export class CooperativePassScheduler {
|
|
|
150
167
|
get hasActiveSlowRealtime() {
|
|
151
168
|
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
152
169
|
}
|
|
170
|
+
/** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
|
|
171
|
+
get activeRealtimeWorkerCount() {
|
|
172
|
+
return ((this.active?.workClass === "realtime" ? 1 : 0) +
|
|
173
|
+
(this.activeFastRealtime === null ? 0 : 1) +
|
|
174
|
+
this.realtimeOccupants.size);
|
|
175
|
+
}
|
|
153
176
|
pendingLaneCount(lane) {
|
|
154
177
|
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
155
178
|
}
|
|
156
|
-
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false, signal, resourceCap = workClass === "chore") {
|
|
179
|
+
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false, signal, resourceCap = workClass === "chore", allowConcurrentHandoff = false) {
|
|
157
180
|
if (this.stopped)
|
|
158
181
|
return Promise.resolve(0);
|
|
159
182
|
if (signal?.aborted) {
|
|
@@ -169,6 +192,7 @@ export class CooperativePassScheduler {
|
|
|
169
192
|
lane,
|
|
170
193
|
allowFastRealtimeOvertake,
|
|
171
194
|
resourceCap: workClass === "chore" && resourceCap,
|
|
195
|
+
allowConcurrentHandoff,
|
|
172
196
|
// An immediately admitted task cannot starve, so avoid sampling the
|
|
173
197
|
// cadence clock merely to stamp it. This preserves the loop's deadline
|
|
174
198
|
// accounting and keeps the timestamp meaningful only for real queues.
|
|
@@ -180,7 +204,23 @@ export class CooperativePassScheduler {
|
|
|
180
204
|
if (this.canStartFastRealtime(directEntry)) {
|
|
181
205
|
return this.startFastRealtime(directEntry);
|
|
182
206
|
}
|
|
183
|
-
if (
|
|
207
|
+
if (workClass === "realtime" &&
|
|
208
|
+
this._handoffActive &&
|
|
209
|
+
directEntry.allowConcurrentHandoff &&
|
|
210
|
+
this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
|
|
211
|
+
return new Promise((resolve, reject) => {
|
|
212
|
+
const entry = {
|
|
213
|
+
...directEntry,
|
|
214
|
+
enqueuedAt: this.now(),
|
|
215
|
+
resolve,
|
|
216
|
+
reject,
|
|
217
|
+
occupancyArrival: true,
|
|
218
|
+
};
|
|
219
|
+
this.startRealtimeOccupant(entry);
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
if (!this._handoffActive &&
|
|
223
|
+
this.active === null &&
|
|
184
224
|
this.activeFastRealtime === null &&
|
|
185
225
|
this.pending.length === 0) {
|
|
186
226
|
return this.start(directEntry);
|
|
@@ -206,6 +246,12 @@ export class CooperativePassScheduler {
|
|
|
206
246
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
207
247
|
entry.abort = () => signal.removeEventListener("abort", onAbort);
|
|
208
248
|
}
|
|
249
|
+
if (workClass === "realtime" &&
|
|
250
|
+
this._handoffActive &&
|
|
251
|
+
directEntry.allowConcurrentHandoff &&
|
|
252
|
+
this.withinHandoffAdmission()) {
|
|
253
|
+
entry.occupancyArrival = true;
|
|
254
|
+
}
|
|
209
255
|
this.enqueuePending(entry);
|
|
210
256
|
this.drain();
|
|
211
257
|
});
|
|
@@ -258,6 +304,10 @@ export class CooperativePassScheduler {
|
|
|
258
304
|
drain() {
|
|
259
305
|
if (this.stopped || this.pending.length === 0)
|
|
260
306
|
return;
|
|
307
|
+
if (this._handoffActive) {
|
|
308
|
+
this.promoteOccupancyArrivals();
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
261
311
|
if (this.active !== null) {
|
|
262
312
|
const fastRealtime = this.nextFastRealtime();
|
|
263
313
|
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
@@ -370,6 +420,92 @@ export class CooperativePassScheduler {
|
|
|
370
420
|
signal: entry.controller.signal,
|
|
371
421
|
});
|
|
372
422
|
}
|
|
423
|
+
withinHandoffAdmission() {
|
|
424
|
+
if (this.handoffDeferredAt === null)
|
|
425
|
+
return false;
|
|
426
|
+
return this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
|
|
427
|
+
}
|
|
428
|
+
waitOccupantChange(seenEpoch) {
|
|
429
|
+
if (this.occupantEpoch !== seenEpoch)
|
|
430
|
+
return Promise.resolve();
|
|
431
|
+
return new Promise((resolve) => {
|
|
432
|
+
this.occupantWaiters.push(resolve);
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
notifyOccupantChange() {
|
|
436
|
+
this.occupantEpoch += 1;
|
|
437
|
+
const waiters = this.occupantWaiters;
|
|
438
|
+
this.occupantWaiters = [];
|
|
439
|
+
for (const resolve of waiters)
|
|
440
|
+
resolve();
|
|
441
|
+
}
|
|
442
|
+
/** Run one realtime execute without aborting it and without taking the chore slot. */
|
|
443
|
+
startRealtimeOccupant(entry) {
|
|
444
|
+
this.clearAdmission(entry);
|
|
445
|
+
this.realtimeOccupants.add(entry);
|
|
446
|
+
this.handoffAdmitted = true;
|
|
447
|
+
let running;
|
|
448
|
+
try {
|
|
449
|
+
running = this.execute(entry);
|
|
450
|
+
}
|
|
451
|
+
catch (error) {
|
|
452
|
+
this.realtimeOccupants.delete(entry);
|
|
453
|
+
if (this.handoffBacklog === entry)
|
|
454
|
+
this.handoffBacklog = null;
|
|
455
|
+
entry.reject(error);
|
|
456
|
+
this.notifyOccupantChange();
|
|
457
|
+
return;
|
|
458
|
+
}
|
|
459
|
+
void running.then((outcome) => {
|
|
460
|
+
this.realtimeOccupants.delete(entry);
|
|
461
|
+
if (this.handoffBacklog === entry)
|
|
462
|
+
this.handoffBacklog = null;
|
|
463
|
+
entry.resolve(outcome);
|
|
464
|
+
this.notifyOccupantChange();
|
|
465
|
+
}, (error) => {
|
|
466
|
+
this.realtimeOccupants.delete(entry);
|
|
467
|
+
if (this.handoffBacklog === entry)
|
|
468
|
+
this.handoffBacklog = null;
|
|
469
|
+
entry.reject(error);
|
|
470
|
+
this.notifyOccupantChange();
|
|
471
|
+
});
|
|
472
|
+
}
|
|
473
|
+
/** Eligible arrivals may take a free slot before hand-off admission closes. */
|
|
474
|
+
promoteOccupancyArrivals() {
|
|
475
|
+
if (!this._handoffActive || !this.withinHandoffAdmission())
|
|
476
|
+
return;
|
|
477
|
+
while (this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
|
|
478
|
+
const index = this.pending.findIndex((pending) => pending.workClass === "realtime" &&
|
|
479
|
+
pending.occupancyArrival === true &&
|
|
480
|
+
pending.allowConcurrentHandoff);
|
|
481
|
+
if (index < 0)
|
|
482
|
+
return;
|
|
483
|
+
const arrival = this.pending.splice(index, 1)[0];
|
|
484
|
+
this.startRealtimeOccupant(arrival);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
/**
|
|
488
|
+
* Pull at most one already-queued realtime item while the admission window
|
|
489
|
+
* is open and a worker slot is free. Further backlog waits for that
|
|
490
|
+
* execute() so pre-queued hand-offs stay one durable writer at a time.
|
|
491
|
+
*/
|
|
492
|
+
admitHandoffBacklog() {
|
|
493
|
+
if (!this._handoffActive || this.handoffBacklog !== null)
|
|
494
|
+
return;
|
|
495
|
+
if (!this.withinHandoffAdmission())
|
|
496
|
+
return;
|
|
497
|
+
if (this.realtimeOccupants.size >= REALTIME_WORKER_OCCUPANCY_CAP)
|
|
498
|
+
return;
|
|
499
|
+
const realtimeIndex = this.nextRealtimeIndex();
|
|
500
|
+
if (realtimeIndex < 0)
|
|
501
|
+
return;
|
|
502
|
+
const pending = this.pending[realtimeIndex];
|
|
503
|
+
if (pending?.occupancyArrival)
|
|
504
|
+
return;
|
|
505
|
+
const realtime = this.pending.splice(realtimeIndex, 1)[0];
|
|
506
|
+
this.handoffBacklog = realtime;
|
|
507
|
+
this.startRealtimeOccupant(realtime);
|
|
508
|
+
}
|
|
373
509
|
async drainRealtime(entry) {
|
|
374
510
|
if (this.stopped ||
|
|
375
511
|
entry.workClass !== "chore" ||
|
|
@@ -377,46 +513,48 @@ export class CooperativePassScheduler {
|
|
|
377
513
|
!this.pending.some((pending) => pending.workClass === "realtime")) {
|
|
378
514
|
return false;
|
|
379
515
|
}
|
|
380
|
-
// `entry` is suspended at a caller-declared durable boundary.
|
|
381
|
-
//
|
|
382
|
-
//
|
|
383
|
-
//
|
|
384
|
-
// completion so a 45s upload is not retried every 30s.
|
|
385
|
-
const deferredAt = this.now();
|
|
386
|
-
const burstMs = realtimeDrainBurstMs();
|
|
516
|
+
// `entry` is suspended at a caller-declared durable boundary. The burst
|
|
517
|
+
// cap only stops admitting further backlog. An execute() that already
|
|
518
|
+
// started is never aborted. A realtime item that arrives while one is in
|
|
519
|
+
// flight starts on the free occupancy slot (see REALTIME_WORKER_OCCUPANCY_CAP).
|
|
387
520
|
this.active = null;
|
|
388
521
|
this._handoffActive = true;
|
|
522
|
+
this.handoffDeferredAt = this.now();
|
|
523
|
+
this.handoffAdmitted = false;
|
|
524
|
+
this.handoffBacklog = null;
|
|
389
525
|
this.options.caps?.suspendForHandoff();
|
|
390
|
-
let handedOff = false;
|
|
391
526
|
try {
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
if (
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
handedOff = true;
|
|
399
|
-
this.active = realtime;
|
|
400
|
-
try {
|
|
401
|
-
const outcome = await this.execute(realtime);
|
|
402
|
-
realtime.resolve(outcome);
|
|
403
|
-
}
|
|
404
|
-
catch (error) {
|
|
405
|
-
realtime.reject(error);
|
|
406
|
-
}
|
|
407
|
-
finally {
|
|
408
|
-
if (this.active === realtime)
|
|
409
|
-
this.active = null;
|
|
527
|
+
this.admitHandoffBacklog();
|
|
528
|
+
while (!this.stopped && (this.realtimeOccupants.size > 0 || this.withinHandoffAdmission())) {
|
|
529
|
+
if (this.realtimeOccupants.size === 0) {
|
|
530
|
+
this.admitHandoffBacklog();
|
|
531
|
+
if (this.realtimeOccupants.size === 0)
|
|
532
|
+
break;
|
|
410
533
|
}
|
|
534
|
+
const seen = this.occupantEpoch;
|
|
535
|
+
if (this.realtimeOccupants.size === 0)
|
|
536
|
+
break;
|
|
537
|
+
await this.waitOccupantChange(seen);
|
|
538
|
+
this.promoteOccupancyArrivals();
|
|
539
|
+
this.admitHandoffBacklog();
|
|
540
|
+
}
|
|
541
|
+
while (this.realtimeOccupants.size > 0) {
|
|
542
|
+
const seen = this.occupantEpoch;
|
|
543
|
+
if (this.realtimeOccupants.size === 0)
|
|
544
|
+
break;
|
|
545
|
+
await this.waitOccupantChange(seen);
|
|
546
|
+
this.promoteOccupancyArrivals();
|
|
411
547
|
}
|
|
412
548
|
}
|
|
413
549
|
finally {
|
|
414
550
|
this._handoffActive = false;
|
|
551
|
+
this.handoffDeferredAt = null;
|
|
552
|
+
this.handoffBacklog = null;
|
|
415
553
|
if (!this.stopped)
|
|
416
554
|
this.active = entry;
|
|
417
555
|
this.options.caps?.resumeAfterHandoff();
|
|
418
556
|
}
|
|
419
|
-
return
|
|
557
|
+
return this.handoffAdmitted;
|
|
420
558
|
}
|
|
421
559
|
async checkpoint(entry) {
|
|
422
560
|
if (this.stopped || entry.workClass !== "chore" || this.active !== entry) {
|
|
@@ -1112,7 +1250,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1112
1250
|
refresh.unref?.();
|
|
1113
1251
|
}
|
|
1114
1252
|
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, { caps: fullPassCaps });
|
|
1115
|
-
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint) => {
|
|
1253
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry) => {
|
|
1116
1254
|
const reportedAppliedUpserts = new Set();
|
|
1117
1255
|
const recordDurableAppliedUpserts = (upserts) => {
|
|
1118
1256
|
for (const applied of upserts) {
|
|
@@ -1161,6 +1299,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1161
1299
|
processOwnsOperationLock) {
|
|
1162
1300
|
acquireOperationLockUser();
|
|
1163
1301
|
try {
|
|
1302
|
+
telemetry?.mark("network");
|
|
1164
1303
|
prepare?.();
|
|
1165
1304
|
const outcome = await invokePass();
|
|
1166
1305
|
recordAppliedUpserts(outcome);
|
|
@@ -1175,6 +1314,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1175
1314
|
coordinatorWholeMutationChores === 0) {
|
|
1176
1315
|
acquireOperationLockUser();
|
|
1177
1316
|
try {
|
|
1317
|
+
telemetry?.mark("network");
|
|
1178
1318
|
prepare?.();
|
|
1179
1319
|
const outcome = await invokePass();
|
|
1180
1320
|
recordAppliedUpserts(outcome);
|
|
@@ -1191,6 +1331,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1191
1331
|
deferToWaiters: true,
|
|
1192
1332
|
});
|
|
1193
1333
|
try {
|
|
1334
|
+
telemetry?.mark("network");
|
|
1194
1335
|
if (lane === "slow") {
|
|
1195
1336
|
processOwnsOperationLock = true;
|
|
1196
1337
|
acquireOperationLockUser();
|
|
@@ -1224,6 +1365,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1224
1365
|
}
|
|
1225
1366
|
try {
|
|
1226
1367
|
return await withOperationLock(hqRoot, "sync", async () => {
|
|
1368
|
+
telemetry?.mark("network");
|
|
1227
1369
|
if (lane === "slow") {
|
|
1228
1370
|
processOwnsOperationLock = true;
|
|
1229
1371
|
acquireOperationLockUser();
|
|
@@ -1361,12 +1503,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1361
1503
|
const resolveStoppedQueue = () => {
|
|
1362
1504
|
passScheduler.stop();
|
|
1363
1505
|
};
|
|
1364
|
-
const runGuardedTask = (task, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false) =>
|
|
1506
|
+
const runGuardedTask = (task, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, allowConcurrentHandoff = false) =>
|
|
1365
1507
|
// Lanes still choose the pass execution profile, but no longer decide
|
|
1366
1508
|
// chore ownership. Realtime normally waits for a chore hand-off; the
|
|
1367
1509
|
// watcher fast path explicitly opts into its established scoped exception.
|
|
1368
|
-
passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap);
|
|
1369
|
-
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false
|
|
1510
|
+
passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowConcurrentHandoff);
|
|
1511
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false) => {
|
|
1512
|
+
telemetry?.mark("operationGate");
|
|
1513
|
+
return runGuardedTask((context) => {
|
|
1514
|
+
telemetry?.mark("appendLock");
|
|
1515
|
+
telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
|
|
1516
|
+
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry);
|
|
1517
|
+
}, lane, workClass, allowFastRealtimeOvertake, signal, resourceCap, allowConcurrentHandoff);
|
|
1518
|
+
};
|
|
1370
1519
|
/*
|
|
1371
1520
|
* Area storage and the parallel coordinator default to AUTO. AUTO turns
|
|
1372
1521
|
* them on after durable personal-vault cutover; explicit off is the kill
|
|
@@ -1441,15 +1590,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1441
1590
|
coordinatorRefusedPaths: [],
|
|
1442
1591
|
};
|
|
1443
1592
|
}
|
|
1593
|
+
telemetry?.mark("worker");
|
|
1444
1594
|
const execution = {
|
|
1445
1595
|
run: async (slice) => {
|
|
1446
|
-
telemetry?.mark("worker");
|
|
1447
1596
|
telemetry?.mark("conflict");
|
|
1448
|
-
telemetry
|
|
1449
|
-
telemetry?.mark("appendLock");
|
|
1450
|
-
telemetry?.mark("network");
|
|
1597
|
+
const result = await task(slice, telemetry, true);
|
|
1451
1598
|
telemetry?.mark("commit");
|
|
1452
|
-
return
|
|
1599
|
+
return result;
|
|
1453
1600
|
},
|
|
1454
1601
|
wholeMutation: options.wholeMutation ?? false,
|
|
1455
1602
|
};
|
|
@@ -2479,13 +2626,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2479
2626
|
? "fast"
|
|
2480
2627
|
: "slow";
|
|
2481
2628
|
const execution = parallelCoordinatorEnabled
|
|
2482
|
-
? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice) => {
|
|
2629
|
+
? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry, coordinatorFenced) => {
|
|
2483
2630
|
const admittedPaths = slice.paths.map((path) => path.canonicalPath);
|
|
2484
2631
|
const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
|
|
2485
2632
|
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
2486
2633
|
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
2487
2634
|
? () => prepareWatcherChanges(admittedChanges)
|
|
2488
|
-
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal);
|
|
2635
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry, coordinatorFenced);
|
|
2489
2636
|
})
|
|
2490
2637
|
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
2491
2638
|
? () => prepareWatcherChanges(group.changes)
|
|
@@ -2565,7 +2712,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2565
2712
|
? "fast"
|
|
2566
2713
|
: "slow";
|
|
2567
2714
|
const execution = parallelCoordinatorEnabled
|
|
2568
|
-
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", false, slice.signal))
|
|
2715
|
+
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced))
|
|
2569
2716
|
: { result: await runGuarded(pullArgv, undefined, lane, "realtime"), accepted: true };
|
|
2570
2717
|
const exit = scopedRouteDrainExit(execution);
|
|
2571
2718
|
if (exit !== 0 && worstExit === 0)
|
|
@@ -2919,7 +3066,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2919
3066
|
// Submit every route in this lane before awaiting any result. In
|
|
2920
3067
|
// particular, a later route's fast receipt must never wait for an
|
|
2921
3068
|
// earlier route's slow pull to finish.
|
|
2922
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime",
|
|
3069
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced))));
|
|
2923
3070
|
for (const execution of completed) {
|
|
2924
3071
|
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
2925
3072
|
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
@@ -3018,10 +3165,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3018
3165
|
let outcome;
|
|
3019
3166
|
try {
|
|
3020
3167
|
if (parallelCoordinatorEnabled) {
|
|
3021
|
-
const execution = await runRealtimePass("wake", "pull", [], "slow", async () => runGuardedTask(() => runTaskWithLock(async () => {
|
|
3168
|
+
const execution = await runRealtimePass("wake", "pull", [], "slow", async (_slice, _telemetry, coordinatorFenced) => runGuardedTask(() => runTaskWithLock(async () => {
|
|
3022
3169
|
await drain();
|
|
3023
3170
|
return 0;
|
|
3024
|
-
}, "realtime"), "slow", "realtime"));
|
|
3171
|
+
}, "realtime"), "slow", "realtime", false, undefined, false, coordinatorFenced));
|
|
3025
3172
|
outcome = execution.accepted ? execution.result : 1;
|
|
3026
3173
|
}
|
|
3027
3174
|
else {
|