@indigoai-us/hq-cloud 6.16.73 → 6.16.74-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +37 -5
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +182 -43
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.test.js +329 -14
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +6 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +13 -2
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +31 -4
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -24,6 +24,14 @@ export declare const REALTIME_ADMISSION_DEADLINE_MS: number;
|
|
|
24
24
|
* that value only promotes a deferred chore to the front of the queue.
|
|
25
25
|
*/
|
|
26
26
|
export declare const REALTIME_DRAIN_BURST_MS: number;
|
|
27
|
+
/**
|
|
28
|
+
* How many realtime executes may run during one chore hand-off.
|
|
29
|
+
* HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
|
|
30
|
+
* workers. One long transfer keeps its worker; a realtime item that arrives
|
|
31
|
+
* while that execute() is in flight starts on the other slot instead of
|
|
32
|
+
* waiting for the whole batch. In-flight work is never aborted.
|
|
33
|
+
*/
|
|
34
|
+
export declare const REALTIME_WORKER_OCCUPANCY_CAP = 2;
|
|
27
35
|
export declare function setRealtimeAdmissionDeadlineMsForTesting(ms: number | undefined): void;
|
|
28
36
|
export declare function setRealtimeDrainBurstMsForTesting(ms: number | undefined): void;
|
|
29
37
|
export declare class RealtimeAdmissionTimeoutError extends Error {
|
|
@@ -56,11 +64,13 @@ export interface GuardedPassContext {
|
|
|
56
64
|
signal: AbortSignal;
|
|
57
65
|
}
|
|
58
66
|
/**
|
|
59
|
-
* One durable-state owner at a time for a full chore. Realtime work
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
67
|
+
* One durable-state owner at a time for a full chore. Realtime work runs only
|
|
68
|
+
* while that chore is suspended at `checkpoint()`. Up to
|
|
69
|
+
* {@link REALTIME_WORKER_OCCUPANCY_CAP} realtime executes may occupy that
|
|
70
|
+
* hand-off; an arrival does not wait out a transfer that is already running.
|
|
71
|
+
* A separately tracked, explicitly opted-in watcher fast pass may use the
|
|
72
|
+
* pre-existing scoped fast-over-slow exception, but never while a chore owns
|
|
73
|
+
* durable state.
|
|
64
74
|
*/
|
|
65
75
|
export declare class CooperativePassScheduler<T> {
|
|
66
76
|
private readonly now;
|
|
@@ -68,6 +78,13 @@ export declare class CooperativePassScheduler<T> {
|
|
|
68
78
|
private readonly options;
|
|
69
79
|
private readonly pending;
|
|
70
80
|
private active;
|
|
81
|
+
/** Realtime executes started by the current hand-off. Not the chore slot. */
|
|
82
|
+
private readonly realtimeOccupants;
|
|
83
|
+
private handoffBacklog;
|
|
84
|
+
private handoffDeferredAt;
|
|
85
|
+
private handoffAdmitted;
|
|
86
|
+
private occupantEpoch;
|
|
87
|
+
private occupantWaiters;
|
|
71
88
|
private activeFastRealtime;
|
|
72
89
|
private stopped;
|
|
73
90
|
private _handoffActive;
|
|
@@ -77,6 +94,8 @@ export declare class CooperativePassScheduler<T> {
|
|
|
77
94
|
get handoffActive(): boolean;
|
|
78
95
|
get hasActiveChore(): boolean;
|
|
79
96
|
get hasActiveSlowRealtime(): boolean;
|
|
97
|
+
/** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
|
|
98
|
+
get activeRealtimeWorkerCount(): number;
|
|
80
99
|
pendingLaneCount(lane: "fast" | "slow"): number;
|
|
81
100
|
submit(task: (context: GuardedPassContext) => Promise<T>, workClass: GuardedWorkClass, lane?: "fast" | "slow", allowFastRealtimeOvertake?: boolean, signal?: AbortSignal, resourceCap?: boolean): Promise<T>;
|
|
82
101
|
stop(): void;
|
|
@@ -97,6 +116,19 @@ export declare class CooperativePassScheduler<T> {
|
|
|
97
116
|
private startFastRealtime;
|
|
98
117
|
private completeFastRealtime;
|
|
99
118
|
private execute;
|
|
119
|
+
private withinHandoffAdmission;
|
|
120
|
+
private waitOccupantChange;
|
|
121
|
+
private notifyOccupantChange;
|
|
122
|
+
/** Run one realtime execute without aborting it and without taking the chore slot. */
|
|
123
|
+
private startRealtimeOccupant;
|
|
124
|
+
/** Items that showed up while a transfer was already running take a free slot. */
|
|
125
|
+
private promoteOccupancyArrivals;
|
|
126
|
+
/**
|
|
127
|
+
* Pull at most one already-queued realtime item while the admission window
|
|
128
|
+
* is open and a worker slot is free. Further backlog waits for that
|
|
129
|
+
* execute() so pre-queued hand-offs stay one durable writer at a time.
|
|
130
|
+
*/
|
|
131
|
+
private admitHandoffBacklog;
|
|
100
132
|
private drainRealtime;
|
|
101
133
|
private checkpoint;
|
|
102
134
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAG1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAG1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAmBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,GAClC,OAAO,CAAC,CAAC,CAAC;IA8Eb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,kFAAkF;IAClF,OAAO,CAAC,wBAAwB;IAYhC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AACjE,eAAO,MAAM,yBAAyB,EAAE,MAAM,GAAG,SAAuB,CAAC;AAEzE,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,GAAG,SAAS,CAOpB;AAiBD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAwzGjB"}
|
|
@@ -49,6 +49,14 @@ export const REALTIME_ADMISSION_DEADLINE_MS = 2 * DEFAULT_REALTIME_AUTHORIZATION
|
|
|
49
49
|
* that value only promotes a deferred chore to the front of the queue.
|
|
50
50
|
*/
|
|
51
51
|
export const REALTIME_DRAIN_BURST_MS = REALTIME_ADMISSION_DEADLINE_MS;
|
|
52
|
+
/**
|
|
53
|
+
* How many realtime executes may run during one chore hand-off.
|
|
54
|
+
* HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
|
|
55
|
+
* workers. One long transfer keeps its worker; a realtime item that arrives
|
|
56
|
+
* while that execute() is in flight starts on the other slot instead of
|
|
57
|
+
* waiting for the whole batch. In-flight work is never aborted.
|
|
58
|
+
*/
|
|
59
|
+
export const REALTIME_WORKER_OCCUPANCY_CAP = 2;
|
|
52
60
|
let realtimeAdmissionDeadlineMsForTest;
|
|
53
61
|
let realtimeDrainBurstMsForTest;
|
|
54
62
|
export function setRealtimeAdmissionDeadlineMsForTesting(ms) {
|
|
@@ -118,11 +126,13 @@ export function realtimeTelemetryReason(paths) {
|
|
|
118
126
|
return undefined;
|
|
119
127
|
}
|
|
120
128
|
/**
|
|
121
|
-
* One durable-state owner at a time for a full chore. Realtime work
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
129
|
+
* One durable-state owner at a time for a full chore. Realtime work runs only
|
|
130
|
+
* while that chore is suspended at `checkpoint()`. Up to
|
|
131
|
+
* {@link REALTIME_WORKER_OCCUPANCY_CAP} realtime executes may occupy that
|
|
132
|
+
* hand-off; an arrival does not wait out a transfer that is already running.
|
|
133
|
+
* A separately tracked, explicitly opted-in watcher fast pass may use the
|
|
134
|
+
* pre-existing scoped fast-over-slow exception, but never while a chore owns
|
|
135
|
+
* durable state.
|
|
126
136
|
*/
|
|
127
137
|
export class CooperativePassScheduler {
|
|
128
138
|
now;
|
|
@@ -130,6 +140,13 @@ export class CooperativePassScheduler {
|
|
|
130
140
|
options;
|
|
131
141
|
pending = [];
|
|
132
142
|
active = null;
|
|
143
|
+
/** Realtime executes started by the current hand-off. Not the chore slot. */
|
|
144
|
+
realtimeOccupants = new Set();
|
|
145
|
+
handoffBacklog = null;
|
|
146
|
+
handoffDeferredAt = null;
|
|
147
|
+
handoffAdmitted = false;
|
|
148
|
+
occupantEpoch = 0;
|
|
149
|
+
occupantWaiters = [];
|
|
133
150
|
// Scoped fast realtime work has the established in-process lock exception
|
|
134
151
|
// while a scoped slow realtime upload is in flight. Keep that slot separate
|
|
135
152
|
// from `active`: a chore still has exactly one durable-state owner.
|
|
@@ -150,6 +167,12 @@ export class CooperativePassScheduler {
|
|
|
150
167
|
get hasActiveSlowRealtime() {
|
|
151
168
|
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
152
169
|
}
|
|
170
|
+
/** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
|
|
171
|
+
get activeRealtimeWorkerCount() {
|
|
172
|
+
return ((this.active?.workClass === "realtime" ? 1 : 0) +
|
|
173
|
+
(this.activeFastRealtime === null ? 0 : 1) +
|
|
174
|
+
this.realtimeOccupants.size);
|
|
175
|
+
}
|
|
153
176
|
pendingLaneCount(lane) {
|
|
154
177
|
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
155
178
|
}
|
|
@@ -180,6 +203,20 @@ export class CooperativePassScheduler {
|
|
|
180
203
|
if (this.canStartFastRealtime(directEntry)) {
|
|
181
204
|
return this.startFastRealtime(directEntry);
|
|
182
205
|
}
|
|
206
|
+
if (workClass === "realtime" &&
|
|
207
|
+
this._handoffActive &&
|
|
208
|
+
this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
|
|
209
|
+
return new Promise((resolve, reject) => {
|
|
210
|
+
const entry = {
|
|
211
|
+
...directEntry,
|
|
212
|
+
enqueuedAt: this.now(),
|
|
213
|
+
resolve,
|
|
214
|
+
reject,
|
|
215
|
+
occupancyArrival: true,
|
|
216
|
+
};
|
|
217
|
+
this.startRealtimeOccupant(entry);
|
|
218
|
+
});
|
|
219
|
+
}
|
|
183
220
|
if (this.active === null &&
|
|
184
221
|
this.activeFastRealtime === null &&
|
|
185
222
|
this.pending.length === 0) {
|
|
@@ -206,6 +243,9 @@ export class CooperativePassScheduler {
|
|
|
206
243
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
207
244
|
entry.abort = () => signal.removeEventListener("abort", onAbort);
|
|
208
245
|
}
|
|
246
|
+
if (workClass === "realtime" && this._handoffActive) {
|
|
247
|
+
entry.occupancyArrival = true;
|
|
248
|
+
}
|
|
209
249
|
this.enqueuePending(entry);
|
|
210
250
|
this.drain();
|
|
211
251
|
});
|
|
@@ -258,6 +298,10 @@ export class CooperativePassScheduler {
|
|
|
258
298
|
drain() {
|
|
259
299
|
if (this.stopped || this.pending.length === 0)
|
|
260
300
|
return;
|
|
301
|
+
if (this._handoffActive) {
|
|
302
|
+
this.promoteOccupancyArrivals();
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
261
305
|
if (this.active !== null) {
|
|
262
306
|
const fastRealtime = this.nextFastRealtime();
|
|
263
307
|
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
@@ -370,6 +414,90 @@ export class CooperativePassScheduler {
|
|
|
370
414
|
signal: entry.controller.signal,
|
|
371
415
|
});
|
|
372
416
|
}
|
|
417
|
+
withinHandoffAdmission() {
|
|
418
|
+
if (this.handoffDeferredAt === null)
|
|
419
|
+
return false;
|
|
420
|
+
return this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
|
|
421
|
+
}
|
|
422
|
+
waitOccupantChange(seenEpoch) {
|
|
423
|
+
if (this.occupantEpoch !== seenEpoch)
|
|
424
|
+
return Promise.resolve();
|
|
425
|
+
return new Promise((resolve) => {
|
|
426
|
+
this.occupantWaiters.push(resolve);
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
notifyOccupantChange() {
|
|
430
|
+
this.occupantEpoch += 1;
|
|
431
|
+
const waiters = this.occupantWaiters;
|
|
432
|
+
this.occupantWaiters = [];
|
|
433
|
+
for (const resolve of waiters)
|
|
434
|
+
resolve();
|
|
435
|
+
}
|
|
436
|
+
/** Run one realtime execute without aborting it and without taking the chore slot. */
|
|
437
|
+
startRealtimeOccupant(entry) {
|
|
438
|
+
this.clearAdmission(entry);
|
|
439
|
+
this.realtimeOccupants.add(entry);
|
|
440
|
+
this.handoffAdmitted = true;
|
|
441
|
+
let running;
|
|
442
|
+
try {
|
|
443
|
+
running = this.execute(entry);
|
|
444
|
+
}
|
|
445
|
+
catch (error) {
|
|
446
|
+
this.realtimeOccupants.delete(entry);
|
|
447
|
+
if (this.handoffBacklog === entry)
|
|
448
|
+
this.handoffBacklog = null;
|
|
449
|
+
entry.reject(error);
|
|
450
|
+
this.notifyOccupantChange();
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
void running.then((outcome) => {
|
|
454
|
+
this.realtimeOccupants.delete(entry);
|
|
455
|
+
if (this.handoffBacklog === entry)
|
|
456
|
+
this.handoffBacklog = null;
|
|
457
|
+
entry.resolve(outcome);
|
|
458
|
+
this.notifyOccupantChange();
|
|
459
|
+
}, (error) => {
|
|
460
|
+
this.realtimeOccupants.delete(entry);
|
|
461
|
+
if (this.handoffBacklog === entry)
|
|
462
|
+
this.handoffBacklog = null;
|
|
463
|
+
entry.reject(error);
|
|
464
|
+
this.notifyOccupantChange();
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
/** Items that showed up while a transfer was already running take a free slot. */
|
|
468
|
+
promoteOccupancyArrivals() {
|
|
469
|
+
if (!this._handoffActive)
|
|
470
|
+
return;
|
|
471
|
+
while (this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
|
|
472
|
+
const index = this.pending.findIndex((pending) => pending.workClass === "realtime" && pending.occupancyArrival === true);
|
|
473
|
+
if (index < 0)
|
|
474
|
+
return;
|
|
475
|
+
const arrival = this.pending.splice(index, 1)[0];
|
|
476
|
+
this.startRealtimeOccupant(arrival);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* Pull at most one already-queued realtime item while the admission window
|
|
481
|
+
* is open and a worker slot is free. Further backlog waits for that
|
|
482
|
+
* execute() so pre-queued hand-offs stay one durable writer at a time.
|
|
483
|
+
*/
|
|
484
|
+
admitHandoffBacklog() {
|
|
485
|
+
if (!this._handoffActive || this.handoffBacklog !== null)
|
|
486
|
+
return;
|
|
487
|
+
if (!this.withinHandoffAdmission())
|
|
488
|
+
return;
|
|
489
|
+
if (this.realtimeOccupants.size >= REALTIME_WORKER_OCCUPANCY_CAP)
|
|
490
|
+
return;
|
|
491
|
+
const realtimeIndex = this.nextRealtimeIndex();
|
|
492
|
+
if (realtimeIndex < 0)
|
|
493
|
+
return;
|
|
494
|
+
const pending = this.pending[realtimeIndex];
|
|
495
|
+
if (pending?.occupancyArrival)
|
|
496
|
+
return;
|
|
497
|
+
const realtime = this.pending.splice(realtimeIndex, 1)[0];
|
|
498
|
+
this.handoffBacklog = realtime;
|
|
499
|
+
this.startRealtimeOccupant(realtime);
|
|
500
|
+
}
|
|
373
501
|
async drainRealtime(entry) {
|
|
374
502
|
if (this.stopped ||
|
|
375
503
|
entry.workClass !== "chore" ||
|
|
@@ -377,46 +505,48 @@ export class CooperativePassScheduler {
|
|
|
377
505
|
!this.pending.some((pending) => pending.workClass === "realtime")) {
|
|
378
506
|
return false;
|
|
379
507
|
}
|
|
380
|
-
// `entry` is suspended at a caller-declared durable boundary.
|
|
381
|
-
//
|
|
382
|
-
//
|
|
383
|
-
//
|
|
384
|
-
// completion so a 45s upload is not retried every 30s.
|
|
385
|
-
const deferredAt = this.now();
|
|
386
|
-
const burstMs = realtimeDrainBurstMs();
|
|
508
|
+
// `entry` is suspended at a caller-declared durable boundary. The burst
|
|
509
|
+
// cap only stops admitting further backlog. An execute() that already
|
|
510
|
+
// started is never aborted. A realtime item that arrives while one is in
|
|
511
|
+
// flight starts on the free occupancy slot (see REALTIME_WORKER_OCCUPANCY_CAP).
|
|
387
512
|
this.active = null;
|
|
388
513
|
this._handoffActive = true;
|
|
514
|
+
this.handoffDeferredAt = this.now();
|
|
515
|
+
this.handoffAdmitted = false;
|
|
516
|
+
this.handoffBacklog = null;
|
|
389
517
|
this.options.caps?.suspendForHandoff();
|
|
390
|
-
let handedOff = false;
|
|
391
518
|
try {
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
if (
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
handedOff = true;
|
|
399
|
-
this.active = realtime;
|
|
400
|
-
try {
|
|
401
|
-
const outcome = await this.execute(realtime);
|
|
402
|
-
realtime.resolve(outcome);
|
|
403
|
-
}
|
|
404
|
-
catch (error) {
|
|
405
|
-
realtime.reject(error);
|
|
406
|
-
}
|
|
407
|
-
finally {
|
|
408
|
-
if (this.active === realtime)
|
|
409
|
-
this.active = null;
|
|
519
|
+
this.admitHandoffBacklog();
|
|
520
|
+
while (!this.stopped && (this.realtimeOccupants.size > 0 || this.withinHandoffAdmission())) {
|
|
521
|
+
if (this.realtimeOccupants.size === 0) {
|
|
522
|
+
this.admitHandoffBacklog();
|
|
523
|
+
if (this.realtimeOccupants.size === 0)
|
|
524
|
+
break;
|
|
410
525
|
}
|
|
526
|
+
const seen = this.occupantEpoch;
|
|
527
|
+
if (this.realtimeOccupants.size === 0)
|
|
528
|
+
break;
|
|
529
|
+
await this.waitOccupantChange(seen);
|
|
530
|
+
this.promoteOccupancyArrivals();
|
|
531
|
+
this.admitHandoffBacklog();
|
|
532
|
+
}
|
|
533
|
+
while (!this.stopped && this.realtimeOccupants.size > 0) {
|
|
534
|
+
const seen = this.occupantEpoch;
|
|
535
|
+
if (this.realtimeOccupants.size === 0)
|
|
536
|
+
break;
|
|
537
|
+
await this.waitOccupantChange(seen);
|
|
538
|
+
this.promoteOccupancyArrivals();
|
|
411
539
|
}
|
|
412
540
|
}
|
|
413
541
|
finally {
|
|
414
542
|
this._handoffActive = false;
|
|
543
|
+
this.handoffDeferredAt = null;
|
|
544
|
+
this.handoffBacklog = null;
|
|
415
545
|
if (!this.stopped)
|
|
416
546
|
this.active = entry;
|
|
417
547
|
this.options.caps?.resumeAfterHandoff();
|
|
418
548
|
}
|
|
419
|
-
return
|
|
549
|
+
return this.handoffAdmitted;
|
|
420
550
|
}
|
|
421
551
|
async checkpoint(entry) {
|
|
422
552
|
if (this.stopped || entry.workClass !== "chore" || this.active !== entry) {
|
|
@@ -1112,7 +1242,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1112
1242
|
refresh.unref?.();
|
|
1113
1243
|
}
|
|
1114
1244
|
const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, { caps: fullPassCaps });
|
|
1115
|
-
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint) => {
|
|
1245
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry) => {
|
|
1116
1246
|
const reportedAppliedUpserts = new Set();
|
|
1117
1247
|
const recordDurableAppliedUpserts = (upserts) => {
|
|
1118
1248
|
for (const applied of upserts) {
|
|
@@ -1161,6 +1291,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1161
1291
|
processOwnsOperationLock) {
|
|
1162
1292
|
acquireOperationLockUser();
|
|
1163
1293
|
try {
|
|
1294
|
+
telemetry?.mark("network");
|
|
1164
1295
|
prepare?.();
|
|
1165
1296
|
const outcome = await invokePass();
|
|
1166
1297
|
recordAppliedUpserts(outcome);
|
|
@@ -1175,6 +1306,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1175
1306
|
coordinatorWholeMutationChores === 0) {
|
|
1176
1307
|
acquireOperationLockUser();
|
|
1177
1308
|
try {
|
|
1309
|
+
telemetry?.mark("network");
|
|
1178
1310
|
prepare?.();
|
|
1179
1311
|
const outcome = await invokePass();
|
|
1180
1312
|
recordAppliedUpserts(outcome);
|
|
@@ -1191,6 +1323,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1191
1323
|
deferToWaiters: true,
|
|
1192
1324
|
});
|
|
1193
1325
|
try {
|
|
1326
|
+
telemetry?.mark("network");
|
|
1194
1327
|
if (lane === "slow") {
|
|
1195
1328
|
processOwnsOperationLock = true;
|
|
1196
1329
|
acquireOperationLockUser();
|
|
@@ -1224,6 +1357,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1224
1357
|
}
|
|
1225
1358
|
try {
|
|
1226
1359
|
return await withOperationLock(hqRoot, "sync", async () => {
|
|
1360
|
+
telemetry?.mark("network");
|
|
1227
1361
|
if (lane === "slow") {
|
|
1228
1362
|
processOwnsOperationLock = true;
|
|
1229
1363
|
acquireOperationLockUser();
|
|
@@ -1366,7 +1500,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1366
1500
|
// chore ownership. Realtime normally waits for a chore hand-off; the
|
|
1367
1501
|
// watcher fast path explicitly opts into its established scoped exception.
|
|
1368
1502
|
passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap);
|
|
1369
|
-
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false
|
|
1503
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry) => {
|
|
1504
|
+
telemetry?.mark("operationGate");
|
|
1505
|
+
return runGuardedTask((context) => {
|
|
1506
|
+
telemetry?.mark("appendLock");
|
|
1507
|
+
telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
|
|
1508
|
+
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry);
|
|
1509
|
+
}, lane, workClass, allowFastRealtimeOvertake, signal, resourceCap);
|
|
1510
|
+
};
|
|
1370
1511
|
/*
|
|
1371
1512
|
* Area storage and the parallel coordinator default to AUTO. AUTO turns
|
|
1372
1513
|
* them on after durable personal-vault cutover; explicit off is the kill
|
|
@@ -1441,15 +1582,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1441
1582
|
coordinatorRefusedPaths: [],
|
|
1442
1583
|
};
|
|
1443
1584
|
}
|
|
1585
|
+
telemetry?.mark("worker");
|
|
1444
1586
|
const execution = {
|
|
1445
1587
|
run: async (slice) => {
|
|
1446
|
-
telemetry?.mark("worker");
|
|
1447
1588
|
telemetry?.mark("conflict");
|
|
1448
|
-
telemetry
|
|
1449
|
-
telemetry?.mark("appendLock");
|
|
1450
|
-
telemetry?.mark("network");
|
|
1589
|
+
const result = await task(slice, telemetry);
|
|
1451
1590
|
telemetry?.mark("commit");
|
|
1452
|
-
return
|
|
1591
|
+
return result;
|
|
1453
1592
|
},
|
|
1454
1593
|
wholeMutation: options.wholeMutation ?? false,
|
|
1455
1594
|
};
|
|
@@ -2479,13 +2618,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2479
2618
|
? "fast"
|
|
2480
2619
|
: "slow";
|
|
2481
2620
|
const execution = parallelCoordinatorEnabled
|
|
2482
|
-
? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice) => {
|
|
2621
|
+
? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry) => {
|
|
2483
2622
|
const admittedPaths = slice.paths.map((path) => path.canonicalPath);
|
|
2484
2623
|
const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
|
|
2485
2624
|
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
2486
2625
|
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
2487
2626
|
? () => prepareWatcherChanges(admittedChanges)
|
|
2488
|
-
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal);
|
|
2627
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry);
|
|
2489
2628
|
})
|
|
2490
2629
|
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
2491
2630
|
? () => prepareWatcherChanges(group.changes)
|
|
@@ -2565,7 +2704,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2565
2704
|
? "fast"
|
|
2566
2705
|
: "slow";
|
|
2567
2706
|
const execution = parallelCoordinatorEnabled
|
|
2568
|
-
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", false, slice.signal))
|
|
2707
|
+
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", false, slice.signal, false, telemetry))
|
|
2569
2708
|
: { result: await runGuarded(pullArgv, undefined, lane, "realtime"), accepted: true };
|
|
2570
2709
|
const exit = scopedRouteDrainExit(execution);
|
|
2571
2710
|
if (exit !== 0 && worstExit === 0)
|
|
@@ -2919,7 +3058,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2919
3058
|
// Submit every route in this lane before awaiting any result. In
|
|
2920
3059
|
// particular, a later route's fast receipt must never wait for an
|
|
2921
3060
|
// earlier route's slow pull to finish.
|
|
2922
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime",
|
|
3061
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry))));
|
|
2923
3062
|
for (const execution of completed) {
|
|
2924
3063
|
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
2925
3064
|
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|