@indigoai-us/hq-cloud 6.16.72 → 6.16.74-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,6 +24,14 @@ export declare const REALTIME_ADMISSION_DEADLINE_MS: number;
24
24
  * that value only promotes a deferred chore to the front of the queue.
25
25
  */
26
26
  export declare const REALTIME_DRAIN_BURST_MS: number;
27
+ /**
28
+ * How many realtime executes may run during one chore hand-off.
29
+ * HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
30
+ * workers. One long transfer keeps its worker; a realtime item that arrives
31
+ * while that execute() is in flight starts on the other slot instead of
32
+ * waiting for the whole batch. In-flight work is never aborted.
33
+ */
34
+ export declare const REALTIME_WORKER_OCCUPANCY_CAP = 2;
27
35
  export declare function setRealtimeAdmissionDeadlineMsForTesting(ms: number | undefined): void;
28
36
  export declare function setRealtimeDrainBurstMsForTesting(ms: number | undefined): void;
29
37
  export declare class RealtimeAdmissionTimeoutError extends Error {
@@ -56,11 +64,13 @@ export interface GuardedPassContext {
56
64
  signal: AbortSignal;
57
65
  }
58
66
  /**
59
- * One durable-state owner at a time for a full chore. Realtime work never runs
60
- * beside that chore: it explicitly awaits `checkpoint()` after a completed
61
- * durable unit, then resumes once the hand-off is finished. A separately
62
- * tracked, explicitly opted-in watcher fast pass may use the pre-existing
63
- * scoped fast-over-slow exception, but never while a chore owns durable state.
67
+ * One durable-state owner at a time for a full chore. Realtime work runs only
68
+ * while that chore is suspended at `checkpoint()`. Up to
69
+ * {@link REALTIME_WORKER_OCCUPANCY_CAP} realtime executes may occupy that
70
+ * hand-off; an arrival does not wait out a transfer that is already running.
71
+ * A separately tracked, explicitly opted-in watcher fast pass may use the
72
+ * pre-existing scoped fast-over-slow exception, but never while a chore owns
73
+ * durable state.
64
74
  */
65
75
  export declare class CooperativePassScheduler<T> {
66
76
  private readonly now;
@@ -68,6 +78,13 @@ export declare class CooperativePassScheduler<T> {
68
78
  private readonly options;
69
79
  private readonly pending;
70
80
  private active;
81
+ /** Realtime executes started by the current hand-off. Not the chore slot. */
82
+ private readonly realtimeOccupants;
83
+ private handoffBacklog;
84
+ private handoffDeferredAt;
85
+ private handoffAdmitted;
86
+ private occupantEpoch;
87
+ private occupantWaiters;
71
88
  private activeFastRealtime;
72
89
  private stopped;
73
90
  private _handoffActive;
@@ -77,6 +94,8 @@ export declare class CooperativePassScheduler<T> {
77
94
  get handoffActive(): boolean;
78
95
  get hasActiveChore(): boolean;
79
96
  get hasActiveSlowRealtime(): boolean;
97
+ /** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
98
+ get activeRealtimeWorkerCount(): number;
80
99
  pendingLaneCount(lane: "fast" | "slow"): number;
81
100
  submit(task: (context: GuardedPassContext) => Promise<T>, workClass: GuardedWorkClass, lane?: "fast" | "slow", allowFastRealtimeOvertake?: boolean, signal?: AbortSignal, resourceCap?: boolean): Promise<T>;
82
101
  stop(): void;
@@ -97,6 +116,19 @@ export declare class CooperativePassScheduler<T> {
97
116
  private startFastRealtime;
98
117
  private completeFastRealtime;
99
118
  private execute;
119
+ private withinHandoffAdmission;
120
+ private waitOccupantChange;
121
+ private notifyOccupantChange;
122
+ /** Run one realtime execute without aborting it and without taking the chore slot. */
123
+ private startRealtimeOccupant;
124
+ /** Items that showed up while a transfer was already running take a free slot. */
125
+ private promoteOccupancyArrivals;
126
+ /**
127
+ * Pull at most one already-queued realtime item while the admission window
128
+ * is open and a worker slot is free. Further backlog waits for that
129
+ * execute() so pre-queued hand-offs stay one durable writer at a time.
130
+ */
131
+ private admitHandoffBacklog;
100
132
  private drainRealtime;
101
133
  private checkpoint;
102
134
  }
@@ -1 +1 @@
1
- {"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAG1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AAKtE,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAiBD;;;;;;GAMG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAWnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAZ1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IAInD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,GAClC,OAAO,CAAC,CAAC,CAAC;IA2Db,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAeb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;YAQD,aAAa;YA8Cb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AACjE,eAAO,MAAM,yBAAyB,EAAE,MAAM,GAAG,SAAuB,CAAC;AAEzE,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,GAAG,SAAS,CAOpB;AAiBD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAuyGjB"}
1
+ {"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAG1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAmBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,GAClC,OAAO,CAAC,CAAC,CAAC;IA8Eb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,kFAAkF;IAClF,OAAO,CAAC,wBAAwB;IAYhC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AACjE,eAAO,MAAM,yBAAyB,EAAE,MAAM,GAAG,SAAuB,CAAC;AAEzE,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,GAAG,SAAS,CAOpB;AAiBD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAwzGjB"}
@@ -49,6 +49,14 @@ export const REALTIME_ADMISSION_DEADLINE_MS = 2 * DEFAULT_REALTIME_AUTHORIZATION
49
49
  * that value only promotes a deferred chore to the front of the queue.
50
50
  */
51
51
  export const REALTIME_DRAIN_BURST_MS = REALTIME_ADMISSION_DEADLINE_MS;
52
+ /**
53
+ * How many realtime executes may run during one chore hand-off.
54
+ * HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
55
+ * workers. One long transfer keeps its worker; a realtime item that arrives
56
+ * while that execute() is in flight starts on the other slot instead of
57
+ * waiting for the whole batch. In-flight work is never aborted.
58
+ */
59
+ export const REALTIME_WORKER_OCCUPANCY_CAP = 2;
52
60
  let realtimeAdmissionDeadlineMsForTest;
53
61
  let realtimeDrainBurstMsForTest;
54
62
  export function setRealtimeAdmissionDeadlineMsForTesting(ms) {
@@ -118,11 +126,13 @@ export function realtimeTelemetryReason(paths) {
118
126
  return undefined;
119
127
  }
120
128
  /**
121
- * One durable-state owner at a time for a full chore. Realtime work never runs
122
- * beside that chore: it explicitly awaits `checkpoint()` after a completed
123
- * durable unit, then resumes once the hand-off is finished. A separately
124
- * tracked, explicitly opted-in watcher fast pass may use the pre-existing
125
- * scoped fast-over-slow exception, but never while a chore owns durable state.
129
+ * One durable-state owner at a time for a full chore. Realtime work runs only
130
+ * while that chore is suspended at `checkpoint()`. Up to
131
+ * {@link REALTIME_WORKER_OCCUPANCY_CAP} realtime executes may occupy that
132
+ * hand-off; an arrival does not wait out a transfer that is already running.
133
+ * A separately tracked, explicitly opted-in watcher fast pass may use the
134
+ * pre-existing scoped fast-over-slow exception, but never while a chore owns
135
+ * durable state.
126
136
  */
127
137
  export class CooperativePassScheduler {
128
138
  now;
@@ -130,6 +140,13 @@ export class CooperativePassScheduler {
130
140
  options;
131
141
  pending = [];
132
142
  active = null;
143
+ /** Realtime executes started by the current hand-off. Not the chore slot. */
144
+ realtimeOccupants = new Set();
145
+ handoffBacklog = null;
146
+ handoffDeferredAt = null;
147
+ handoffAdmitted = false;
148
+ occupantEpoch = 0;
149
+ occupantWaiters = [];
133
150
  // Scoped fast realtime work has the established in-process lock exception
134
151
  // while a scoped slow realtime upload is in flight. Keep that slot separate
135
152
  // from `active`: a chore still has exactly one durable-state owner.
@@ -150,6 +167,12 @@ export class CooperativePassScheduler {
150
167
  get hasActiveSlowRealtime() {
151
168
  return this.active?.workClass === "realtime" && this.active.lane === "slow";
152
169
  }
170
+ /** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
171
+ get activeRealtimeWorkerCount() {
172
+ return ((this.active?.workClass === "realtime" ? 1 : 0) +
173
+ (this.activeFastRealtime === null ? 0 : 1) +
174
+ this.realtimeOccupants.size);
175
+ }
153
176
  pendingLaneCount(lane) {
154
177
  return this.pending.filter((entry) => entry.lane === lane).length;
155
178
  }
@@ -180,6 +203,20 @@ export class CooperativePassScheduler {
180
203
  if (this.canStartFastRealtime(directEntry)) {
181
204
  return this.startFastRealtime(directEntry);
182
205
  }
206
+ if (workClass === "realtime" &&
207
+ this._handoffActive &&
208
+ this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
209
+ return new Promise((resolve, reject) => {
210
+ const entry = {
211
+ ...directEntry,
212
+ enqueuedAt: this.now(),
213
+ resolve,
214
+ reject,
215
+ occupancyArrival: true,
216
+ };
217
+ this.startRealtimeOccupant(entry);
218
+ });
219
+ }
183
220
  if (this.active === null &&
184
221
  this.activeFastRealtime === null &&
185
222
  this.pending.length === 0) {
@@ -206,6 +243,9 @@ export class CooperativePassScheduler {
206
243
  signal.addEventListener("abort", onAbort, { once: true });
207
244
  entry.abort = () => signal.removeEventListener("abort", onAbort);
208
245
  }
246
+ if (workClass === "realtime" && this._handoffActive) {
247
+ entry.occupancyArrival = true;
248
+ }
209
249
  this.enqueuePending(entry);
210
250
  this.drain();
211
251
  });
@@ -258,6 +298,10 @@ export class CooperativePassScheduler {
258
298
  drain() {
259
299
  if (this.stopped || this.pending.length === 0)
260
300
  return;
301
+ if (this._handoffActive) {
302
+ this.promoteOccupancyArrivals();
303
+ return;
304
+ }
261
305
  if (this.active !== null) {
262
306
  const fastRealtime = this.nextFastRealtime();
263
307
  if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
@@ -370,6 +414,90 @@ export class CooperativePassScheduler {
370
414
  signal: entry.controller.signal,
371
415
  });
372
416
  }
417
+ withinHandoffAdmission() {
418
+ if (this.handoffDeferredAt === null)
419
+ return false;
420
+ return this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
421
+ }
422
+ waitOccupantChange(seenEpoch) {
423
+ if (this.occupantEpoch !== seenEpoch)
424
+ return Promise.resolve();
425
+ return new Promise((resolve) => {
426
+ this.occupantWaiters.push(resolve);
427
+ });
428
+ }
429
+ notifyOccupantChange() {
430
+ this.occupantEpoch += 1;
431
+ const waiters = this.occupantWaiters;
432
+ this.occupantWaiters = [];
433
+ for (const resolve of waiters)
434
+ resolve();
435
+ }
436
+ /** Run one realtime execute without aborting it and without taking the chore slot. */
437
+ startRealtimeOccupant(entry) {
438
+ this.clearAdmission(entry);
439
+ this.realtimeOccupants.add(entry);
440
+ this.handoffAdmitted = true;
441
+ let running;
442
+ try {
443
+ running = this.execute(entry);
444
+ }
445
+ catch (error) {
446
+ this.realtimeOccupants.delete(entry);
447
+ if (this.handoffBacklog === entry)
448
+ this.handoffBacklog = null;
449
+ entry.reject(error);
450
+ this.notifyOccupantChange();
451
+ return;
452
+ }
453
+ void running.then((outcome) => {
454
+ this.realtimeOccupants.delete(entry);
455
+ if (this.handoffBacklog === entry)
456
+ this.handoffBacklog = null;
457
+ entry.resolve(outcome);
458
+ this.notifyOccupantChange();
459
+ }, (error) => {
460
+ this.realtimeOccupants.delete(entry);
461
+ if (this.handoffBacklog === entry)
462
+ this.handoffBacklog = null;
463
+ entry.reject(error);
464
+ this.notifyOccupantChange();
465
+ });
466
+ }
467
+ /** Items that showed up while a transfer was already running take a free slot. */
468
+ promoteOccupancyArrivals() {
469
+ if (!this._handoffActive)
470
+ return;
471
+ while (this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
472
+ const index = this.pending.findIndex((pending) => pending.workClass === "realtime" && pending.occupancyArrival === true);
473
+ if (index < 0)
474
+ return;
475
+ const arrival = this.pending.splice(index, 1)[0];
476
+ this.startRealtimeOccupant(arrival);
477
+ }
478
+ }
479
+ /**
480
+ * Pull at most one already-queued realtime item while the admission window
481
+ * is open and a worker slot is free. Further backlog waits for that
482
+ * execute() so pre-queued hand-offs stay one durable writer at a time.
483
+ */
484
+ admitHandoffBacklog() {
485
+ if (!this._handoffActive || this.handoffBacklog !== null)
486
+ return;
487
+ if (!this.withinHandoffAdmission())
488
+ return;
489
+ if (this.realtimeOccupants.size >= REALTIME_WORKER_OCCUPANCY_CAP)
490
+ return;
491
+ const realtimeIndex = this.nextRealtimeIndex();
492
+ if (realtimeIndex < 0)
493
+ return;
494
+ const pending = this.pending[realtimeIndex];
495
+ if (pending?.occupancyArrival)
496
+ return;
497
+ const realtime = this.pending.splice(realtimeIndex, 1)[0];
498
+ this.handoffBacklog = realtime;
499
+ this.startRealtimeOccupant(realtime);
500
+ }
373
501
  async drainRealtime(entry) {
374
502
  if (this.stopped ||
375
503
  entry.workClass !== "chore" ||
@@ -377,46 +505,48 @@ export class CooperativePassScheduler {
377
505
  !this.pending.some((pending) => pending.workClass === "realtime")) {
378
506
  return false;
379
507
  }
380
- // `entry` is suspended at a caller-declared durable boundary. It remains
381
- // the sole full-pass continuation and will resume after this bounded burst.
382
- // The burst cap only stops *admitting* further realtime items; an
383
- // execute() that already started (presign + PUT + journal) runs to
384
- // completion so a 45s upload is not retried every 30s.
385
- const deferredAt = this.now();
386
- const burstMs = realtimeDrainBurstMs();
508
+ // `entry` is suspended at a caller-declared durable boundary. The burst
509
+ // cap only stops admitting further backlog. An execute() that already
510
+ // started is never aborted. A realtime item that arrives while one is in
511
+ // flight starts on the free occupancy slot (see REALTIME_WORKER_OCCUPANCY_CAP).
387
512
  this.active = null;
388
513
  this._handoffActive = true;
514
+ this.handoffDeferredAt = this.now();
515
+ this.handoffAdmitted = false;
516
+ this.handoffBacklog = null;
389
517
  this.options.caps?.suspendForHandoff();
390
- let handedOff = false;
391
518
  try {
392
- while (!this.stopped && this.now() - deferredAt < burstMs) {
393
- const realtimeIndex = this.nextRealtimeIndex();
394
- if (realtimeIndex < 0)
395
- break;
396
- const realtime = this.pending.splice(realtimeIndex, 1)[0];
397
- this.clearAdmission(realtime);
398
- handedOff = true;
399
- this.active = realtime;
400
- try {
401
- const outcome = await this.execute(realtime);
402
- realtime.resolve(outcome);
403
- }
404
- catch (error) {
405
- realtime.reject(error);
406
- }
407
- finally {
408
- if (this.active === realtime)
409
- this.active = null;
519
+ this.admitHandoffBacklog();
520
+ while (!this.stopped && (this.realtimeOccupants.size > 0 || this.withinHandoffAdmission())) {
521
+ if (this.realtimeOccupants.size === 0) {
522
+ this.admitHandoffBacklog();
523
+ if (this.realtimeOccupants.size === 0)
524
+ break;
410
525
  }
526
+ const seen = this.occupantEpoch;
527
+ if (this.realtimeOccupants.size === 0)
528
+ break;
529
+ await this.waitOccupantChange(seen);
530
+ this.promoteOccupancyArrivals();
531
+ this.admitHandoffBacklog();
532
+ }
533
+ while (!this.stopped && this.realtimeOccupants.size > 0) {
534
+ const seen = this.occupantEpoch;
535
+ if (this.realtimeOccupants.size === 0)
536
+ break;
537
+ await this.waitOccupantChange(seen);
538
+ this.promoteOccupancyArrivals();
411
539
  }
412
540
  }
413
541
  finally {
414
542
  this._handoffActive = false;
543
+ this.handoffDeferredAt = null;
544
+ this.handoffBacklog = null;
415
545
  if (!this.stopped)
416
546
  this.active = entry;
417
547
  this.options.caps?.resumeAfterHandoff();
418
548
  }
419
- return handedOff;
549
+ return this.handoffAdmitted;
420
550
  }
421
551
  async checkpoint(entry) {
422
552
  if (this.stopped || entry.workClass !== "chore" || this.active !== entry) {
@@ -1112,7 +1242,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1112
1242
  refresh.unref?.();
1113
1243
  }
1114
1244
  const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, { caps: fullPassCaps });
1115
- const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint) => {
1245
+ const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry) => {
1116
1246
  const reportedAppliedUpserts = new Set();
1117
1247
  const recordDurableAppliedUpserts = (upserts) => {
1118
1248
  for (const applied of upserts) {
@@ -1161,6 +1291,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1161
1291
  processOwnsOperationLock) {
1162
1292
  acquireOperationLockUser();
1163
1293
  try {
1294
+ telemetry?.mark("network");
1164
1295
  prepare?.();
1165
1296
  const outcome = await invokePass();
1166
1297
  recordAppliedUpserts(outcome);
@@ -1175,6 +1306,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1175
1306
  coordinatorWholeMutationChores === 0) {
1176
1307
  acquireOperationLockUser();
1177
1308
  try {
1309
+ telemetry?.mark("network");
1178
1310
  prepare?.();
1179
1311
  const outcome = await invokePass();
1180
1312
  recordAppliedUpserts(outcome);
@@ -1191,6 +1323,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1191
1323
  deferToWaiters: true,
1192
1324
  });
1193
1325
  try {
1326
+ telemetry?.mark("network");
1194
1327
  if (lane === "slow") {
1195
1328
  processOwnsOperationLock = true;
1196
1329
  acquireOperationLockUser();
@@ -1224,6 +1357,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1224
1357
  }
1225
1358
  try {
1226
1359
  return await withOperationLock(hqRoot, "sync", async () => {
1360
+ telemetry?.mark("network");
1227
1361
  if (lane === "slow") {
1228
1362
  processOwnsOperationLock = true;
1229
1363
  acquireOperationLockUser();
@@ -1366,7 +1500,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1366
1500
  // chore ownership. Realtime normally waits for a chore hand-off; the
1367
1501
  // watcher fast path explicitly opts into its established scoped exception.
1368
1502
  passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap);
1369
- const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false) => runGuardedTask((context) => runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined), lane, workClass, allowFastRealtimeOvertake, signal, resourceCap);
1503
+ const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry) => {
1504
+ telemetry?.mark("operationGate");
1505
+ return runGuardedTask((context) => {
1506
+ telemetry?.mark("appendLock");
1507
+ telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
1508
+ return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry);
1509
+ }, lane, workClass, allowFastRealtimeOvertake, signal, resourceCap);
1510
+ };
1370
1511
  /*
1371
1512
  * Area storage and the parallel coordinator default to AUTO. AUTO turns
1372
1513
  * them on after durable personal-vault cutover; explicit off is the kill
@@ -1441,15 +1582,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1441
1582
  coordinatorRefusedPaths: [],
1442
1583
  };
1443
1584
  }
1585
+ telemetry?.mark("worker");
1444
1586
  const execution = {
1445
1587
  run: async (slice) => {
1446
- telemetry?.mark("worker");
1447
1588
  telemetry?.mark("conflict");
1448
- telemetry?.mark("operationGate");
1449
- telemetry?.mark("appendLock");
1450
- telemetry?.mark("network");
1589
+ const result = await task(slice, telemetry);
1451
1590
  telemetry?.mark("commit");
1452
- return task(slice);
1591
+ return result;
1453
1592
  },
1454
1593
  wholeMutation: options.wholeMutation ?? false,
1455
1594
  };
@@ -2479,13 +2618,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2479
2618
  ? "fast"
2480
2619
  : "slow";
2481
2620
  const execution = parallelCoordinatorEnabled
2482
- ? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice) => {
2621
+ ? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry) => {
2483
2622
  const admittedPaths = slice.paths.map((path) => path.canonicalPath);
2484
2623
  const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
2485
2624
  const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
2486
2625
  return runGuarded(scopedArgv, admittedChanges.size > 0
2487
2626
  ? () => prepareWatcherChanges(admittedChanges)
2488
- : undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal);
2627
+ : undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry);
2489
2628
  })
2490
2629
  : { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
2491
2630
  ? () => prepareWatcherChanges(group.changes)
@@ -2565,7 +2704,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2565
2704
  ? "fast"
2566
2705
  : "slow";
2567
2706
  const execution = parallelCoordinatorEnabled
2568
- ? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", false, slice.signal))
2707
+ ? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", false, slice.signal, false, telemetry))
2569
2708
  : { result: await runGuarded(pullArgv, undefined, lane, "realtime"), accepted: true };
2570
2709
  const exit = scopedRouteDrainExit(execution);
2571
2710
  if (exit !== 0 && worstExit === 0)
@@ -2919,7 +3058,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2919
3058
  // Submit every route in this lane before awaiting any result. In
2920
3059
  // particular, a later route's fast receipt must never wait for an
2921
3060
  // earlier route's slow pull to finish.
2922
- const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", false, slice.signal))));
3061
+ const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry))));
2923
3062
  for (const execution of completed) {
2924
3063
  if (!execution.accepted || passExitCode(execution.result) !== 0) {
2925
3064
  throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);