@indigoai-us/hq-cloud 6.18.43-alpha.2 → 6.18.44-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bandwidth.d.ts +2 -3
- package/dist/bandwidth.d.ts.map +1 -1
- package/dist/bandwidth.js +16 -21
- package/dist/bandwidth.js.map +1 -1
- package/dist/bandwidth.test.js +0 -23
- package/dist/bandwidth.test.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.d.ts +0 -3
- package/dist/bin/sync-runner-heap-telemetry.d.ts.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.js +0 -3
- package/dist/bin/sync-runner-heap-telemetry.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +41 -25
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +142 -96
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +1 -3
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +0 -13
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +1 -2
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +601 -289
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/index.d.ts +3 -7
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +1 -5
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +6 -1
- package/dist/cli/sync.js.map +1 -1
- package/dist/file-mode-portability.d.ts +33 -0
- package/dist/file-mode-portability.d.ts.map +1 -0
- package/dist/file-mode-portability.js +41 -0
- package/dist/file-mode-portability.js.map +1 -0
- package/dist/journal-snapshot-memory.test.js +22 -7
- package/dist/journal-snapshot-memory.test.js.map +1 -1
- package/dist/journal-snapshot-rows.d.ts +0 -2
- package/dist/journal-snapshot-rows.d.ts.map +1 -1
- package/dist/journal-snapshot-rows.js +0 -4
- package/dist/journal-snapshot-rows.js.map +1 -1
- package/dist/private-folder-transport.d.ts.map +1 -1
- package/dist/private-folder-transport.js +5 -1
- package/dist/private-folder-transport.js.map +1 -1
- package/dist/s3.d.ts +7 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +82 -4
- package/dist/s3.js.map +1 -1
- package/dist/s3.windows-portability.test.d.ts +12 -0
- package/dist/s3.windows-portability.test.d.ts.map +1 -0
- package/dist/s3.windows-portability.test.js +223 -0
- package/dist/s3.windows-portability.test.js.map +1 -0
- package/dist/script-eol.d.ts +34 -0
- package/dist/script-eol.d.ts.map +1 -0
- package/dist/script-eol.js +60 -0
- package/dist/script-eol.js.map +1 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +4 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +44 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.d.ts +0 -7
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
- package/dist/sync/parallel-sync-coordinator.js +0 -22
- package/dist/sync/parallel-sync-coordinator.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +0 -31
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +11 -6
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +56 -1
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +2 -24
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +3 -47
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +8 -47
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/package.json +1 -1
- package/dist/daemon.d.ts +0 -12
- package/dist/daemon.d.ts.map +0 -1
- package/dist/daemon.js +0 -90
- package/dist/daemon.js.map +0 -1
|
@@ -3,7 +3,7 @@ import { readJournal } from "../journal.js";
|
|
|
3
3
|
import { type FeatureSwitch } from "../sync/feature-flags.js";
|
|
4
4
|
import { HeapCensus } from "./sync-runner-heap-telemetry.js";
|
|
5
5
|
import { FullPassCapController } from "../sync/full-pass-caps.js";
|
|
6
|
-
import { type
|
|
6
|
+
import { type RealtimeTelemetryReason } from "../sync/realtime-telemetry.js";
|
|
7
7
|
import { type PathWorkOutcome } from "../sync/sync-work.js";
|
|
8
8
|
import type { TelemetryClaims } from "../telemetry-events.js";
|
|
9
9
|
import type { JournalEntry } from "../types.js";
|
|
@@ -38,6 +38,20 @@ export declare const REALTIME_WORKER_OCCUPANCY_CAP = 2;
|
|
|
38
38
|
export declare const REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES: number;
|
|
39
39
|
export declare function setRealtimeAdmissionDeadlineMsForTesting(ms: number | undefined): void;
|
|
40
40
|
export declare function setRealtimeDrainBurstMsForTesting(ms: number | undefined): void;
|
|
41
|
+
/**
|
|
42
|
+
* Retry delay for a watcher push or pull refused at the admission deadline.
|
|
43
|
+
* The refusal is a deferral, not a pass failure, so it does not advance the
|
|
44
|
+
* route's failure backoff. Each retry can wait up to the deadline again,
|
|
45
|
+
* so a long chore costs at most one attempt per deadline plus this delay.
|
|
46
|
+
*/
|
|
47
|
+
export declare const REALTIME_ADMISSION_RETRY_MS = 5000;
|
|
48
|
+
/**
|
|
49
|
+
* In-process result of a scoped drain whose only shortfall was admission
|
|
50
|
+
* deferrals: the paths are restored and a retry is scheduled. Never a
|
|
51
|
+
* process exit code (negative), so it cannot reach the menubar's exit
|
|
52
|
+
* handling, and it is not 0, so it does not count as a completed pass.
|
|
53
|
+
*/
|
|
54
|
+
export declare const SCOPED_DRAIN_DEFERRED = -1;
|
|
41
55
|
export declare class RealtimeAdmissionTimeoutError extends Error {
|
|
42
56
|
readonly retryable = true;
|
|
43
57
|
constructor(deadlineMs: number);
|
|
@@ -79,12 +93,6 @@ export interface GuardedPassContext {
|
|
|
79
93
|
/** Aborted when the caller cancels the slice. */
|
|
80
94
|
signal: AbortSignal;
|
|
81
95
|
}
|
|
82
|
-
/** `choreKind` is the coordinator chore kind of the holding entry; absent for a chore outside the coordinator. */
|
|
83
|
-
export type SchedulerBlocker = {
|
|
84
|
-
workClass: GuardedWorkClass;
|
|
85
|
-
lane: "fast" | "slow";
|
|
86
|
-
choreKind?: string;
|
|
87
|
-
};
|
|
88
96
|
/**
|
|
89
97
|
* One durable-state owner at a time for a full chore. Realtime work runs only
|
|
90
98
|
* while that chore is suspended at `checkpoint()`. Only coordinator-fenced
|
|
@@ -94,12 +102,6 @@ export type SchedulerBlocker = {
|
|
|
94
102
|
* pre-existing scoped fast-over-slow exception, but never while a chore owns
|
|
95
103
|
* durable state.
|
|
96
104
|
*/
|
|
97
|
-
/**
|
|
98
|
-
* Maps what holds a realtime op to its closed telemetry label. A chore kind
|
|
99
|
-
* outside the label set becomes `chore:unscoped`, so a new kind can never
|
|
100
|
-
* make the timing record fail its low-cardinality check.
|
|
101
|
-
*/
|
|
102
|
-
export declare function realtimeBlockerLabel(blocker: SchedulerBlocker): RealtimeBlocker;
|
|
103
105
|
export declare class CooperativePassScheduler<T> {
|
|
104
106
|
private readonly now;
|
|
105
107
|
private readonly starvationCapMs;
|
|
@@ -109,8 +111,6 @@ export declare class CooperativePassScheduler<T> {
|
|
|
109
111
|
/** Realtime executes started by the current hand-off. Not the chore slot. */
|
|
110
112
|
private readonly realtimeOccupants;
|
|
111
113
|
private handoffBacklog;
|
|
112
|
-
/** The chore suspended at checkpoint() while it lends the slot. Diagnostics only. */
|
|
113
|
-
private handoffOwner;
|
|
114
114
|
private handoffDeferredAt;
|
|
115
115
|
private handoffAdmitted;
|
|
116
116
|
private occupantEpoch;
|
|
@@ -136,24 +136,40 @@ export declare class CooperativePassScheduler<T> {
|
|
|
136
136
|
});
|
|
137
137
|
get handoffActive(): boolean;
|
|
138
138
|
get hasActiveChore(): boolean;
|
|
139
|
+
/**
|
|
140
|
+
* A chore owns the slot, or has lent it through a checkpoint or network
|
|
141
|
+
* hand-off and will take it back. Hand-offs only ever start from a chore.
|
|
142
|
+
*/
|
|
143
|
+
private get choreHoldsSlot();
|
|
139
144
|
get hasActiveSlowRealtime(): boolean;
|
|
140
145
|
/** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
|
|
141
146
|
get activeRealtimeWorkerCount(): number;
|
|
142
147
|
pendingLaneCount(lane: "fast" | "slow"): number;
|
|
143
|
-
submit(task: (context: GuardedPassContext) => Promise<T>, workClass: GuardedWorkClass, lane?: "fast" | "slow", allowFastRealtimeOvertake?: boolean, signal?: AbortSignal, resourceCap?: boolean, allowConcurrentHandoff?: boolean, transferDirection?: "push" | "pull", transferSizeClass?: "small" | "large" | "unknown"
|
|
144
|
-
/** Diagnostics only: called once if this task queues, naming what holds the scheduler. */
|
|
145
|
-
onQueued?: (blocker: SchedulerBlocker) => void, choreKind?: string): Promise<T>;
|
|
146
|
-
/**
|
|
147
|
-
* Diagnostics only: the work holding a task that cannot start. The occupant
|
|
148
|
-
* of the task's own lane comes first, then the slot owner, then a hand-off
|
|
149
|
-
* occupant that is what refuses admission, then the chore lending the slot.
|
|
150
|
-
*/
|
|
151
|
-
private blockerFor;
|
|
152
|
-
private blockingOccupant;
|
|
148
|
+
submit(task: (context: GuardedPassContext) => Promise<T>, workClass: GuardedWorkClass, lane?: "fast" | "slow", allowFastRealtimeOvertake?: boolean, signal?: AbortSignal, resourceCap?: boolean, allowConcurrentHandoff?: boolean, transferDirection?: "push" | "pull", transferSizeClass?: "small" | "large" | "unknown"): Promise<T>;
|
|
153
149
|
stop(): void;
|
|
154
150
|
private abortedError;
|
|
155
151
|
/** Realtime stays ahead of a queued full chore so retries are not buried. */
|
|
156
152
|
private enqueuePending;
|
|
153
|
+
/**
|
|
154
|
+
* Bound how long a queued realtime entry sits behind a chore, counted from
|
|
155
|
+
* the first moment a chore holds the slot while it is queued. Time spent
|
|
156
|
+
* behind other realtime work does not count against it. Idempotent while
|
|
157
|
+
* the same chore holds the slot: an entry keeps its first deadline across
|
|
158
|
+
* that chore's hand-offs. Cleared when the chore finishes.
|
|
159
|
+
*/
|
|
160
|
+
private armAdmission;
|
|
161
|
+
/**
|
|
162
|
+
* A chore has taken (or taken back) the slot. Realtime queued while it
|
|
163
|
+
* was lent, or behind other realtime, would otherwise wait for the chore's
|
|
164
|
+
* next checkpoint with no bound.
|
|
165
|
+
*/
|
|
166
|
+
private armQueuedRealtime;
|
|
167
|
+
/**
|
|
168
|
+
* No chore holds the slot any more. Queued realtime now waits only behind
|
|
169
|
+
* other realtime, which does not count against the deadline; the next
|
|
170
|
+
* chore to take the slot arms a fresh one.
|
|
171
|
+
*/
|
|
172
|
+
private disarmQueuedRealtime;
|
|
157
173
|
private clearAdmission;
|
|
158
174
|
private dropPending;
|
|
159
175
|
private expireAdmission;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAmCA,OAAO,EAML,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EAYZ,MAAM,eAAe,CAAC;AAevB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAKzC,OAAO,EAEL,qBAAqB,EAEtB,MAAM,2BAA2B,CAAC;AAOnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAahD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAI1B,+EAA+E;AAC/E,eAAO,MAAM,mCAAmC,OAAQ,CAAC;AACzD,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAC/C,+EAA+E;AAC/E,eAAO,MAAM,qCAAqC,QAA4B,CAAC;AAK/E,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,OAAQ,CAAC;AAEjD;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,KAAK,CAAC;AAExC,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;;;;;GASG;AACH,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,aAAa,EAAE,SAAS,MAAM,EAAE,GAC/B,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAuBrB;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,wBAAgB,oCAAoC,CAClD,SAAS,EAAE,gBAAgB,EAC3B,iBAAiB,EAAE,OAAO,GACzB,OAAO,CAGT;AAED,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAwBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IA+BnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAhC1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;IAC/B,4EAA4E;IAC5E,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,aAAa,CAAqC;IAC1D,OAAO,CAAC,sBAAsB,CAAS;IACvC,OAAO,CAAC,oBAAoB,CAAK;IACjC,OAAO,CAAC,oBAAoB,CAAK;IACjC,OAAO,CAAC,sBAAsB,CAAyB;IACvD,OAAO,CAAC,sBAAsB,CAAiC;IAC/D,OAAO,CAAC,6BAA6B,CAAiD;IACtF,4EAA4E;IAC5E,OAAO,CAAC,8BAA8B,CAAK;IAC3C,OAAO,CAAC,2BAA2B,CAA8B;IACjE,OAAO,CAAC,kCAAkC,CAA6B;gBAGpD,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED;;;OAGG;IACH,OAAO,KAAK,cAAc,GAEzB;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,EACnC,sBAAsB,UAAQ,EAC9B,iBAAiB,CAAC,EAAE,MAAM,GAAG,MAAM,EACnC,iBAAiB,GAAE,OAAO,GAAG,OAAO,GAAG,SAAqB,GAC3D,OAAO,CAAC,CAAC,CAAC;IAoFb,IAAI,IAAI,IAAI;IAaZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB;;;;;;OAMG;IACH,OAAO,CAAC,YAAY;IASpB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAQ5B,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAoBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IA0Bb,OAAO,CAAC,QAAQ;IAMhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAiBf,OAAO,CAAC,qBAAqB;IAO7B,OAAO,CAAC,sBAAsB;IAM9B;;;;OAIG;YACW,uBAAuB;YAgIvB,wBAAwB;IAiCtC,OAAO,CAAC,+BAA+B;IASvC,OAAO,CAAC,kCAAkC;IAU1C,OAAO,CAAC,yBAAyB;IAOjC,OAAO,CAAC,2BAA2B;IAOnC,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IAgC7B,+EAA+E;IAC/E,OAAO,CAAC,wBAAwB;IAmBhC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IAa3B,OAAO,CAAC,wBAAwB;YAQlB,aAAa;YAoDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE,iFAAiF;AACjF,wBAAgB,4BAA4B,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAc9E;AAED;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAoC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,6FAA6F;AAC7F,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAgB3F;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,OAAO,CAInE;AAED,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAEvE;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,OAAO,GAAG,OAAO,CAQvF;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AACjE,eAAO,MAAM,yBAAyB,EAAE,MAAM,GAAG,SAA2B,CAAC;AAE7E,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,GAAG,SAAS,CAOpB;AAiBD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,EACT,gBAAgB,CAAC,EAAE,OAAO,GAAG,UAAU,EACvC,SAAS,CAAC,EAAE,WAAW,EACvB,YAAY,CAAC,EAAE,MAAM,IAAI,EACzB,sBAAsB,CAAC,EAAE,MAAM,EAC/B,gBAAgB,CAAC,EAAE,CAAC,aAAa,EAAE,aAAa,CAAC,MAAM,CAAC,KAAK,IAAI,KAC9D,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAwC5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;GAOG;AACH,wBAAgB,oCAAoC,CAClD,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAQpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAeD,wBAAgB,0CAA0C,CACxD,QAAQ,EAAE,OAAO,EACjB,QAAQ,EAAE,MAAM,GACf,OAAO,CAET;AA8ED,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAwsIjB"}
|
|
@@ -2,7 +2,6 @@ import * as os from "os";
|
|
|
2
2
|
import * as path from "path";
|
|
3
3
|
import * as fs from "fs";
|
|
4
4
|
import { MULTIPART_PART_SIZE_BYTES } from "../sync/multipart-uploader.js";
|
|
5
|
-
import { AsyncLocalStorage } from "node:async_hooks";
|
|
6
5
|
import * as v8 from "node:v8";
|
|
7
6
|
import { performance } from "node:perf_hooks";
|
|
8
7
|
import { inventoryManagedSnapshots } from "../backup-prune.js";
|
|
@@ -37,7 +36,7 @@ import { SESSION_LOGS_PUSH_ONLY_PREFIX } from "../sync/pull-scope.js";
|
|
|
37
36
|
import { createHqFlagsCapSource, FullPassCapController, isFullPassDurableProgressEnabled, } from "../sync/full-pass-caps.js";
|
|
38
37
|
import { canSkipScheduledFullReconcile, FullPassContinuity, isFullPassJournalHealthy, } from "../sync/idle-full-reconcile.js";
|
|
39
38
|
import { DEFAULT_REALTIME_AUTHORIZATION_TIMEOUT_MS } from "../sync/realtime-rollout.js";
|
|
40
|
-
import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr,
|
|
39
|
+
import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr, } from "../sync/realtime-telemetry.js";
|
|
41
40
|
import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
42
41
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, scopePathForRoute, } from "./sync-runner-watch-routes.js";
|
|
43
42
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
@@ -85,6 +84,20 @@ function realtimeAdmissionDeadlineMs() {
|
|
|
85
84
|
function realtimeDrainBurstMs() {
|
|
86
85
|
return realtimeDrainBurstMsForTest ?? REALTIME_DRAIN_BURST_MS;
|
|
87
86
|
}
|
|
87
|
+
/**
|
|
88
|
+
* Retry delay for a watcher push or pull refused at the admission deadline.
|
|
89
|
+
* The refusal is a deferral, not a pass failure, so it does not advance the
|
|
90
|
+
* route's failure backoff. Each retry can wait up to the deadline again,
|
|
91
|
+
* so a long chore costs at most one attempt per deadline plus this delay.
|
|
92
|
+
*/
|
|
93
|
+
export const REALTIME_ADMISSION_RETRY_MS = 5_000;
|
|
94
|
+
/**
|
|
95
|
+
* In-process result of a scoped drain whose only shortfall was admission
|
|
96
|
+
* deferrals: the paths are restored and a retry is scheduled. Never a
|
|
97
|
+
* process exit code (negative), so it cannot reach the menubar's exit
|
|
98
|
+
* handling, and it is not 0, so it does not count as a completed pass.
|
|
99
|
+
*/
|
|
100
|
+
export const SCOPED_DRAIN_DEFERRED = -1;
|
|
88
101
|
export class RealtimeAdmissionTimeoutError extends Error {
|
|
89
102
|
retryable = true;
|
|
90
103
|
constructor(deadlineMs) {
|
|
@@ -185,17 +198,6 @@ export function allowConcurrentHandoffForGuardedWork(workClass, explicitlyAllowe
|
|
|
185
198
|
* pre-existing scoped fast-over-slow exception, but never while a chore owns
|
|
186
199
|
* durable state.
|
|
187
200
|
*/
|
|
188
|
-
/**
|
|
189
|
-
* Maps what holds a realtime op to its closed telemetry label. A chore kind
|
|
190
|
-
* outside the label set becomes `chore:unscoped`, so a new kind can never
|
|
191
|
-
* make the timing record fail its low-cardinality check.
|
|
192
|
-
*/
|
|
193
|
-
export function realtimeBlockerLabel(blocker) {
|
|
194
|
-
if (blocker.workClass === "realtime")
|
|
195
|
-
return `realtime:${blocker.lane}`;
|
|
196
|
-
const label = `chore:${blocker.choreKind ?? "unscoped"}`;
|
|
197
|
-
return REALTIME_BLOCKERS.includes(label) ? label : "chore:unscoped";
|
|
198
|
-
}
|
|
199
201
|
export class CooperativePassScheduler {
|
|
200
202
|
now;
|
|
201
203
|
starvationCapMs;
|
|
@@ -205,8 +207,6 @@ export class CooperativePassScheduler {
|
|
|
205
207
|
/** Realtime executes started by the current hand-off. Not the chore slot. */
|
|
206
208
|
realtimeOccupants = new Set();
|
|
207
209
|
handoffBacklog = null;
|
|
208
|
-
/** The chore suspended at checkpoint() while it lends the slot. Diagnostics only. */
|
|
209
|
-
handoffOwner = null;
|
|
210
210
|
handoffDeferredAt = null;
|
|
211
211
|
handoffAdmitted = false;
|
|
212
212
|
occupantEpoch = 0;
|
|
@@ -241,6 +241,13 @@ export class CooperativePassScheduler {
|
|
|
241
241
|
get hasActiveChore() {
|
|
242
242
|
return this.active?.workClass === "chore";
|
|
243
243
|
}
|
|
244
|
+
/**
|
|
245
|
+
* A chore owns the slot, or has lent it through a checkpoint or network
|
|
246
|
+
* hand-off and will take it back. Hand-offs only ever start from a chore.
|
|
247
|
+
*/
|
|
248
|
+
get choreHoldsSlot() {
|
|
249
|
+
return this.hasActiveChore || this._handoffActive || this.detachedChore !== null;
|
|
250
|
+
}
|
|
244
251
|
get hasActiveSlowRealtime() {
|
|
245
252
|
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
246
253
|
}
|
|
@@ -253,9 +260,7 @@ export class CooperativePassScheduler {
|
|
|
253
260
|
pendingLaneCount(lane) {
|
|
254
261
|
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
255
262
|
}
|
|
256
|
-
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false, signal, resourceCap = workClass === "chore", allowConcurrentHandoff = false, transferDirection, transferSizeClass = "unknown"
|
|
257
|
-
/** Diagnostics only: called once if this task queues, naming what holds the scheduler. */
|
|
258
|
-
onQueued, choreKind) {
|
|
263
|
+
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false, signal, resourceCap = workClass === "chore", allowConcurrentHandoff = false, transferDirection, transferSizeClass = "unknown") {
|
|
259
264
|
if (this.stopped)
|
|
260
265
|
return Promise.resolve(0);
|
|
261
266
|
if (signal?.aborted) {
|
|
@@ -274,7 +279,6 @@ export class CooperativePassScheduler {
|
|
|
274
279
|
allowConcurrentHandoff,
|
|
275
280
|
transferDirection,
|
|
276
281
|
transferSizeClass,
|
|
277
|
-
...(workClass === "chore" && choreKind !== undefined ? { choreKind } : {}),
|
|
278
282
|
// An immediately admitted task cannot starve, so avoid sampling the
|
|
279
283
|
// cadence clock merely to stamp it. This preserves the loop's deadline
|
|
280
284
|
// accounting and keeps the timestamp meaningful only for real queues.
|
|
@@ -314,12 +318,8 @@ export class CooperativePassScheduler {
|
|
|
314
318
|
resolve,
|
|
315
319
|
reject,
|
|
316
320
|
};
|
|
317
|
-
if (workClass === "realtime" && this.
|
|
318
|
-
|
|
319
|
-
entry.admissionTimer = setTimeout(() => {
|
|
320
|
-
this.expireAdmission(entry, deadlineMs);
|
|
321
|
-
}, deadlineMs);
|
|
322
|
-
entry.admissionTimer.unref?.();
|
|
321
|
+
if (workClass === "realtime" && this.choreHoldsSlot) {
|
|
322
|
+
this.armAdmission(entry);
|
|
323
323
|
}
|
|
324
324
|
if (signal) {
|
|
325
325
|
const onAbort = () => {
|
|
@@ -334,49 +334,10 @@ export class CooperativePassScheduler {
|
|
|
334
334
|
this.withinHandoffAdmission()) {
|
|
335
335
|
entry.occupancyArrival = true;
|
|
336
336
|
}
|
|
337
|
-
const holder = this.blockerFor(entry);
|
|
338
337
|
this.enqueuePending(entry);
|
|
339
338
|
this.drain();
|
|
340
|
-
if (onQueued && holder && this.pending.includes(entry)) {
|
|
341
|
-
try {
|
|
342
|
-
onQueued({
|
|
343
|
-
workClass: holder.workClass,
|
|
344
|
-
lane: holder.lane,
|
|
345
|
-
...(holder.choreKind !== undefined ? { choreKind: holder.choreKind } : {}),
|
|
346
|
-
});
|
|
347
|
-
}
|
|
348
|
-
catch {
|
|
349
|
-
// Diagnostics must never affect admission.
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
339
|
});
|
|
353
340
|
}
|
|
354
|
-
/**
|
|
355
|
-
* Diagnostics only: the work holding a task that cannot start. The occupant
|
|
356
|
-
* of the task's own lane comes first, then the slot owner, then a hand-off
|
|
357
|
-
* occupant that is what refuses admission, then the chore lending the slot.
|
|
358
|
-
*/
|
|
359
|
-
blockerFor(entry) {
|
|
360
|
-
const ownLane = entry.lane === "fast" ? this.activeFastRealtime : null;
|
|
361
|
-
return ownLane ??
|
|
362
|
-
this.active ??
|
|
363
|
-
this.blockingOccupant(entry) ??
|
|
364
|
-
this.detachedChore ??
|
|
365
|
-
this.handoffOwner ??
|
|
366
|
-
this.activeFastRealtime ??
|
|
367
|
-
this.pending.find((pending) => pending !== entry);
|
|
368
|
-
}
|
|
369
|
-
blockingOccupant(entry) {
|
|
370
|
-
if (entry.workClass !== "realtime" || !this._handoffActive)
|
|
371
|
-
return null;
|
|
372
|
-
const occupants = [...this.realtimeOccupants];
|
|
373
|
-
const sameDirection = entry.transferDirection === undefined
|
|
374
|
-
? undefined
|
|
375
|
-
: occupants.find((occupant) => occupant.transferDirection === entry.transferDirection);
|
|
376
|
-
if (sameDirection)
|
|
377
|
-
return sameDirection;
|
|
378
|
-
return occupants.length >= REALTIME_WORKER_OCCUPANCY_CAP ? occupants[0] ?? null : null;
|
|
379
|
-
}
|
|
380
341
|
stop() {
|
|
381
342
|
this.stopped = true;
|
|
382
343
|
this.active?.controller.abort();
|
|
@@ -407,6 +368,44 @@ export class CooperativePassScheduler {
|
|
|
407
368
|
}
|
|
408
369
|
this.pending.push(entry);
|
|
409
370
|
}
|
|
371
|
+
/**
|
|
372
|
+
* Bound how long a queued realtime entry sits behind a chore, counted from
|
|
373
|
+
* the first moment a chore holds the slot while it is queued. Time spent
|
|
374
|
+
* behind other realtime work does not count against it. Idempotent while
|
|
375
|
+
* the same chore holds the slot: an entry keeps its first deadline across
|
|
376
|
+
* that chore's hand-offs. Cleared when the chore finishes.
|
|
377
|
+
*/
|
|
378
|
+
armAdmission(entry) {
|
|
379
|
+
if (entry.workClass !== "realtime" || entry.admissionTimer !== undefined)
|
|
380
|
+
return;
|
|
381
|
+
const deadlineMs = realtimeAdmissionDeadlineMs();
|
|
382
|
+
entry.admissionTimer = setTimeout(() => {
|
|
383
|
+
this.expireAdmission(entry, deadlineMs);
|
|
384
|
+
}, deadlineMs);
|
|
385
|
+
entry.admissionTimer.unref?.();
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* A chore has taken (or taken back) the slot. Realtime queued while it
|
|
389
|
+
* was lent, or behind other realtime, would otherwise wait for the chore's
|
|
390
|
+
* next checkpoint with no bound.
|
|
391
|
+
*/
|
|
392
|
+
armQueuedRealtime() {
|
|
393
|
+
for (const entry of this.pending)
|
|
394
|
+
this.armAdmission(entry);
|
|
395
|
+
}
|
|
396
|
+
/**
|
|
397
|
+
* No chore holds the slot any more. Queued realtime now waits only behind
|
|
398
|
+
* other realtime, which does not count against the deadline; the next
|
|
399
|
+
* chore to take the slot arms a fresh one.
|
|
400
|
+
*/
|
|
401
|
+
disarmQueuedRealtime() {
|
|
402
|
+
for (const entry of this.pending) {
|
|
403
|
+
if (entry.admissionTimer === undefined)
|
|
404
|
+
continue;
|
|
405
|
+
clearTimeout(entry.admissionTimer);
|
|
406
|
+
entry.admissionTimer = undefined;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
410
409
|
clearAdmission(entry) {
|
|
411
410
|
if (entry.admissionTimer !== undefined) {
|
|
412
411
|
clearTimeout(entry.admissionTimer);
|
|
@@ -487,6 +486,8 @@ export class CooperativePassScheduler {
|
|
|
487
486
|
start(entry) {
|
|
488
487
|
this.clearAdmission(entry);
|
|
489
488
|
this.active = entry;
|
|
489
|
+
if (entry.workClass === "chore")
|
|
490
|
+
this.armQueuedRealtime();
|
|
490
491
|
let current;
|
|
491
492
|
try {
|
|
492
493
|
current = this.execute(entry);
|
|
@@ -512,6 +513,8 @@ export class CooperativePassScheduler {
|
|
|
512
513
|
complete(entry) {
|
|
513
514
|
if (this.active === entry)
|
|
514
515
|
this.active = null;
|
|
516
|
+
if (entry.workClass === "chore" && !this.choreHoldsSlot)
|
|
517
|
+
this.disarmQueuedRealtime();
|
|
515
518
|
this.drain();
|
|
516
519
|
}
|
|
517
520
|
startFastRealtime(entry) {
|
|
@@ -664,8 +667,10 @@ export class CooperativePassScheduler {
|
|
|
664
667
|
this.handoffDeferredAt = null;
|
|
665
668
|
this.handoffBacklog = null;
|
|
666
669
|
this.detachedChore = null;
|
|
667
|
-
if (!this.stopped)
|
|
670
|
+
if (!this.stopped) {
|
|
668
671
|
this.active = entry;
|
|
672
|
+
this.armQueuedRealtime();
|
|
673
|
+
}
|
|
669
674
|
this.options.caps?.resumeAfterHandoff();
|
|
670
675
|
const resolveRelease = this.resolveNetworkTransferRelease;
|
|
671
676
|
const realtimeRan = this.handoffAdmitted;
|
|
@@ -857,7 +862,6 @@ export class CooperativePassScheduler {
|
|
|
857
862
|
// flight starts on the free occupancy slot (see REALTIME_WORKER_OCCUPANCY_CAP).
|
|
858
863
|
this.active = null;
|
|
859
864
|
this._handoffActive = true;
|
|
860
|
-
this.handoffOwner = entry;
|
|
861
865
|
this.handoffDeferredAt = this.now();
|
|
862
866
|
this.handoffAdmitted = false;
|
|
863
867
|
this.handoffBacklog = null;
|
|
@@ -887,11 +891,12 @@ export class CooperativePassScheduler {
|
|
|
887
891
|
}
|
|
888
892
|
finally {
|
|
889
893
|
this._handoffActive = false;
|
|
890
|
-
this.handoffOwner = null;
|
|
891
894
|
this.handoffDeferredAt = null;
|
|
892
895
|
this.handoffBacklog = null;
|
|
893
|
-
if (!this.stopped)
|
|
896
|
+
if (!this.stopped) {
|
|
894
897
|
this.active = entry;
|
|
898
|
+
this.armQueuedRealtime();
|
|
899
|
+
}
|
|
895
900
|
this.options.caps?.resumeAfterHandoff();
|
|
896
901
|
}
|
|
897
902
|
return this.handoffAdmitted;
|
|
@@ -1715,13 +1720,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1715
1720
|
// executing, a coordinator realtime task must not take the normal fast-path
|
|
1716
1721
|
// lock carve-out: it has no bounded chore lease to prove independence.
|
|
1717
1722
|
let coordinatorWholeMutationChores = 0;
|
|
1718
|
-
// Each coordinator chore runs inside this context so the scheduler entry it
|
|
1719
|
-
// submits carries its kind; blocker telemetry reads it from the holding entry.
|
|
1720
|
-
const coordinatorChoreKind = new AsyncLocalStorage();
|
|
1721
|
-
const operationGateBlockerRecorder = (telemetry) => telemetry
|
|
1722
|
-
? (blocker) => telemetry.recordOperationGateBlocker(realtimeBlockerLabel(blocker))
|
|
1723
|
-
: undefined;
|
|
1724
|
-
const coordinatorTelemetry = new WeakMap();
|
|
1725
1723
|
let operationLockUsers = 0;
|
|
1726
1724
|
let resolveNoOperationLockUsers = null;
|
|
1727
1725
|
const acquireOperationLockUser = () => {
|
|
@@ -2063,12 +2061,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2063
2061
|
const resolveStoppedQueue = () => {
|
|
2064
2062
|
passScheduler.stop();
|
|
2065
2063
|
};
|
|
2066
|
-
const runGuardedTask = (task, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, allowConcurrentHandoff = false, transferDirection, transferSizeClass = "unknown"
|
|
2064
|
+
const runGuardedTask = (task, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, allowConcurrentHandoff = false, transferDirection, transferSizeClass = "unknown") => {
|
|
2067
2065
|
const allowFlaggedRealtimeHandoff = allowConcurrentHandoffForGuardedWork(workClass, allowConcurrentHandoff);
|
|
2068
2066
|
// Lanes still choose the pass execution profile, but no longer decide
|
|
2069
2067
|
// chore ownership. Realtime normally waits for a chore hand-off; the
|
|
2070
2068
|
// watcher fast path explicitly opts into its established scoped exception.
|
|
2071
|
-
return passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowFlaggedRealtimeHandoff, transferDirection, transferSizeClass
|
|
2069
|
+
return passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowFlaggedRealtimeHandoff, transferDirection, transferSizeClass);
|
|
2072
2070
|
};
|
|
2073
2071
|
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal) => {
|
|
2074
2072
|
telemetry?.mark("operationGate");
|
|
@@ -2079,7 +2077,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2079
2077
|
telemetry?.mark("appendLock");
|
|
2080
2078
|
telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
|
|
2081
2079
|
return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal);
|
|
2082
|
-
}, lane, workClass, allowFastRealtimeOvertake, admissionSignal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass
|
|
2080
|
+
}, lane, workClass, allowFastRealtimeOvertake, admissionSignal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass);
|
|
2083
2081
|
};
|
|
2084
2082
|
/*
|
|
2085
2083
|
* Area storage and the parallel coordinator default to AUTO. AUTO turns
|
|
@@ -2106,9 +2104,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2106
2104
|
if (slice.work.class === "chore" && execution.wholeMutation)
|
|
2107
2105
|
coordinatorWholeMutationChores += 1;
|
|
2108
2106
|
try {
|
|
2109
|
-
execution.result = slice
|
|
2110
|
-
? await coordinatorChoreKind.run(slice.work.kind, () => execution.run(slice))
|
|
2111
|
-
: await execution.run(slice);
|
|
2107
|
+
execution.result = await execution.run(slice);
|
|
2112
2108
|
}
|
|
2113
2109
|
finally {
|
|
2114
2110
|
if (slice.work.class === "chore" && execution.wholeMutation)
|
|
@@ -2139,11 +2135,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2139
2135
|
// therefore remains behind the runner's guarded operation queue until
|
|
2140
2136
|
// those executors can provide concrete scopes.
|
|
2141
2137
|
planChorePaths: () => [],
|
|
2142
|
-
onRealtimeAdmissionBlocked: (queued, heldBy) => {
|
|
2143
|
-
coordinatorTelemetry.get(queued)?.recordAdmissionBlocker(heldBy.class === "realtime"
|
|
2144
|
-
? `realtime:${heldBy.lane === "fast" ? "fast" : "slow"}`
|
|
2145
|
-
: realtimeBlockerLabel({ workClass: "chore", lane: "slow", choreKind: heldBy.kind }));
|
|
2146
|
-
},
|
|
2147
2138
|
});
|
|
2148
2139
|
let coordinator = parallelCoordinatorEnabled ? createCoordinator() : null;
|
|
2149
2140
|
const runCoordinatorWork = async (work, task, options = {}) => {
|
|
@@ -2163,12 +2154,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2163
2154
|
};
|
|
2164
2155
|
}
|
|
2165
2156
|
telemetry?.mark("worker");
|
|
2166
|
-
if (telemetry)
|
|
2167
|
-
coordinatorTelemetry.set(work, telemetry);
|
|
2168
2157
|
const execution = {
|
|
2169
2158
|
run: async (slice) => {
|
|
2170
2159
|
telemetry?.mark("conflict");
|
|
2171
|
-
|
|
2160
|
+
let result;
|
|
2161
|
+
try {
|
|
2162
|
+
result = await task(slice, telemetry, true);
|
|
2163
|
+
}
|
|
2164
|
+
catch (error) {
|
|
2165
|
+
if (error instanceof RealtimeAdmissionTimeoutError) {
|
|
2166
|
+
execution.admissionTimeout = error;
|
|
2167
|
+
}
|
|
2168
|
+
throw error;
|
|
2169
|
+
}
|
|
2172
2170
|
telemetry?.mark("commit");
|
|
2173
2171
|
return result;
|
|
2174
2172
|
},
|
|
@@ -2191,12 +2189,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2191
2189
|
: "retryable";
|
|
2192
2190
|
telemetry.finish(outcomeLabel, realtimeTelemetryReason(outcome.paths));
|
|
2193
2191
|
}
|
|
2192
|
+
const admissionTimeout = execution.result === undefined &&
|
|
2193
|
+
execution.admissionTimeout !== undefined &&
|
|
2194
|
+
outcome.paths.length > 0 &&
|
|
2195
|
+
outcome.paths.every((path) => path.status === "retryable")
|
|
2196
|
+
? execution.admissionTimeout
|
|
2197
|
+
: undefined;
|
|
2194
2198
|
return {
|
|
2195
2199
|
result: execution.result ?? (allRefused ? 0 : 1),
|
|
2196
2200
|
accepted,
|
|
2197
2201
|
coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
|
|
2198
2202
|
.filter((path) => path.status === "refused")
|
|
2199
2203
|
.map((path) => path.path)),
|
|
2204
|
+
...(admissionTimeout ? { admissionTimeout } : {}),
|
|
2200
2205
|
};
|
|
2201
2206
|
}
|
|
2202
2207
|
catch (error) {
|
|
@@ -3513,6 +3518,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3513
3518
|
emitScopedPushFailureTelemetry?.(event);
|
|
3514
3519
|
}
|
|
3515
3520
|
};
|
|
3521
|
+
/**
|
|
3522
|
+
* The push never ran: a chore held the slot past the admission deadline.
|
|
3523
|
+
* Put the paths back and retry without counting a failure, so the route
|
|
3524
|
+
* backoff, the scoped-push-failure line and its telemetry stay for real
|
|
3525
|
+
* pass failures.
|
|
3526
|
+
*/
|
|
3527
|
+
const deferPushAfterAdmissionTimeout = (group, routeId) => {
|
|
3528
|
+
restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
|
|
3529
|
+
scheduleRoutePushRetry(routeId, REALTIME_ADMISSION_RETRY_MS);
|
|
3530
|
+
};
|
|
3516
3531
|
if (watcherDrainRunsPush && grouped) {
|
|
3517
3532
|
for (const group of grouped.values()) {
|
|
3518
3533
|
// Paused workspaces: drop the scoped push without uploading or
|
|
@@ -3610,6 +3625,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3610
3625
|
}
|
|
3611
3626
|
}
|
|
3612
3627
|
}
|
|
3628
|
+
else if ("admissionTimeout" in execution && execution.admissionTimeout) {
|
|
3629
|
+
deferPushAfterAdmissionTimeout(group, routeId);
|
|
3630
|
+
}
|
|
3613
3631
|
else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
|
|
3614
3632
|
routePushFailureBackoff.succeeded(routeId);
|
|
3615
3633
|
clearRoutePushRetry(routeId);
|
|
@@ -3628,6 +3646,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3628
3646
|
}
|
|
3629
3647
|
}
|
|
3630
3648
|
catch (err) {
|
|
3649
|
+
if (err instanceof RealtimeAdmissionTimeoutError) {
|
|
3650
|
+
deferPushAfterAdmissionTimeout(group, routeId);
|
|
3651
|
+
continue;
|
|
3652
|
+
}
|
|
3631
3653
|
recordPushFailure(group.route, 1, pushFailureText(1, group.route, err));
|
|
3632
3654
|
restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
|
|
3633
3655
|
}
|
|
@@ -3642,6 +3664,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3642
3664
|
// falls back to the unscoped drain shape.
|
|
3643
3665
|
if (grouped) {
|
|
3644
3666
|
let worstExit = 0;
|
|
3667
|
+
let deferred = false;
|
|
3645
3668
|
let ranPersonalDrain = false;
|
|
3646
3669
|
for (const group of grouped.values()) {
|
|
3647
3670
|
// Paused workspaces sync in NO direction while Off.
|
|
@@ -3672,12 +3695,25 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3672
3695
|
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
3673
3696
|
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced);
|
|
3674
3697
|
})
|
|
3675
|
-
:
|
|
3698
|
+
: await runGuarded(pullArgv, undefined, lane, "realtime").then((result) => ({ result, accepted: true }), (error) => {
|
|
3699
|
+
if (error instanceof RealtimeAdmissionTimeoutError) {
|
|
3700
|
+
return { accepted: false, admissionTimeout: error };
|
|
3701
|
+
}
|
|
3702
|
+
throw error;
|
|
3703
|
+
});
|
|
3704
|
+
if ("admissionTimeout" in execution && execution.admissionTimeout) {
|
|
3705
|
+
// The pull never ran. Keep its paths and retry, without the
|
|
3706
|
+
// pass-failure handling the shared exit code would get.
|
|
3707
|
+
restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
|
|
3708
|
+
scheduleRoutePushRetry(`pull:${routeKey(group.route)}`, REALTIME_ADMISSION_RETRY_MS);
|
|
3709
|
+
deferred = true;
|
|
3710
|
+
continue;
|
|
3711
|
+
}
|
|
3676
3712
|
const exit = scopedRouteDrainExit(execution);
|
|
3677
3713
|
if (exit !== 0 && worstExit === 0)
|
|
3678
3714
|
worstExit = exit;
|
|
3679
3715
|
}
|
|
3680
|
-
return worstExit;
|
|
3716
|
+
return worstExit === 0 && deferred ? SCOPED_DRAIN_DEFERRED : worstExit;
|
|
3681
3717
|
}
|
|
3682
3718
|
// An idle poll has no sound path scope. It also must not turn into an
|
|
3683
3719
|
// unlabelled all-vault pull: scheduled reconciles and the explicit safety
|
|
@@ -3908,7 +3944,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3908
3944
|
continue;
|
|
3909
3945
|
const generation = ++nextWatcherBatchGeneration;
|
|
3910
3946
|
void runScopedDrain(lane, generation, true).then((exit) => {
|
|
3911
|
-
|
|
3947
|
+
// A deferral already restored its paths and scheduled a retry.
|
|
3948
|
+
if (exit === 0 || exit === SCOPED_DRAIN_DEFERRED)
|
|
3912
3949
|
return;
|
|
3913
3950
|
restorePendingWatcherPaths(lane.paths, generation);
|
|
3914
3951
|
interruptWait();
|
|
@@ -3929,7 +3966,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3929
3966
|
// the same dirty signal as the compatibility drain.
|
|
3930
3967
|
realtimeState.scheduler?.signal("watcher");
|
|
3931
3968
|
void runScopedDrain(lanes.fast, generation, true, true).then((exit) => {
|
|
3932
|
-
if (exit === 0)
|
|
3969
|
+
if (exit === 0 || exit === SCOPED_DRAIN_DEFERRED)
|
|
3933
3970
|
return;
|
|
3934
3971
|
restorePendingWatcherPaths(lanes.fast.paths, generation);
|
|
3935
3972
|
interruptWait();
|
|
@@ -4089,6 +4126,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4089
4126
|
return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal));
|
|
4090
4127
|
});
|
|
4091
4128
|
const pathCount = events.length;
|
|
4129
|
+
// Rethrow the typed refusal so the receiver keeps the lease and
|
|
4130
|
+
// retries the whole batch, instead of bisecting a "failed" pull.
|
|
4131
|
+
if (execution.admissionTimeout)
|
|
4132
|
+
throw execution.admissionTimeout;
|
|
4092
4133
|
const exitCode = passExitCode(execution.result);
|
|
4093
4134
|
if (shouldAcknowledgeParallelUnresolvedCompany(execution.accepted, exitCode)) {
|
|
4094
4135
|
if (!unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)) {
|
|
@@ -4213,10 +4254,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4213
4254
|
let outcome;
|
|
4214
4255
|
try {
|
|
4215
4256
|
if (parallelCoordinatorEnabled) {
|
|
4216
|
-
const execution = await runRealtimePass("wake", "pull", [], "slow", async (_slice,
|
|
4257
|
+
const execution = await runRealtimePass("wake", "pull", [], "slow", async (_slice, _telemetry, coordinatorFenced) => runGuardedTask(() => runTaskWithLock(async () => {
|
|
4217
4258
|
await drain();
|
|
4218
4259
|
return 0;
|
|
4219
|
-
}, "realtime"), "slow", "realtime", false, undefined, false, coordinatorFenced
|
|
4260
|
+
}, "realtime"), "slow", "realtime", false, undefined, false, coordinatorFenced));
|
|
4220
4261
|
outcome = execution.accepted ? execution.result : 1;
|
|
4221
4262
|
}
|
|
4222
4263
|
else {
|
|
@@ -4822,7 +4863,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4822
4863
|
const catchupBatch = workingBatch;
|
|
4823
4864
|
const catchupGeneration = pending.generation;
|
|
4824
4865
|
void runScopedDrain(catchupBatch, catchupGeneration, true).then((exit) => {
|
|
4825
|
-
if (exit === 0)
|
|
4866
|
+
if (exit === 0 || exit === SCOPED_DRAIN_DEFERRED)
|
|
4826
4867
|
return;
|
|
4827
4868
|
restorePendingWatcherPaths(catchupBatch.paths, catchupGeneration);
|
|
4828
4869
|
interruptWait();
|
|
@@ -4915,6 +4956,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4915
4956
|
// Keep the watcher alive while preserving the one-shot non-zero code.
|
|
4916
4957
|
process.stderr.write("hq-sync-runner: watch pass skipped — unresolved company target; retrying next poll\n");
|
|
4917
4958
|
}
|
|
4959
|
+
else if (exitCode === SCOPED_DRAIN_DEFERRED) {
|
|
4960
|
+
// Realtime admission deferred part of the scoped drain behind a
|
|
4961
|
+
// chore. The paths are restored and a retry is scheduled; nothing
|
|
4962
|
+
// failed, so nothing is reported.
|
|
4963
|
+
}
|
|
4918
4964
|
else if (exitCode !== 0) {
|
|
4919
4965
|
// A scoped company-resolution or per-pass failure is isolated to that
|
|
4920
4966
|
// pass. The error was emitted by the runner; terminating the daemon
|