@indigoai-us/hq-cloud 6.16.70 → 6.16.71-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +25 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +124 -24
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +58 -1
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.test.js +290 -5
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +11 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +44 -10
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +36 -1
- package/dist/cli/share.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +2 -0
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +17 -0
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -3,6 +3,8 @@ import { readJournal } from "../journal.js";
|
|
|
3
3
|
import { type FeatureSwitch } from "../sync/feature-flags.js";
|
|
4
4
|
import { HeapCensus } from "./sync-runner-heap-telemetry.js";
|
|
5
5
|
import { FullPassCapController } from "../sync/full-pass-caps.js";
|
|
6
|
+
import { type RealtimeTelemetryReason } from "../sync/realtime-telemetry.js";
|
|
7
|
+
import { type PathWorkOutcome } from "../sync/sync-work.js";
|
|
6
8
|
import type { TelemetryClaims } from "../telemetry-events.js";
|
|
7
9
|
import type { JournalEntry } from "../types.js";
|
|
8
10
|
import type { CooperativePassCheckpoint, Direction, RunnerLoopDeps, RunnerPassOutcome } from "./sync-runner.js";
|
|
@@ -15,11 +17,32 @@ export declare const CHORE_STARVATION_CAP_MS: number;
|
|
|
15
17
|
* still has one 15s yield window to admit the item.
|
|
16
18
|
*/
|
|
17
19
|
export declare const REALTIME_ADMISSION_DEADLINE_MS: number;
|
|
20
|
+
/**
|
|
21
|
+
* How long a chore checkpoint may spend executing realtime before it must
|
|
22
|
+
* resume. Distinct from {@link CHORE_STARVATION_CAP_MS}: that value only
|
|
23
|
+
* promotes a deferred chore to the front of the queue.
|
|
24
|
+
*/
|
|
25
|
+
export declare const REALTIME_DRAIN_BURST_MS: number;
|
|
18
26
|
export declare function setRealtimeAdmissionDeadlineMsForTesting(ms: number | undefined): void;
|
|
27
|
+
export declare function setRealtimeDrainBurstMsForTesting(ms: number | undefined): void;
|
|
19
28
|
export declare class RealtimeAdmissionTimeoutError extends Error {
|
|
20
29
|
readonly retryable = true;
|
|
21
30
|
constructor(deadlineMs: number);
|
|
22
31
|
}
|
|
32
|
+
export declare class RealtimeDrainBurstError extends Error {
|
|
33
|
+
readonly retryable = true;
|
|
34
|
+
constructor(burstMs: number);
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Coordinator path status for one sync-pass exit. Exit 19 (vanished company)
|
|
38
|
+
* is durable so a leftover local slug is not restored and retried forever;
|
|
39
|
+
* transient 75 and partial 2 stay retryable.
|
|
40
|
+
*/
|
|
41
|
+
export declare function coordinatorPathOutcome(exitCode: number): {
|
|
42
|
+
status: "durable" | "retryable";
|
|
43
|
+
reason?: string;
|
|
44
|
+
};
|
|
45
|
+
export declare function realtimeTelemetryReason(paths: readonly PathWorkOutcome[]): RealtimeTelemetryReason | undefined;
|
|
23
46
|
export type GuardedWorkClass = "realtime" | "chore";
|
|
24
47
|
export interface GuardedPassContext {
|
|
25
48
|
/**
|
|
@@ -28,6 +51,8 @@ export interface GuardedPassContext {
|
|
|
28
51
|
* root lock under a single-flight cooperative hand-off.
|
|
29
52
|
*/
|
|
30
53
|
checkpoint: CooperativePassCheckpoint;
|
|
54
|
+
/** Aborted when a drain burst expires or the caller cancels the slice. */
|
|
55
|
+
signal: AbortSignal;
|
|
31
56
|
}
|
|
32
57
|
/**
|
|
33
58
|
* One durable-state owner at a time for a full chore. Realtime work never runs
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAG1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AAKtE,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,0EAA0E;IAC1E,MAAM,EAAE,WAAW,CAAC;CACrB;AAiBD;;;;;;GAMG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAWnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAZ1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IAInD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,GAClC,OAAO,CAAC,CAAC,CAAC;IA2Db,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAeb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;YAQD,aAAa;YAyDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AACjE,eAAO,MAAM,yBAAyB,EAAE,MAAM,GAAG,SAAuB,CAAC;AAEzE,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,GAAG,SAAS,CAOpB;AAiBD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAuyGjB"}
|
|
@@ -42,13 +42,26 @@ export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
|
42
42
|
* still has one 15s yield window to admit the item.
|
|
43
43
|
*/
|
|
44
44
|
export const REALTIME_ADMISSION_DEADLINE_MS = 2 * DEFAULT_REALTIME_AUTHORIZATION_TIMEOUT_MS;
|
|
45
|
+
/**
|
|
46
|
+
* How long a chore checkpoint may spend executing realtime before it must
|
|
47
|
+
* resume. Distinct from {@link CHORE_STARVATION_CAP_MS}: that value only
|
|
48
|
+
* promotes a deferred chore to the front of the queue.
|
|
49
|
+
*/
|
|
50
|
+
export const REALTIME_DRAIN_BURST_MS = REALTIME_ADMISSION_DEADLINE_MS;
|
|
45
51
|
let realtimeAdmissionDeadlineMsForTest;
|
|
52
|
+
let realtimeDrainBurstMsForTest;
|
|
46
53
|
export function setRealtimeAdmissionDeadlineMsForTesting(ms) {
|
|
47
54
|
realtimeAdmissionDeadlineMsForTest = ms;
|
|
48
55
|
}
|
|
56
|
+
export function setRealtimeDrainBurstMsForTesting(ms) {
|
|
57
|
+
realtimeDrainBurstMsForTest = ms;
|
|
58
|
+
}
|
|
49
59
|
function realtimeAdmissionDeadlineMs() {
|
|
50
60
|
return realtimeAdmissionDeadlineMsForTest ?? REALTIME_ADMISSION_DEADLINE_MS;
|
|
51
61
|
}
|
|
62
|
+
function realtimeDrainBurstMs() {
|
|
63
|
+
return realtimeDrainBurstMsForTest ?? REALTIME_DRAIN_BURST_MS;
|
|
64
|
+
}
|
|
52
65
|
export class RealtimeAdmissionTimeoutError extends Error {
|
|
53
66
|
retryable = true;
|
|
54
67
|
constructor(deadlineMs) {
|
|
@@ -56,7 +69,30 @@ export class RealtimeAdmissionTimeoutError extends Error {
|
|
|
56
69
|
this.name = "RealtimeAdmissionTimeout";
|
|
57
70
|
}
|
|
58
71
|
}
|
|
59
|
-
|
|
72
|
+
export class RealtimeDrainBurstError extends Error {
|
|
73
|
+
retryable = true;
|
|
74
|
+
constructor(burstMs) {
|
|
75
|
+
super(`realtime drain burst exceeded ${burstMs}ms`);
|
|
76
|
+
this.name = "RealtimeDrainBurst";
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Coordinator path status for one sync-pass exit. Exit 19 (vanished company)
|
|
81
|
+
* is durable so a leftover local slug is not restored and retried forever;
|
|
82
|
+
* transient 75 and partial 2 stay retryable.
|
|
83
|
+
*/
|
|
84
|
+
export function coordinatorPathOutcome(exitCode) {
|
|
85
|
+
if (exitCode === 0)
|
|
86
|
+
return { status: "durable" };
|
|
87
|
+
if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
88
|
+
return { status: "durable", reason: "unresolved company" };
|
|
89
|
+
}
|
|
90
|
+
return {
|
|
91
|
+
status: "retryable",
|
|
92
|
+
reason: `sync pass exited ${exitCode}`,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
export function realtimeTelemetryReason(paths) {
|
|
60
96
|
for (const path of paths) {
|
|
61
97
|
const reason = path.reason;
|
|
62
98
|
if (!reason)
|
|
@@ -67,6 +103,12 @@ function realtimeTelemetryReason(paths) {
|
|
|
67
103
|
return "stale-epoch";
|
|
68
104
|
if (reason.startsWith("realtime admission timeout"))
|
|
69
105
|
return "admission-timeout";
|
|
106
|
+
if (reason.startsWith("realtime drain burst"))
|
|
107
|
+
return "drain-burst";
|
|
108
|
+
if (reason === "unresolved company" ||
|
|
109
|
+
reason === `sync pass exited ${UNRESOLVED_COMPANY_EXIT}`) {
|
|
110
|
+
return "unresolved-company";
|
|
111
|
+
}
|
|
70
112
|
if (reason.startsWith("sync pass exited"))
|
|
71
113
|
return "pass-exit";
|
|
72
114
|
if (reason === "coordinator is shutting down")
|
|
@@ -116,6 +158,10 @@ export class CooperativePassScheduler {
|
|
|
116
158
|
if (signal?.aborted) {
|
|
117
159
|
return Promise.reject(this.abortedError());
|
|
118
160
|
}
|
|
161
|
+
const controller = new AbortController();
|
|
162
|
+
if (signal) {
|
|
163
|
+
signal.addEventListener("abort", () => controller.abort(), { once: true });
|
|
164
|
+
}
|
|
119
165
|
const directEntry = {
|
|
120
166
|
task,
|
|
121
167
|
workClass,
|
|
@@ -128,6 +174,7 @@ export class CooperativePassScheduler {
|
|
|
128
174
|
enqueuedAt: 0,
|
|
129
175
|
resolve: () => undefined,
|
|
130
176
|
reject: () => undefined,
|
|
177
|
+
controller,
|
|
131
178
|
};
|
|
132
179
|
if (this.canStartFastRealtime(directEntry)) {
|
|
133
180
|
return this.startFastRealtime(directEntry);
|
|
@@ -317,7 +364,10 @@ export class CooperativePassScheduler {
|
|
|
317
364
|
execute(entry) {
|
|
318
365
|
if (entry.resourceCap)
|
|
319
366
|
this.options.caps?.beginChore();
|
|
320
|
-
return entry.task({
|
|
367
|
+
return entry.task({
|
|
368
|
+
checkpoint: () => this.checkpoint(entry),
|
|
369
|
+
signal: entry.controller.signal,
|
|
370
|
+
});
|
|
321
371
|
}
|
|
322
372
|
async drainRealtime(entry) {
|
|
323
373
|
if (this.stopped ||
|
|
@@ -329,12 +379,16 @@ export class CooperativePassScheduler {
|
|
|
329
379
|
// `entry` is suspended at a caller-declared durable boundary. It remains
|
|
330
380
|
// the sole full-pass continuation and will resume after this bounded burst.
|
|
331
381
|
const deferredAt = this.now();
|
|
382
|
+
const burstMs = realtimeDrainBurstMs();
|
|
332
383
|
this.active = null;
|
|
333
384
|
this._handoffActive = true;
|
|
334
385
|
this.options.caps?.suspendForHandoff();
|
|
335
386
|
let handedOff = false;
|
|
336
387
|
try {
|
|
337
|
-
while (!this.stopped && this.now() - deferredAt <
|
|
388
|
+
while (!this.stopped && this.now() - deferredAt < burstMs) {
|
|
389
|
+
const remainingMs = burstMs - (this.now() - deferredAt);
|
|
390
|
+
if (remainingMs <= 0)
|
|
391
|
+
break;
|
|
338
392
|
const realtimeIndex = this.nextRealtimeIndex();
|
|
339
393
|
if (realtimeIndex < 0)
|
|
340
394
|
break;
|
|
@@ -342,13 +396,27 @@ export class CooperativePassScheduler {
|
|
|
342
396
|
this.clearAdmission(realtime);
|
|
343
397
|
handedOff = true;
|
|
344
398
|
this.active = realtime;
|
|
399
|
+
let burstTimer;
|
|
400
|
+
const burstError = new RealtimeDrainBurstError(burstMs);
|
|
401
|
+
const burstExpired = new Promise((_, reject) => {
|
|
402
|
+
burstTimer = setTimeout(() => {
|
|
403
|
+
realtime.controller.abort();
|
|
404
|
+
reject(burstError);
|
|
405
|
+
}, remainingMs);
|
|
406
|
+
burstTimer.unref?.();
|
|
407
|
+
});
|
|
408
|
+
const running = this.execute(realtime);
|
|
409
|
+
void running.catch(() => undefined);
|
|
345
410
|
try {
|
|
346
|
-
|
|
411
|
+
const outcome = await Promise.race([running, burstExpired]);
|
|
412
|
+
realtime.resolve(outcome);
|
|
347
413
|
}
|
|
348
414
|
catch (error) {
|
|
349
415
|
realtime.reject(error);
|
|
350
416
|
}
|
|
351
417
|
finally {
|
|
418
|
+
if (burstTimer !== undefined)
|
|
419
|
+
clearTimeout(burstTimer);
|
|
352
420
|
if (this.active === realtime)
|
|
353
421
|
this.active = null;
|
|
354
422
|
}
|
|
@@ -1342,15 +1410,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1342
1410
|
if (slice.work.class === "chore" && execution.wholeMutation)
|
|
1343
1411
|
coordinatorWholeMutationChores -= 1;
|
|
1344
1412
|
}
|
|
1345
|
-
const
|
|
1413
|
+
const pathOutcome = coordinatorPathOutcome(passExitCode(execution.result));
|
|
1346
1414
|
return {
|
|
1347
1415
|
paths: slice.paths.map((item) => ({
|
|
1348
1416
|
path: item.canonicalPath,
|
|
1349
1417
|
epoch: item.epoch,
|
|
1350
|
-
|
|
1351
|
-
...(exitCode === 0
|
|
1352
|
-
? {}
|
|
1353
|
-
: { reason: `sync pass exited ${exitCode}` }),
|
|
1418
|
+
...pathOutcome,
|
|
1354
1419
|
})),
|
|
1355
1420
|
};
|
|
1356
1421
|
};
|
|
@@ -1403,7 +1468,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1403
1468
|
coordinatorExecutions.set(work, execution);
|
|
1404
1469
|
try {
|
|
1405
1470
|
const outcome = await coordinator.submit(work);
|
|
1406
|
-
const executorSucceeded = execution.result !== undefined &&
|
|
1471
|
+
const executorSucceeded = execution.result !== undefined &&
|
|
1472
|
+
coordinatorPathOutcome(passExitCode(execution.result)).status === "durable";
|
|
1407
1473
|
const allRefused = outcome.paths.length > 0 && outcome.paths.every((path) => path.status === "refused");
|
|
1408
1474
|
const accepted = executorSucceeded
|
|
1409
1475
|
? outcome.paths.every((path) => path.status === "durable" || path.status === "refused")
|
|
@@ -1427,7 +1493,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1427
1493
|
catch (error) {
|
|
1428
1494
|
telemetry?.finish("retryable", error instanceof RealtimeAdmissionTimeoutError
|
|
1429
1495
|
? "admission-timeout"
|
|
1430
|
-
:
|
|
1496
|
+
: error instanceof RealtimeDrainBurstError
|
|
1497
|
+
? "drain-burst"
|
|
1498
|
+
: undefined);
|
|
1431
1499
|
throw error;
|
|
1432
1500
|
}
|
|
1433
1501
|
finally {
|
|
@@ -1612,6 +1680,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1612
1680
|
/** Vanished never-journaled file deletes removed from batches at drain time. */
|
|
1613
1681
|
let drainedUntrackedDeletes = 0;
|
|
1614
1682
|
const pendingWatcherPaths = new Map();
|
|
1683
|
+
const droppedUnresolvedWatcherSlugs = new Set();
|
|
1684
|
+
const logDroppedUnresolvedWatcherRoute = (slug) => {
|
|
1685
|
+
if (droppedUnresolvedWatcherSlugs.has(slug))
|
|
1686
|
+
return;
|
|
1687
|
+
droppedUnresolvedWatcherSlugs.add(slug);
|
|
1688
|
+
process.stderr.write(`${JSON.stringify({
|
|
1689
|
+
type: "error",
|
|
1690
|
+
company: slug,
|
|
1691
|
+
path: "(company-resolution)",
|
|
1692
|
+
message: "watcher route dropped: company unresolved (404); will retry on next full pass or filesystem event",
|
|
1693
|
+
})}\n`);
|
|
1694
|
+
};
|
|
1615
1695
|
const pendingWatcherChanges = new Map();
|
|
1616
1696
|
// A warm-up scan observes filesystem state over time. Keep an ingress
|
|
1617
1697
|
// sequence so its older observations cannot overwrite a live watcher event
|
|
@@ -2423,7 +2503,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2423
2503
|
? () => prepareWatcherChanges(group.changes)
|
|
2424
2504
|
: undefined, lane, "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
|
|
2425
2505
|
const result = execution.result;
|
|
2426
|
-
|
|
2506
|
+
const pushExit = passExitCode(result);
|
|
2507
|
+
if (execution.accepted && pushExit === 0) {
|
|
2427
2508
|
const refusedPaths = new Set(typeof result === "number"
|
|
2428
2509
|
? []
|
|
2429
2510
|
: (result.result?.pushPathResults ?? [])
|
|
@@ -2449,6 +2530,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2449
2530
|
}
|
|
2450
2531
|
}
|
|
2451
2532
|
}
|
|
2533
|
+
else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
|
|
2534
|
+
if (group.route.kind === "company") {
|
|
2535
|
+
logDroppedUnresolvedWatcherRoute(group.route.slug);
|
|
2536
|
+
}
|
|
2537
|
+
}
|
|
2452
2538
|
else {
|
|
2453
2539
|
restorePendingWatcherPaths(group.paths, watcherBatchGeneration);
|
|
2454
2540
|
if (wakeOnPushFailure)
|
|
@@ -3491,18 +3577,32 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3491
3577
|
}
|
|
3492
3578
|
else {
|
|
3493
3579
|
try {
|
|
3494
|
-
|
|
3495
|
-
|
|
3496
|
-
|
|
3497
|
-
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3580
|
+
if (watcherWarmupCatchup) {
|
|
3581
|
+
emitDecision("catch-up", reasons);
|
|
3582
|
+
const catchupBatch = workingBatch;
|
|
3583
|
+
const catchupGeneration = pending.generation;
|
|
3584
|
+
void runScopedDrain(catchupBatch, catchupGeneration, true).then((exit) => {
|
|
3585
|
+
if (exit === 0)
|
|
3586
|
+
return;
|
|
3587
|
+
restorePendingWatcherPaths(catchupBatch.paths, catchupGeneration);
|
|
3588
|
+
interruptWait();
|
|
3589
|
+
}).catch((err) => {
|
|
3590
|
+
restorePendingWatcherPaths(catchupBatch.paths, catchupGeneration);
|
|
3591
|
+
process.stderr.write(`watch warmup catch-up failed, will retry next tick: ${describeError(err)}\n`);
|
|
3592
|
+
interruptWait();
|
|
3593
|
+
});
|
|
3594
|
+
result = 0;
|
|
3595
|
+
}
|
|
3596
|
+
else {
|
|
3597
|
+
result = overflowWithKnownRoutes
|
|
3598
|
+
? await runOverflowRouteDrain(workingBatch)
|
|
3599
|
+
: await runScopedDrain(workingBatch, pending.generation);
|
|
3600
|
+
// Known-route overflow intentionally runs whole *routes* (the paths
|
|
3601
|
+
// were dropped); ordinary batches remain path-scoped. Label those
|
|
3602
|
+
// distinct scopes so operators can distinguish them from a full
|
|
3603
|
+
// all-vault reconcile.
|
|
3604
|
+
emitDecision(overflowWithKnownRoutes ? "route" : "scoped", reasons);
|
|
3605
|
+
}
|
|
3506
3606
|
}
|
|
3507
3607
|
catch (err) {
|
|
3508
3608
|
// Never silently drop a change: any failure inside the targeted
|