@cotal-ai/runtime 0.48.1 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/host-starvation.d.ts +145 -0
- package/dist/host-starvation.d.ts.map +1 -0
- package/dist/host-starvation.js +281 -0
- package/dist/host-starvation.js.map +1 -0
- package/dist/index.js +7 -4
- package/dist/index.js.map +1 -1
- package/dist/mesh-handler.d.ts +57 -3
- package/dist/mesh-handler.d.ts.map +1 -1
- package/dist/mesh-handler.js +109 -2
- package/dist/mesh-handler.js.map +1 -1
- package/dist/migrate.js +13 -0
- package/dist/migrate.js.map +1 -1
- package/dist/run-command.d.ts.map +1 -1
- package/dist/run-command.js +109 -8
- package/dist/run-command.js.map +1 -1
- package/dist/run-effect-host.d.ts +2 -2
- package/dist/run-effect-host.d.ts.map +1 -1
- package/dist/run-effect-host.js +9 -3
- package/dist/run-effect-host.js.map +1 -1
- package/dist/run-host.d.ts.map +1 -1
- package/dist/run-host.js +9 -2
- package/dist/run-host.js.map +1 -1
- package/dist/run-wait-host.d.ts +2 -1
- package/dist/run-wait-host.d.ts.map +1 -1
- package/dist/run-wait-host.js +10 -2
- package/dist/run-wait-host.js.map +1 -1
- package/package.json +4 -4
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The host's two clocks, as ONE injectable object.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately a single source rather than two independent `now`/`wall` parameters. With two, a
|
|
5
|
+
* suite could step the wall while handing the window a clock that by construction never moved, so
|
|
6
|
+
* the clock-step cells passed just as happily against the very wall-clock window they were written
|
|
7
|
+
* to refuse. Measured on this tree: with `now` and `wall` separate, forcing the window back onto
|
|
8
|
+
* `Date.now()` left all 41 cells GREEN. One clock means a test steps THE HOST, and which member the
|
|
9
|
+
* window reads becomes the implementation's own choice, which a cell can therefore grade.
|
|
10
|
+
*/
|
|
11
|
+
export interface HostClock {
|
|
12
|
+
/**
|
|
13
|
+
* Never steps, counts only forward at its own rate, origin arbitrary. Every window is measured
|
|
14
|
+
* on this. Not `Date.now()`, and that is the whole correction above: a wall clock states what
|
|
15
|
+
* time it IS, which an operator or NTP daemon may revise at any moment, while this file asks how
|
|
16
|
+
* much time PASSED and whether this process got to run during it.
|
|
17
|
+
*/
|
|
18
|
+
monotonic(): number;
|
|
19
|
+
/** Wall time. Steps under NTP, operator edits, VM resume. Reported in the window, never measured on. */
|
|
20
|
+
wall(): number;
|
|
21
|
+
}
|
|
22
|
+
/** The real host. */
|
|
23
|
+
export declare const systemClock: HostClock;
|
|
24
|
+
/** How often the observer checks in. Short enough that a window of a few seconds carries enough
|
|
25
|
+
* ticks for a shortfall to mean something, long enough that the observer is not itself load. */
|
|
26
|
+
export declare const LOOP_TICK_MS = 250;
|
|
27
|
+
/** A point in the observer's history. Windows are differences between two of these. */
|
|
28
|
+
export interface LoopLagMark {
|
|
29
|
+
/** Monotonic. The window's length is a difference of these, never of wall readings. */
|
|
30
|
+
readonly at: number;
|
|
31
|
+
/** The WALL reading at the same instant, carried only so a step can be NOTICED. */
|
|
32
|
+
readonly wallAt: number;
|
|
33
|
+
readonly lagMs: number;
|
|
34
|
+
readonly ticks: number;
|
|
35
|
+
}
|
|
36
|
+
/** What the loop did across one window. */
|
|
37
|
+
export interface LoopLagWindow {
|
|
38
|
+
/** Wall time the window covers. */
|
|
39
|
+
readonly elapsedMs: number;
|
|
40
|
+
/** Total lateness the observer's ticks accumulated inside it. */
|
|
41
|
+
readonly lagMs: number;
|
|
42
|
+
/** Ticks the window was long enough to contain. */
|
|
43
|
+
readonly ticksExpected: number;
|
|
44
|
+
/** Ticks that actually ran. */
|
|
45
|
+
readonly ticksObserved: number;
|
|
46
|
+
/**
|
|
47
|
+
* How far the WALL clock disagreed with the monotonic window across this interval.
|
|
48
|
+
*
|
|
49
|
+
* Reported rather than acted on, and that is the point: the verdict does not depend on it, so a
|
|
50
|
+
* step cannot change the answer. It exists so an operator reading a starvation notice under a
|
|
51
|
+
* simultaneous NTP correction can see that both happened, and so the suite can assert that the
|
|
52
|
+
* step occurred and the window did NOT move with it.
|
|
53
|
+
*/
|
|
54
|
+
readonly wallSkewMs: number;
|
|
55
|
+
}
|
|
56
|
+
export interface LoopLagObserver {
|
|
57
|
+
/** Take a point to measure from. */
|
|
58
|
+
mark(): LoopLagMark;
|
|
59
|
+
/** What the loop did between `mark` and now. */
|
|
60
|
+
since(mark: LoopLagMark): LoopLagWindow;
|
|
61
|
+
}
|
|
62
|
+
/** An observer for a test to drive. Pass a `clock` whose `monotonic` STEPS and you are grading
|
|
63
|
+
* exactly the failure mode the real `systemClock` exists to remove. */
|
|
64
|
+
export declare function loopLagObserver(tickMs?: number, clock?: HostClock): LoopLagObserver & {
|
|
65
|
+
stop(): void;
|
|
66
|
+
};
|
|
67
|
+
export declare function loopLag(): LoopLagObserver;
|
|
68
|
+
/** Lag under this is never starvation, whatever share of the window it is: a 40ms window in which
|
|
69
|
+
* the loop was 12ms late is a loop that is running. */
|
|
70
|
+
export declare const STARVED_FLOOR_MS = 1000;
|
|
71
|
+
/** And it must be a real share of the window. `4` = a quarter. A read that took a minute and lost
|
|
72
|
+
* two seconds to lag was served by a loop that was running for 58 of them. */
|
|
73
|
+
export declare const STARVED_SHARE_DIVISOR = 4;
|
|
74
|
+
/** And the ticks must actually be missing. `2` = the loop kept at least half of them. A window that
|
|
75
|
+
* reports large lag while losing no ticks is a clock that moved, not a loop that stopped. */
|
|
76
|
+
export declare const STARVED_TICK_KEPT_DIVISOR = 2;
|
|
77
|
+
/**
|
|
78
|
+
* Is this failure the SHAPE a client-side deadline produces?
|
|
79
|
+
*
|
|
80
|
+
* Read off the error's CLASS, never its message. The NATS client's timeout carries the bare text
|
|
81
|
+
* `timeout`, and so do several unrelated failures in this tree and in other people's, so matching the
|
|
82
|
+
* word would classify a broker's refusal as starvation the moment somebody phrased one that way.
|
|
83
|
+
*
|
|
84
|
+
* Two accepting branches, because the client raises the deadline in two shapes: on its own for a
|
|
85
|
+
* plain API call, and wrapped in a `RequestError` when the deadline ends a request. Each has a
|
|
86
|
+
* refusing cell that differs from it only in the class involved.
|
|
87
|
+
*/
|
|
88
|
+
export declare function isDeadlineShaped(error: unknown): boolean;
|
|
89
|
+
/** Was this process demonstrably not scheduled across the window? */
|
|
90
|
+
export declare function wasStarved(w: LoopLagWindow): boolean;
|
|
91
|
+
/**
|
|
92
|
+
* What a failed pause-plane operation actually was.
|
|
93
|
+
*
|
|
94
|
+
* `fault` is TODAY'S ANSWER AND IT IS UNCHANGED: the caller raises, the interpreter records
|
|
95
|
+
* `L4000`, and a genuine broken host or handler reads exactly as it did before this file existed.
|
|
96
|
+
* The one thing that moves is the case where the error is deadline-shaped AND this process can show
|
|
97
|
+
* its own loop was not running, which was never evidence about the plane at all.
|
|
98
|
+
*/
|
|
99
|
+
export type PauseCondition = {
|
|
100
|
+
readonly condition: "starved";
|
|
101
|
+
readonly window: LoopLagWindow;
|
|
102
|
+
} | {
|
|
103
|
+
readonly condition: "fault";
|
|
104
|
+
};
|
|
105
|
+
export declare function classifyPauseFailure(error: unknown, window: LoopLagWindow): PauseCondition;
|
|
106
|
+
/**
|
|
107
|
+
* How many CONSECUTIVE starved attempts a pause operation gets before the host is declared unable
|
|
108
|
+
* to serve it.
|
|
109
|
+
*
|
|
110
|
+
* A BOUND IS MANDATORY and it is the reason this is a count rather than "retry until served": a
|
|
111
|
+
* caller that genuinely cannot be served must fail, not hang, and an unbounded retry on a host that
|
|
112
|
+
* never recovers is a run that is neither alive nor dead. Each attempt costs at least the client's
|
|
113
|
+
* own deadline before it can fail again, so the count is a real wall-clock bound and not a spin.
|
|
114
|
+
*/
|
|
115
|
+
export declare const STARVED_ATTEMPTS = 6;
|
|
116
|
+
/**
|
|
117
|
+
* A macrotask yield between starved attempts.
|
|
118
|
+
*
|
|
119
|
+
* NOT A WIDENED DEADLINE, and the difference is the whole point of this file: the deadline is
|
|
120
|
+
* unchanged, and this is a pause between re-entries of an operation that already elapsed one. It
|
|
121
|
+
* earns its place twice. A retry that re-enters synchronously occupies the very loop whose absence
|
|
122
|
+
* it is reacting to, so it would spin at full CPU during exactly the window the host needs back;
|
|
123
|
+
* and a starved loop cannot run a timer either, so waiting on one is itself a small measurement of
|
|
124
|
+
* whether scheduling has returned.
|
|
125
|
+
*/
|
|
126
|
+
export declare const STARVED_YIELD_MS = 250;
|
|
127
|
+
/**
|
|
128
|
+
* Perform a pause-plane operation, and do not let this host's own starvation be recorded as the
|
|
129
|
+
* effect's failure.
|
|
130
|
+
*
|
|
131
|
+
* THE OPERATION MUST BE IDEMPOTENT, which the pause plane's are by construction: `arm` mints
|
|
132
|
+
* idempotently-if-identical and otherwise attaches, and `settle` reads a one-use fact that is
|
|
133
|
+
* either there or not. Re-entering either after a starved attempt observes the same world.
|
|
134
|
+
*
|
|
135
|
+
* A SLEEP PROMISES AT-LEAST, NOT AT-MOST, so a starved one COMPLETES LATE rather than failing: the
|
|
136
|
+
* broker armed a real timer, it fired while this process was off the CPU, and the fact is waiting
|
|
137
|
+
* to be read. Retrying reads it and the step settles `ok`. That is the whole repair for the
|
|
138
|
+
* incident in #1508, the run in it would have completed.
|
|
139
|
+
*
|
|
140
|
+
* WHAT IS NOT SWALLOWED: a failure that is not deadline-shaped is raised on the FIRST attempt with
|
|
141
|
+
* no retry and no inspection of the loop, and a deadline-shaped failure on a loop that was running
|
|
142
|
+
* is raised the same way. Both keep `L4000` and its message exactly as they are today.
|
|
143
|
+
*/
|
|
144
|
+
export declare function servedDespiteStarvation<T>(operation: () => Promise<T>, lag: LoopLagObserver, what: string, onStarved?: (note: string) => void): Promise<T>;
|
|
145
|
+
//# sourceMappingURL=host-starvation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"host-starvation.d.ts","sourceRoot":"","sources":["../src/host-starvation.ts"],"names":[],"mappings":"AA8CA;;;;;;;;;GASG;AACH,MAAM,WAAW,SAAS;IACxB;;;;;OAKG;IACH,SAAS,IAAI,MAAM,CAAC;IACpB,wGAAwG;IACxG,IAAI,IAAI,MAAM,CAAC;CAChB;AAED,qBAAqB;AACrB,eAAO,MAAM,WAAW,EAAE,SAGzB,CAAC;AAMF;iGACiG;AACjG,eAAO,MAAM,YAAY,MAAM,CAAC;AAEhC,uFAAuF;AACvF,MAAM,WAAW,WAAW;IAC1B,uFAAuF;IACvF,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,mFAAmF;IACnF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,2CAA2C;AAC3C,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,iEAAiE;IACjE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,mDAAmD;IACnD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,+BAA+B;IAC/B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;;;;;;;OAOG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,IAAI,IAAI,WAAW,CAAC;IACpB,gDAAgD;IAChD,KAAK,CAAC,IAAI,EAAE,WAAW,GAAG,aAAa,CAAC;CACzC;AAoFD;wEACwE;AACxE,wBAAgB,eAAe,CAC7B,MAAM,GAAE,MAAqB,EAC7B,KAAK,GAAE,SAAuB,GAC7B,eAAe,GAAG;IAAE,IAAI,IAAI,IAAI,CAAA;CAAE,CAEpC;AAUD,wBAAgB,OAAO,IAAI,eAAe,CAGzC;AAMD;wDACwD;AACxD,eAAO,MAAM,gBAAgB,OAAQ,CAAC;AACtC;+EAC+E;AAC/E,eAAO,MAAM,qBAAqB,IAAI,CAAC;AACvC;8FAC8F;AAC9F,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAO3C;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAIxD;AAED,qEAAqE;AACrE,wBAAgB,UAAU,CAAC,CAAC,EAAE,aAAa,GAAG,OAAO,CASpD;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAA;CAAE,GACjE;IAAE,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAA;CAAE,CAAC;AAEpC,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,GAAG,cAAc,CAI1F;AAMD;;;;;;;;GAQG;AACH,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC;;;;;;;;;GASG;AACH,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAEpC;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,uBAAuB,CAAC,CAAC,EAC7C,SAAS,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC3B,GAAG,EAAE,eAAe,EACpB,IAAI,EAAE,MAAM,EACZ,SAAS,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAoC,GAChE,OAAO,CAAC,CAAC,CAAC,CA+BZ"}
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Did this process get scheduled? The evidence a starved host can obtain about itself, and what
|
|
3
|
+
* follows from it.
|
|
4
|
+
*
|
|
5
|
+
* THE DEFECT THIS EXISTS FOR (#1508). A `sleep` reads the pause plane while it waits. Every one of
|
|
6
|
+
* those reads rides a NATS API request with a 5s client-side deadline, and that deadline is a
|
|
7
|
+
* `setTimeout`. When the host is loaded hard enough that this process does not return to its event
|
|
8
|
+
* loop for longer than the deadline, the deadline's own timer cannot fire either, so it fires the
|
|
9
|
+
* instant the loop frees, rejects with the client's bare `timeout`, and does so even though the
|
|
10
|
+
* broker answered long ago and the reply is sitting in the socket buffer. The interpreter flattens
|
|
11
|
+
* that into `{ code: "L4000", kind: "handler-fault" }`, and the run dies with a record naming the
|
|
12
|
+
* effect as the thing that broke. Measured on this tree before the repair: the identical read
|
|
13
|
+
* returned in 1ms on a healthy loop and rejected `TimeoutError: timeout` after 9001ms with the loop
|
|
14
|
+
* blocked, with the broker healthy immediately before and immediately after.
|
|
15
|
+
*
|
|
16
|
+
* A WALL-CLOCK DEADLINE CANNOT TELL THE TWO APART, which is the whole reason this file is not a
|
|
17
|
+
* larger timeout. "The timer did not fire" and "this process was never scheduled" produce the
|
|
18
|
+
* identical observation, an elapsed deadline with no answer, so widening the deadline only moves
|
|
19
|
+
* the load at which the misattribution happens. The distinction has to come from a DIFFERENT
|
|
20
|
+
* measurement, and there is one the process can take about itself: whether its own event loop ran.
|
|
21
|
+
*
|
|
22
|
+
* TWO INDEPENDENT SIGNALS, and the independence is structural rather than hopeful:
|
|
23
|
+
*
|
|
24
|
+
* - LAG: a repeating tick records how late it was against the instant it was scheduled for. A
|
|
25
|
+
* loop that is running answers within a few ms; a loop that is blocked answers with the whole
|
|
26
|
+
* block.
|
|
27
|
+
* - TICK SHORTFALL: how many ticks the window should have contained against how many it did. A
|
|
28
|
+
* blocked loop loses ticks outright.
|
|
29
|
+
*
|
|
30
|
+
* Lag alone would be fooled by a clock that steps forward, which inflates every interval measured
|
|
31
|
+
* against it while the loop was in fact running fine and losing no ticks. The shortfall is what
|
|
32
|
+
* refuses that case, and it can only refuse it if the two are measured against DIFFERENT clocks.
|
|
33
|
+
*
|
|
34
|
+
* SO THE WINDOW IS MONOTONIC, and this is the correction a reviewer measured rather than argued.
|
|
35
|
+
* An earlier draft derived `ticksExpected` from `Date.now()`, the same reading that inflates the
|
|
36
|
+
* lag, so a forward step inflated BOTH while `ticksObserved` could not follow and a perfectly
|
|
37
|
+
* healthy loop read as starved. Measured on this tree before the repair, loop healthy throughout:
|
|
38
|
+
* a +10s step over a 1.2s window gave `{elapsed 11201, lag 10003, expected 44, observed 4}` and
|
|
39
|
+
* classified `starved`, and +60s gave `{elapsed 61201, lag 60000, expected 244, observed 4}`. Worse
|
|
40
|
+
* in the direction that matters here, a BACKWARD step during a genuine 3s block gave
|
|
41
|
+
* `{elapsed 0, lag 0, expected 0, observed 1}` and classified `fault`, which is #1508's own
|
|
42
|
+
* misattribution returning under an NTP correction. Both are gone once the window is measured on a
|
|
43
|
+
* clock that only moves forward at its own rate.
|
|
44
|
+
*/
|
|
45
|
+
import { EffectError } from "@cotal-ai/lang";
|
|
46
|
+
/** The real host. */
|
|
47
|
+
export const systemClock = {
|
|
48
|
+
monotonic: () => performance.now(),
|
|
49
|
+
wall: () => Date.now(),
|
|
50
|
+
};
|
|
51
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
52
|
+
// The measurement
|
|
53
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
54
|
+
/** How often the observer checks in. Short enough that a window of a few seconds carries enough
|
|
55
|
+
* ticks for a shortfall to mean something, long enough that the observer is not itself load. */
|
|
56
|
+
export const LOOP_TICK_MS = 250;
|
|
57
|
+
/**
|
|
58
|
+
* The observer, as a self-rescheduling unrefed tick.
|
|
59
|
+
*
|
|
60
|
+
* UNREFED, because an observer is not a reason for a process to stay alive: a run that has finished
|
|
61
|
+
* must exit, and a refed 250ms timer would hold it open forever. RESCHEDULED FROM INSIDE THE TICK
|
|
62
|
+
* rather than as an interval, because `setInterval` under a blocked loop coalesces its missed fires
|
|
63
|
+
* into one and the count of what DID run stops being readable, and the count is half the evidence.
|
|
64
|
+
*/
|
|
65
|
+
class TickLoopLag {
|
|
66
|
+
tickMs;
|
|
67
|
+
clock;
|
|
68
|
+
lagMs = 0;
|
|
69
|
+
ticks = 0;
|
|
70
|
+
timer;
|
|
71
|
+
constructor(tickMs = LOOP_TICK_MS,
|
|
72
|
+
/** The host's clock. One object, so a suite steps the HOST and not one chosen reading. */
|
|
73
|
+
clock = systemClock) {
|
|
74
|
+
this.tickMs = tickMs;
|
|
75
|
+
this.clock = clock;
|
|
76
|
+
}
|
|
77
|
+
start() {
|
|
78
|
+
if (this.timer === undefined)
|
|
79
|
+
this.schedule();
|
|
80
|
+
return this;
|
|
81
|
+
}
|
|
82
|
+
/** Stop observing. Only the tests need this; the process observer runs for the process. */
|
|
83
|
+
stop() {
|
|
84
|
+
if (this.timer !== undefined)
|
|
85
|
+
clearTimeout(this.timer);
|
|
86
|
+
this.timer = undefined;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* The reading a TICK measures itself against, as its own named function.
|
|
90
|
+
*
|
|
91
|
+
* Extracted so the wall-clock regression has a single code-only line to be restored on. It was
|
|
92
|
+
* previously reachable only by rewriting `due` and `late` together, which spans the comment
|
|
93
|
+
* between them, and this repo refuses a mutation anchor that includes prose, correctly, since a
|
|
94
|
+
* later tidy of that comment would silently disarm the mutation and nothing would go red.
|
|
95
|
+
*/
|
|
96
|
+
tickNow() {
|
|
97
|
+
return this.clock.monotonic();
|
|
98
|
+
}
|
|
99
|
+
schedule() {
|
|
100
|
+
const due = this.tickNow() + this.tickMs;
|
|
101
|
+
const timer = setTimeout(() => {
|
|
102
|
+
// Lateness against the instant this tick was SCHEDULED for, not against the previous tick:
|
|
103
|
+
// measuring tick-to-tick would report a healthy loop's own jitter as lag and would miss a
|
|
104
|
+
// block that straddled exactly one tick.
|
|
105
|
+
const late = this.tickNow() - due;
|
|
106
|
+
if (late > 0)
|
|
107
|
+
this.lagMs += late;
|
|
108
|
+
this.ticks += 1;
|
|
109
|
+
this.schedule();
|
|
110
|
+
}, this.tickMs);
|
|
111
|
+
// `unref` is present on Node's timer and absent on the DOM's; the host here is Node, and the
|
|
112
|
+
// optional call keeps this file loadable under a bundler that types it the other way.
|
|
113
|
+
timer.unref?.();
|
|
114
|
+
this.timer = timer;
|
|
115
|
+
}
|
|
116
|
+
mark() {
|
|
117
|
+
return {
|
|
118
|
+
at: this.clock.monotonic(),
|
|
119
|
+
wallAt: this.clock.wall(),
|
|
120
|
+
lagMs: this.lagMs,
|
|
121
|
+
ticks: this.ticks,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
since(mark) {
|
|
125
|
+
const elapsedMs = Math.max(0, this.clock.monotonic() - mark.at);
|
|
126
|
+
return {
|
|
127
|
+
elapsedMs,
|
|
128
|
+
lagMs: Math.max(0, this.lagMs - mark.lagMs),
|
|
129
|
+
ticksExpected: Math.floor(elapsedMs / this.tickMs),
|
|
130
|
+
ticksObserved: Math.max(0, this.ticks - mark.ticks),
|
|
131
|
+
// Signed, and NOT clamped: a backward step is as much a fact as a forward one, and the
|
|
132
|
+
// operator reading the notice wants to know which way it went.
|
|
133
|
+
wallSkewMs: (this.clock.wall() - mark.wallAt) - elapsedMs,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
/** An observer for a test to drive. Pass a `clock` whose `monotonic` STEPS and you are grading
|
|
138
|
+
* exactly the failure mode the real `systemClock` exists to remove. */
|
|
139
|
+
export function loopLagObserver(tickMs = LOOP_TICK_MS, clock = systemClock) {
|
|
140
|
+
return new TickLoopLag(tickMs, clock).start();
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* The process's observer.
|
|
144
|
+
*
|
|
145
|
+
* ONE PER PROCESS, because event-loop lag is a property of the process and not of a run, a handler
|
|
146
|
+
* or a pause: two observers would measure the same loop twice and cost two timers to do it. Started
|
|
147
|
+
* on first use so a build that never performs an effect never arms it.
|
|
148
|
+
*/
|
|
149
|
+
let processObserver;
|
|
150
|
+
export function loopLag() {
|
|
151
|
+
processObserver ??= new TickLoopLag().start();
|
|
152
|
+
return processObserver;
|
|
153
|
+
}
|
|
154
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
155
|
+
// The classification
|
|
156
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
157
|
+
/** Lag under this is never starvation, whatever share of the window it is: a 40ms window in which
|
|
158
|
+
* the loop was 12ms late is a loop that is running. */
|
|
159
|
+
export const STARVED_FLOOR_MS = 1_000;
|
|
160
|
+
/** And it must be a real share of the window. `4` = a quarter. A read that took a minute and lost
|
|
161
|
+
* two seconds to lag was served by a loop that was running for 58 of them. */
|
|
162
|
+
export const STARVED_SHARE_DIVISOR = 4;
|
|
163
|
+
/** And the ticks must actually be missing. `2` = the loop kept at least half of them. A window that
|
|
164
|
+
* reports large lag while losing no ticks is a clock that moved, not a loop that stopped. */
|
|
165
|
+
export const STARVED_TICK_KEPT_DIVISOR = 2;
|
|
166
|
+
const nameOf = (e) => {
|
|
167
|
+
const n = e?.name;
|
|
168
|
+
return typeof n === "string" ? n : undefined;
|
|
169
|
+
};
|
|
170
|
+
/**
|
|
171
|
+
* Is this failure the SHAPE a client-side deadline produces?
|
|
172
|
+
*
|
|
173
|
+
* Read off the error's CLASS, never its message. The NATS client's timeout carries the bare text
|
|
174
|
+
* `timeout`, and so do several unrelated failures in this tree and in other people's, so matching the
|
|
175
|
+
* word would classify a broker's refusal as starvation the moment somebody phrased one that way.
|
|
176
|
+
*
|
|
177
|
+
* Two accepting branches, because the client raises the deadline in two shapes: on its own for a
|
|
178
|
+
* plain API call, and wrapped in a `RequestError` when the deadline ends a request. Each has a
|
|
179
|
+
* refusing cell that differs from it only in the class involved.
|
|
180
|
+
*/
|
|
181
|
+
export function isDeadlineShaped(error) {
|
|
182
|
+
if (nameOf(error) === "TimeoutError")
|
|
183
|
+
return true;
|
|
184
|
+
if (nameOf(error) === "RequestError" && nameOf(error.cause) === "TimeoutError")
|
|
185
|
+
return true;
|
|
186
|
+
return false;
|
|
187
|
+
}
|
|
188
|
+
/** Was this process demonstrably not scheduled across the window? */
|
|
189
|
+
export function wasStarved(w) {
|
|
190
|
+
if (w.lagMs < STARVED_FLOOR_MS)
|
|
191
|
+
return false;
|
|
192
|
+
if (w.lagMs * STARVED_SHARE_DIVISOR < w.elapsedMs)
|
|
193
|
+
return false;
|
|
194
|
+
// The second signal, and the one a clock jump cannot fake: a loop that ran is a loop whose ticks
|
|
195
|
+
// happened. `ticksExpected === 0` is a window too short to have contained one, which is no
|
|
196
|
+
// evidence of starvation and is refused here rather than read as a total shortfall.
|
|
197
|
+
if (w.ticksExpected === 0)
|
|
198
|
+
return false;
|
|
199
|
+
if (w.ticksObserved * STARVED_TICK_KEPT_DIVISOR >= w.ticksExpected)
|
|
200
|
+
return false;
|
|
201
|
+
return true;
|
|
202
|
+
}
|
|
203
|
+
export function classifyPauseFailure(error, window) {
|
|
204
|
+
if (!isDeadlineShaped(error))
|
|
205
|
+
return { condition: "fault" };
|
|
206
|
+
if (!wasStarved(window))
|
|
207
|
+
return { condition: "fault" };
|
|
208
|
+
return { condition: "starved", window };
|
|
209
|
+
}
|
|
210
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
211
|
+
// The policy
|
|
212
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
213
|
+
/**
|
|
214
|
+
* How many CONSECUTIVE starved attempts a pause operation gets before the host is declared unable
|
|
215
|
+
* to serve it.
|
|
216
|
+
*
|
|
217
|
+
* A BOUND IS MANDATORY and it is the reason this is a count rather than "retry until served": a
|
|
218
|
+
* caller that genuinely cannot be served must fail, not hang, and an unbounded retry on a host that
|
|
219
|
+
* never recovers is a run that is neither alive nor dead. Each attempt costs at least the client's
|
|
220
|
+
* own deadline before it can fail again, so the count is a real wall-clock bound and not a spin.
|
|
221
|
+
*/
|
|
222
|
+
export const STARVED_ATTEMPTS = 6;
|
|
223
|
+
/**
|
|
224
|
+
* A macrotask yield between starved attempts.
|
|
225
|
+
*
|
|
226
|
+
* NOT A WIDENED DEADLINE, and the difference is the whole point of this file: the deadline is
|
|
227
|
+
* unchanged, and this is a pause between re-entries of an operation that already elapsed one. It
|
|
228
|
+
* earns its place twice. A retry that re-enters synchronously occupies the very loop whose absence
|
|
229
|
+
* it is reacting to, so it would spin at full CPU during exactly the window the host needs back;
|
|
230
|
+
* and a starved loop cannot run a timer either, so waiting on one is itself a small measurement of
|
|
231
|
+
* whether scheduling has returned.
|
|
232
|
+
*/
|
|
233
|
+
export const STARVED_YIELD_MS = 250;
|
|
234
|
+
/**
|
|
235
|
+
* Perform a pause-plane operation, and do not let this host's own starvation be recorded as the
|
|
236
|
+
* effect's failure.
|
|
237
|
+
*
|
|
238
|
+
* THE OPERATION MUST BE IDEMPOTENT, which the pause plane's are by construction: `arm` mints
|
|
239
|
+
* idempotently-if-identical and otherwise attaches, and `settle` reads a one-use fact that is
|
|
240
|
+
* either there or not. Re-entering either after a starved attempt observes the same world.
|
|
241
|
+
*
|
|
242
|
+
* A SLEEP PROMISES AT-LEAST, NOT AT-MOST, so a starved one COMPLETES LATE rather than failing: the
|
|
243
|
+
* broker armed a real timer, it fired while this process was off the CPU, and the fact is waiting
|
|
244
|
+
* to be read. Retrying reads it and the step settles `ok`. That is the whole repair for the
|
|
245
|
+
* incident in #1508, the run in it would have completed.
|
|
246
|
+
*
|
|
247
|
+
* WHAT IS NOT SWALLOWED: a failure that is not deadline-shaped is raised on the FIRST attempt with
|
|
248
|
+
* no retry and no inspection of the loop, and a deadline-shaped failure on a loop that was running
|
|
249
|
+
* is raised the same way. Both keep `L4000` and its message exactly as they are today.
|
|
250
|
+
*/
|
|
251
|
+
export async function servedDespiteStarvation(operation, lag, what, onStarved = (note) => console.error(note)) {
|
|
252
|
+
let last;
|
|
253
|
+
for (let attempt = 1;; attempt += 1) {
|
|
254
|
+
const mark = lag.mark();
|
|
255
|
+
try {
|
|
256
|
+
return await operation();
|
|
257
|
+
}
|
|
258
|
+
catch (error) {
|
|
259
|
+
const verdict = classifyPauseFailure(error, lag.since(mark));
|
|
260
|
+
// The genuine fault: out by the same door it used before this wrapper existed, carrying its
|
|
261
|
+
// own error, so the journal records what it always recorded.
|
|
262
|
+
if (verdict.condition === "fault")
|
|
263
|
+
throw error;
|
|
264
|
+
last = verdict.window;
|
|
265
|
+
const evidence = describe(last);
|
|
266
|
+
if (attempt >= STARVED_ATTEMPTS) {
|
|
267
|
+
throw new EffectError("L4025", "host-starved", `${what} could not be served: this host did not schedule the run's process across ${attempt} consecutive attempts (${evidence}). `
|
|
268
|
+
+ `The pause and its timer are intact on the plane and nothing about the program or the resource it waits on is at fault; `
|
|
269
|
+
+ `the run can be resumed once the host has capacity.`, { attempts: attempt, lagMs: last.lagMs, elapsedMs: last.elapsedMs, ticksExpected: last.ticksExpected, ticksObserved: last.ticksObserved });
|
|
270
|
+
}
|
|
271
|
+
// SAID OUT LOUD, once per starved attempt. The operator reading a slow run is the person who
|
|
272
|
+
// can act on this, and "the host is overloaded" is not deducible from a run that is merely
|
|
273
|
+
// taking a while.
|
|
274
|
+
onStarved(`${what}: attempt ${attempt} reached its client deadline while this host was not scheduling the run's process (${evidence}); retrying rather than failing the step`);
|
|
275
|
+
// Unrefed: a retry loop is not a reason for a finished process to stay alive.
|
|
276
|
+
await new Promise((r) => setTimeout(r, STARVED_YIELD_MS).unref?.());
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
const describe = (w) => `event-loop lag ${Math.round(w.lagMs)}ms of a ${Math.round(w.elapsedMs)}ms window, ${w.ticksObserved} of ${w.ticksExpected} scheduled ticks observed`;
|
|
281
|
+
//# sourceMappingURL=host-starvation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"host-starvation.js","sourceRoot":"","sources":["../src/host-starvation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAwB7C,qBAAqB;AACrB,MAAM,CAAC,MAAM,WAAW,GAAc;IACpC,SAAS,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;IAClC,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;CACvB,CAAC;AAEF,gGAAgG;AAChG,kBAAkB;AAClB,gGAAgG;AAEhG;iGACiG;AACjG,MAAM,CAAC,MAAM,YAAY,GAAG,GAAG,CAAC;AAwChC;;;;;;;GAOG;AACH,MAAM,WAAW;IAMI;IAEA;IAPX,KAAK,GAAG,CAAC,CAAC;IACV,KAAK,GAAG,CAAC,CAAC;IACV,KAAK,CAA4C;IAEzD,YACmB,SAAiB,YAAY;IAC9C,0FAA0F;IACzE,QAAmB,WAAW;QAF9B,WAAM,GAAN,MAAM,CAAuB;QAE7B,UAAK,GAAL,KAAK,CAAyB;IAC9C,CAAC;IAEJ,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;YAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,2FAA2F;IAC3F,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;YAAE,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;IACzB,CAAC;IAED;;;;;;;OAOG;IACK,OAAO;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;IAChC,CAAC;IAEO,QAAQ;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,2FAA2F;YAC3F,0FAA0F;YAC1F,yCAAyC;YACzC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC;YAClC,IAAI,IAAI,GAAG,CAAC;gBAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;YACjC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChB,6FAA6F;QAC7F,sFAAsF;QACtF,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QAChB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,IAAI;QACF,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAiB;QACrB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,OAAO;YACL,SAAS;YACT,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YAC3C,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC;YAClD,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACnD,uFAAuF;YACvF,+DAA+D;YAC/D,UAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,SAAS;SAC1D,CAAC;IACJ,CAAC;CACF;AAED;wEACwE;AACxE,MAAM,UAAU,eAAe,CAC7B,SAAiB,YAAY,EAC7B,QAAmB,WAAW;IAE9B,OAAO,IAAI,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;AAChD,CAAC;AAED;;;;;;GAMG;AACH,IAAI,eAAwC,CAAC;AAC7C,MAAM,UAAU,OAAO;IACrB,eAAe,KAAK,IAAI,WAAW,EAAE,CAAC,KAAK,EAAE,CAAC;IAC9C,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,gGAAgG;AAChG,qBAAqB;AACrB,gGAAgG;AAEhG;wDACwD;AACxD,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AACtC;+EAC+E;AAC/E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AACvC;8FAC8F;AAC9F,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAE3C,MAAM,MAAM,GAAG,CAAC,CAAU,EAAsB,EAAE;IAChD,MAAM,CAAC,GAAI,CAA2C,EAAE,IAAI,CAAC;IAC7D,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/C,CAAC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC7C,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,cAAc;QAAE,OAAO,IAAI,CAAC;IAClD,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,cAAc,IAAI,MAAM,CAAE,KAA6B,CAAC,KAAK,CAAC,KAAK,cAAc;QAAE,OAAO,IAAI,CAAC;IACrH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,UAAU,CAAC,CAAgB;IACzC,IAAI,CAAC,CAAC,KAAK,GAAG,gBAAgB;QAAE,OAAO,KAAK,CAAC;IAC7C,IAAI,CAAC,CAAC,KAAK,GAAG,qBAAqB,GAAG,CAAC,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IAChE,iGAAiG;IACjG,2FAA2F;IAC3F,oFAAoF;IACpF,IAAI,CAAC,CAAC,aAAa,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,CAAC,aAAa,GAAG,yBAAyB,IAAI,CAAC,CAAC,aAAa;QAAE,OAAO,KAAK,CAAC;IACjF,OAAO,IAAI,CAAC;AACd,CAAC;AAcD,MAAM,UAAU,oBAAoB,CAAC,KAAc,EAAE,MAAqB;IACxE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;IAC5D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;IACvD,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAC1C,CAAC;AAED,gGAAgG;AAChG,aAAa;AACb,gGAAgG;AAEhG;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAEpC;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,SAA2B,EAC3B,GAAoB,EACpB,IAAY,EACZ,YAAoC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;IAEjE,IAAI,IAA+B,CAAC;IACpC,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,oBAAoB,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC7D,4FAA4F;YAC5F,6DAA6D;YAC7D,IAAI,OAAO,CAAC,SAAS,KAAK,OAAO;gBAAE,MAAM,KAAK,CAAC;YAC/C,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;YACtB,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,OAAO,IAAI,gBAAgB,EAAE,CAAC;gBAChC,MAAM,IAAI,WAAW,CACnB,OAAO,EACP,cAAc,EACd,GAAG,IAAI,6EAA6E,OAAO,0BAA0B,QAAQ,KAAK;sBAChI,yHAAyH;sBACzH,oDAAoD,EACtD,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,CAC1I,CAAC;YACJ,CAAC;YACD,6FAA6F;YAC7F,2FAA2F;YAC3F,kBAAkB;YAClB,SAAS,CAAC,GAAG,IAAI,aAAa,OAAO,sFAAsF,QAAQ,0CAA0C,CAAC,CAAC;YAC/K,8EAA8E;YAC9E,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,CAAgB,EAAU,EAAE,CAC5C,kBAAkB,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,aAAa,OAAO,CAAC,CAAC,aAAa,2BAA2B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -31,19 +31,22 @@ const runCommand = {
|
|
|
31
31
|
kind: "command",
|
|
32
32
|
name: "run",
|
|
33
33
|
group: "Manager",
|
|
34
|
-
summary: "operate workflow runs — start, resume, list, inspect, answer (hosted by the manager)",
|
|
35
|
-
usage: "run <start --file <program> [--timeout <dur>] | resume <runId> [--local --file <program>] | ps [--endpoint <ep>] | journal <runId> [--endpoint <ep>] | answer <runId> <stepKey> [--value <json>] [--artifact <ref>] [--endpoint <ep>] [--local --by <who>]> [--local]",
|
|
34
|
+
summary: "operate workflow runs — start, resume, list, inspect, answer (hosted by the manager), revoke",
|
|
35
|
+
usage: "run <start --file <program> [--timeout <dur>] | resume <runId> [--local --file <program>] | ps [--endpoint <ep>] | journal <runId> [--endpoint <ep>] | answer <runId> <stepKey> [--value <json>] [--artifact <ref>] [--endpoint <ep>] [--local --by <who>] | revoke <runId> --local --by <who> --reason <text>> [--local [--admit-read <channels> --admit-publish <channels>]]",
|
|
36
36
|
flags: [
|
|
37
37
|
...targetFlags,
|
|
38
38
|
{ name: "file", type: "string", short: "f", value: "<program>", description: "cotal-lang program source (start; resume --local when no program is recorded)" },
|
|
39
39
|
{ name: "local", type: "boolean", description: "drive in this process instead of on the manager (bare broker, or a run the manager cannot host)" },
|
|
40
|
+
{ name: "admit-read", type: "string", value: "<channels>", description: "start --local: the channels the run may read, comma-separated patterns or `none` (required; SPEC 14.8)" },
|
|
41
|
+
{ name: "admit-publish", type: "string", value: "<channels>", description: "start --local: the channels the run may post to, comma-separated patterns or `none` (required; SPEC 14.8)" },
|
|
40
42
|
{ name: "endpoint", type: "string", value: "<ep>", description: "endpoint the run record lives under (ps, journal, answer; default: manager)" },
|
|
41
43
|
{ name: "timeout", type: "string", value: "<dur>", description: "default checkpoint timeout for this drive (default: 1h)" },
|
|
42
|
-
{ name: "by", type: "string", value: "<who>", description: "who is answering (answer --local
|
|
44
|
+
{ name: "by", type: "string", value: "<who>", description: "who is answering or revoking (answer --local, revoke; the manager records the caller for a hosted answer)" },
|
|
45
|
+
{ name: "reason", type: "string", value: "<text>", description: "revoke: why the run's admission is revoked, recorded on the marker (required)" },
|
|
43
46
|
{ name: "value", type: "string", value: "<json>", description: "checkpoint answer payload as JSON (answer)" },
|
|
44
47
|
{ name: "artifact", type: "string", value: "<ref>", description: "artifact reference attached to the answer (answer)" },
|
|
45
48
|
],
|
|
46
|
-
positionals: "<start|resume|ps|journal|answer> …",
|
|
49
|
+
positionals: "<start|resume|ps|journal|answer|revoke> …",
|
|
47
50
|
run: (args) => runWorkflowCommand(args),
|
|
48
51
|
};
|
|
49
52
|
registry.register(runCommand);
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAuG,MAAM,iBAAiB,CAAC;AACtK,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,aAAa,EACb,SAAS,GAGV,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,GAKlB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAyB,MAAM,kBAAkB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,eAAe,EACf,cAAc,EACd,sBAAsB,GAOvB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,QAAQ,EACR,UAAU,EACV,iBAAiB,EACjB,UAAU,EACV,UAAU,GAKX,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEjD,iGAAiG;AACjG,kGAAkG;AAClG,iGAAiG;AACjG,gGAAgG;AAChG,iBAAiB;AACjB,OAAO,EAAE,QAAQ,EAAgB,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,WAAW,IAAI,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,gBAAgB,IAAI,OAAO,EAAE,MAAM,eAAe,CAAC;AAE5D,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AAE3B,MAAM,UAAU,GAAY;IAC1B,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,KAAK;IACX,KAAK,EAAE,SAAS;IAChB,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAuG,MAAM,iBAAiB,CAAC;AACtK,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,aAAa,EACb,SAAS,GAGV,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,GAKlB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAyB,MAAM,kBAAkB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,eAAe,EACf,cAAc,EACd,sBAAsB,GAOvB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,QAAQ,EACR,UAAU,EACV,iBAAiB,EACjB,UAAU,EACV,UAAU,GAKX,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEjD,iGAAiG;AACjG,kGAAkG;AAClG,iGAAiG;AACjG,gGAAgG;AAChG,iBAAiB;AACjB,OAAO,EAAE,QAAQ,EAAgB,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,WAAW,IAAI,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,gBAAgB,IAAI,OAAO,EAAE,MAAM,eAAe,CAAC;AAE5D,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AAE3B,MAAM,UAAU,GAAY;IAC1B,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,KAAK;IACX,KAAK,EAAE,SAAS;IAChB,OAAO,EAAE,8FAA8F;IACvG,KAAK,EACH,gXAAgX;IAClX,KAAK,EAAE;QACL,GAAG,WAAW;QACd,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,+EAA+E,EAAE;QAC9J,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,iGAAiG,EAAE;QAClJ,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,wGAAwG,EAAE;QAClL,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,2GAA2G,EAAE;QACxL,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,6EAA6E,EAAE;QAC/I,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,yDAAyD,EAAE;QAC3H,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,2GAA2G,EAAE;QACxK,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,+EAA+E,EAAE;QACjJ,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,4CAA4C,EAAE;QAC7G,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,oDAAoD,EAAE;KACxH;IACD,WAAW,EAAE,2CAA2C;IACxD,GAAG,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC;CACxC,CAAC;AAEF,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC"}
|
package/dist/mesh-handler.d.ts
CHANGED
|
@@ -1,15 +1,23 @@
|
|
|
1
|
-
import { type CheckpointRef, type CheckpointSettleFact, type EpCaller } from "@cotal-ai/core";
|
|
1
|
+
import { type CheckpointRef, type CheckpointSettleFact, type EpCaller, type RunAdmissionView } from "@cotal-ai/core";
|
|
2
2
|
import type { NatsConnection } from "@nats-io/transport-node";
|
|
3
3
|
import type { JetStreamClient, JetStreamManager } from "@nats-io/jetstream";
|
|
4
4
|
import { type KV } from "@nats-io/kv";
|
|
5
|
-
import { type AskRequest, type AgentHandleValue, type ChannelHandleValue, type ConclaveRequest, type WaitRequest, type CheckpointRaw, type CheckpointRequest, type EffectContext, type JournalEntry, type MonitorRequest, type NotifyRequest, type SleepRequest, type SpawnRequest, type TurnRequest, type TurnResultValue } from "@cotal-ai/lang";
|
|
5
|
+
import { type AskRequest, type AgentHandleValue, type ChannelHandleValue, type ConclaveRequest, type WaitRequest, type CheckpointRaw, type CheckpointRequest, type EffectContext, type JournalEntry, type MonitorRequest, type NotifyRequest, type ObserveRequest, type SleepRequest, type SpawnRequest, type TurnRequest, type TurnResultValue } from "@cotal-ai/lang";
|
|
6
6
|
import type { RunPauseHost } from "./run-pause-host.js";
|
|
7
7
|
import type { RunWaitHost } from "./run-wait-host.js";
|
|
8
|
+
import { type LoopLagObserver } from "./host-starvation.js";
|
|
8
9
|
import type { RunScopeAuthority } from "./run-scope-authority.js";
|
|
9
10
|
export interface RunMeshServices {
|
|
10
11
|
readonly pauses: RunPauseHost;
|
|
11
12
|
readonly waits: RunWaitHost;
|
|
12
13
|
readonly authority: RunScopeAuthority;
|
|
14
|
+
/**
|
|
15
|
+
* The run's ADMISSION (SPEC 14.8), re-read leader-served at every channel effect this handler
|
|
16
|
+
* performs on its own connection: a conclave's registration and membership writes, checked
|
|
17
|
+
* AFTER a generated channel name is derived. The wait host holds the same reader for the
|
|
18
|
+
* channel reads it mediates. A mediated run with no reader admits no channel effect.
|
|
19
|
+
*/
|
|
20
|
+
readonly admission: () => Promise<RunAdmissionView>;
|
|
13
21
|
}
|
|
14
22
|
/** Who this handler acts as, and where. All of it is the DRIVER's identity, not the program's. */
|
|
15
23
|
export interface MeshHandlerBinding {
|
|
@@ -94,7 +102,33 @@ export declare class MeshHandler {
|
|
|
94
102
|
private readonly watcher;
|
|
95
103
|
private readonly clock;
|
|
96
104
|
private readonly services?;
|
|
97
|
-
|
|
105
|
+
/**
|
|
106
|
+
* How this handler learns whether its own process was scheduled (#1508). Injected so a suite
|
|
107
|
+
* can drive a starved loop deterministically; the default is the process-wide observer, which
|
|
108
|
+
* is the only honest source outside a test.
|
|
109
|
+
*/
|
|
110
|
+
private readonly lag;
|
|
111
|
+
/**
|
|
112
|
+
* Where the operator notice for an absorbed starvation goes (#1508). A daemon routes it to its
|
|
113
|
+
* own log rather than stderr, and a suite counts it: an absorbed starvation is otherwise
|
|
114
|
+
* INVISIBLE, and a cell asserting "the sleep completed" cannot tell a completion that survived
|
|
115
|
+
* starvation from one that was never starved at all.
|
|
116
|
+
*/
|
|
117
|
+
private readonly onStarved;
|
|
118
|
+
constructor(nc: NatsConnection, kv: KV, js: JetStreamClient, jsm: JetStreamManager, binding: MeshHandlerBinding, watcher: SettleWatcher, clock?: () => number, services?: RunMeshServices | undefined,
|
|
119
|
+
/**
|
|
120
|
+
* How this handler learns whether its own process was scheduled (#1508). Injected so a suite
|
|
121
|
+
* can drive a starved loop deterministically; the default is the process-wide observer, which
|
|
122
|
+
* is the only honest source outside a test.
|
|
123
|
+
*/
|
|
124
|
+
lag?: LoopLagObserver,
|
|
125
|
+
/**
|
|
126
|
+
* Where the operator notice for an absorbed starvation goes (#1508). A daemon routes it to its
|
|
127
|
+
* own log rather than stderr, and a suite counts it: an absorbed starvation is otherwise
|
|
128
|
+
* INVISIBLE, and a cell asserting "the sleep completed" cannot tell a completion that survived
|
|
129
|
+
* starvation from one that was never starved at all.
|
|
130
|
+
*/
|
|
131
|
+
onStarved?: (note: string) => void);
|
|
98
132
|
/**
|
|
99
133
|
* The resolved manager service, memoized as a PROMISE so concurrent branches share one describe
|
|
100
134
|
* round-trip — and dropped on failure, so a resolve that lost to a manager restart is retried by
|
|
@@ -263,6 +297,24 @@ export declare class MeshHandler {
|
|
|
263
297
|
* and `null` is the whole answer.
|
|
264
298
|
*/
|
|
265
299
|
sleep(req: SleepRequest, ctx: EffectContext): Promise<null>;
|
|
300
|
+
/**
|
|
301
|
+
* `waitUntil`'s cadence, as a durable pause nobody answers.
|
|
302
|
+
*
|
|
303
|
+
* THE SAME PLANE AS `sleep`, for the reason `sleep` gives for reusing the checkpoint plane: a
|
|
304
|
+
* durable pause with a deadline, a token that survives a crash and a one-use settle is what this
|
|
305
|
+
* needs, and a second timer mechanism would be a second thing to get wrong. What differs is that
|
|
306
|
+
* a `waitUntil` parks MANY times under one step, so each observation's pause takes its own
|
|
307
|
+
* DERIVED token (`observe-<n>`), exactly as an ask's re-attempts and a wait's second deadline do.
|
|
308
|
+
* Deriving rather than remembering is what makes it recoverable: a resumed run re-derives the
|
|
309
|
+
* same token from the request id and the attempt index, and re-attaches to the pause the crashed
|
|
310
|
+
* attempt armed instead of arming a second one.
|
|
311
|
+
*
|
|
312
|
+
* THIS HANDLER NEVER OBSERVES ANYTHING. The probe is the program's and the interpreter calls it;
|
|
313
|
+
* all that happens here is the waiting. So there is nothing to bind and nothing to read back:
|
|
314
|
+
* the observations are the journal's, written by the interpreter, and this returns only whether
|
|
315
|
+
* there was time left.
|
|
316
|
+
*/
|
|
317
|
+
observe(req: ObserveRequest, ctx: EffectContext): Promise<boolean>;
|
|
266
318
|
/**
|
|
267
319
|
* `checkpoint` is the same durable pause with somebody expected to answer it.
|
|
268
320
|
*
|
|
@@ -595,6 +647,7 @@ export declare class MeshHandler {
|
|
|
595
647
|
* raised rather than waited on.
|
|
596
648
|
*/
|
|
597
649
|
private arm;
|
|
650
|
+
private armOnce;
|
|
598
651
|
/** Push a live deadline out — the idle window restarting. The heartbeat CAS-advances the
|
|
599
652
|
* generation before replacing the timer, so a fire from the old one no-ops rather than racing. */
|
|
600
653
|
private push;
|
|
@@ -635,6 +688,7 @@ export declare class MeshHandler {
|
|
|
635
688
|
* already in the past.
|
|
636
689
|
*/
|
|
637
690
|
private settle;
|
|
691
|
+
private settleOnce;
|
|
638
692
|
/** Reject with `Cancelled` the moment this branch's signal fires, then claim the pause so its
|
|
639
693
|
* armed schedule cannot fire into a run that has moved on. The claim also writes the one-use
|
|
640
694
|
* settle, which is what lets an abandoned `awaitSettle` poll loop see a fact and end. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mesh-handler.d.ts","sourceRoot":"","sources":["../src/mesh-handler.ts"],"names":[],"mappings":"AAmBA,OAAO,EA4CL,KAAK,aAAa,EAClB,KAAK,oBAAoB,EAGzB,KAAK,QAAQ,
|
|
1
|
+
{"version":3,"file":"mesh-handler.d.ts","sourceRoot":"","sources":["../src/mesh-handler.ts"],"names":[],"mappings":"AAmBA,OAAO,EA4CL,KAAK,aAAa,EAClB,KAAK,oBAAoB,EAGzB,KAAK,QAAQ,EAKb,KAAK,gBAAgB,EAItB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAO,KAAK,EAAE,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EASL,KAAK,UAAU,EACf,KAAK,gBAAgB,EAErB,KAAK,kBAAkB,EACvB,KAAK,eAAe,EACpB,KAAK,WAAW,EAChB,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,eAAe,EAGrB,MAAM,gBAAgB,CAAC;AAExB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAoC,KAAK,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC9F,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAElE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;;;;;OAKG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACrD;AAED,kGAAkG;AAClG,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,4DAA4D;IAC5D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;;;;;;;;;OASG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IACxE;;;;;;;;OAQG;IACH,QAAQ,CAAC,wBAAwB,EAAE,MAAM,CAAC;CAC3C;AAED;;;;;;;;GAQG;AACH,qBAAa,uBAAwB,SAAQ,KAAK;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM;IAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS;gBAApD,KAAK,EAAE,MAAM,EAAW,QAAQ,EAAE,MAAM,GAAG,SAAS;CAS1E;AAED,kGAAkG;AAClG,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,WAAW,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;CAChE;AAED;;;;GAIG;AAEH,qBAAa,WAAW;IAEpB,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAC1B;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,SAAS;gBApBT,EAAE,EAAE,cAAc,EAClB,EAAE,EAAE,EAAE,EACN,EAAE,EAAE,eAAe,EACnB,GAAG,EAAE,gBAAgB,EACrB,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,aAAa,EACtB,KAAK,GAAE,MAAM,MAAyB,EACtC,QAAQ,CAAC,EAAE,eAAe,YAAA;IAC3C;;;;OAIG;IACc,GAAG,GAAE,eAA2B;IACjD;;;;;OAKG;IACc,SAAS,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAoC;IAGpF;;;;OAIG;IACH,OAAO,CAAC,cAAc,CAAuC;IAC7D,OAAO,CAAC,OAAO;IASf,+FAA+F;IAC/F,OAAO,CAAC,OAAO,CAAqC;IACpD,OAAO,CAAC,SAAS;IAYjB,OAAO,CAAC,cAAc,CAA0B;IAChD,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa,CAA0B;IAC/C,OAAO,CAAC,eAAe;IAQvB,OAAO,CAAC,cAAc,CAA0B;IAChD,OAAO,CAAC,eAAe;IAQvB,iGAAiG;YACnF,YAAY;IAI1B;;;uEAGmE;IACnE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmC;IAE7D;;;gGAG4F;IAC5F,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA4I;IACnK,oHAAoH;IACpH,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IACxD,uGAAuG;IACvG,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAE/C;;;mGAG+F;IAC/F,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuE;IAEpG;+FAC2F;IAC3F,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkC;IAE5D;8EAC0E;IAC1E,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAErE;;uFAEmF;IACnF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqC;IAE/D,GAAG,IAAI,MAAM;IAIb;;;;;;;;;OASG;IACG,OAAO,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAWlE;;;;;;OAMG;IACG,oBAAoB,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAW3E,kBAAkB,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,CAAC,GAAG,IAAI;IAK7E;;;;;;;;OAQG;IACH,OAAO,CAAC,YAAY;IA4CpB;;;;;;;;;;;;OAYG;YACW,aAAa;IAqC3B,2FAA2F;IAC3F,OAAO,CAAC,cAAc;IAMtB;uGACmG;IACnG,OAAO,CAAC,iBAAiB;IAKzB;;;;;;;;;;;;OAYG;IACG,SAAS,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDhE;;;;;;;;;;;;;;;;;;;OAmBG;YACW,eAAe;IAmB7B;;;;;;;;;;;;;;OAcG;YACW,cAAc;IA0C5B;;;;;;;;OAQG;YACW,iBAAiB;IAS/B;;;;;;;;OAQG;IACG,KAAK,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBjE;;;;;;;;;;;;;;;;OAgBG;IACG,OAAO,CAAC,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IAyBxE;;;;;;;;;;;;;;;;;;;OAmBG;IACG,UAAU,CAAC,GAAG,EAAE,iBAAiB,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IA+CpF;;;;;;;;;;;;;;OAcG;IACG,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IA4IzE;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;YACW,QAAQ;IA4CtB;;;;;;;;;;;;;;;;;;OAkBG;YACW,WAAW;IAmDzB;;;;;;;;;;;;;;;;;OAiBG;IACG,MAAM,CAAC,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBnE;;;;;;;;;;;;;;;;;;OAkBG;IACG,KAAK,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAoH7E;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAUvB;;sGAEkG;YACpF,YAAY;IAU1B;;;;;;;;;;;;;;;;;;OAkBG;IACG,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAmJ1E;;;;mFAI+E;IAC/E;;;;;OAKG;IACH,OAAO,CAAC,cAAc;YAQR,WAAW;IA2DzB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,GAAG,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IAmEhE;;;;;;;OAOG;IACH,OAAO,CAAC,OAAO;IAYf;;;;;OAKG;IACH,OAAO,CAAC,WAAW;YAOL,QAAQ;IAetB;;;;;;;;;;;;OAYG;YACW,eAAe;IA4B7B;;;;;;;OAOG;YACW,WAAW;IAiCzB;;;;;;;;;;;;;;;OAeG;IACG,OAAO,CAAC,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IASrE;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACG,YAAY,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAezF;;;;;;;;;;OAUG;IACG,aAAa,CAAC,IAAI,EAAE,eAAe,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAU7E;;;sEAGkE;YACpD,YAAY;IA6B1B;;;;sFAIkF;YACpE,mBAAmB;IAiDjC;;;mGAG+F;YACjF,eAAe;IAwB7B;;;8DAG0D;YAC5C,YAAY;IAY1B;;;;;;;;;;;;;;;;;;;;;OAqBG;YACW,GAAG;YAMH,OAAO;IAsDrB;uGACmG;YACrF,IAAI;IAWlB;;;;;;;;;;;;;;OAcG;YACW,QAAQ;IA2BtB;;;;oGAIgG;YAClF,OAAO;IAcrB;;4EAEwE;YAC1D,WAAW;IA+BzB,kFAAkF;YACpE,SAAS;IAevB;;;;;;;OAOG;YACW,MAAM;YAQN,UAAU;IA2BxB;;8FAE0F;IAC1F,OAAO,CAAC,eAAe;IAiBvB,0EAA0E;YAC5D,SAAS;CAQxB;AAED,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAWtE;;;;;GAKG;AACH,4EAA4E;AAC5E,KAAK,cAAc,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAE7D;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,cAAc,CAqB3E;AAkED;;4FAE4F;AAC5F,wBAAgB,SAAS,CAAC,GAAG,EAAE,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CASpE;AAyOD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE;IAAE,EAAE,EAAE,EAAE,CAAC;IAAC,EAAE,EAAE,eAAe,CAAC;IAAC,GAAG,EAAE,gBAAgB,CAAA;CAAE,EAC5D,OAAO,EAAE,IAAI,CAAC,kBAAkB,EAAE,OAAO,GAAG,UAAU,GAAG,YAAY,GAAG,OAAO,CAAC,EAChF,OAAO,EAAE,SAAS,YAAY,EAAE,GAC/B,OAAO,CAAC,MAAM,EAAE,CAAC,CAWnB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,GAAG,MAAM,EAAE,CAYjF;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,gBAAiB,YAAW,aAAa;IAElD,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAFN,GAAG,EAAE,gBAAgB,EACrB,KAAK,EAAE,MAAM,EACb,MAAM,SAAe;IAGlC,WAAW,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,oBAAoB,CAAC;CASrE"}
|