@cotal-ai/runtime 0.48.2 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/host-starvation.d.ts +183 -0
- package/dist/host-starvation.d.ts.map +1 -0
- package/dist/host-starvation.js +329 -0
- package/dist/host-starvation.js.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -6
- package/dist/index.js.map +1 -1
- package/dist/mesh-handler.d.ts +125 -3
- package/dist/mesh-handler.d.ts.map +1 -1
- package/dist/mesh-handler.js +305 -9
- package/dist/mesh-handler.js.map +1 -1
- package/dist/migrate.js +13 -0
- package/dist/migrate.js.map +1 -1
- package/dist/run-command.d.ts +22 -1
- package/dist/run-command.d.ts.map +1 -1
- package/dist/run-command.js +142 -14
- package/dist/run-command.js.map +1 -1
- package/dist/run-effect-host.d.ts +2 -2
- package/dist/run-effect-host.d.ts.map +1 -1
- package/dist/run-effect-host.js +9 -3
- package/dist/run-effect-host.js.map +1 -1
- package/dist/run-host.d.ts +7 -0
- package/dist/run-host.d.ts.map +1 -1
- package/dist/run-host.js +25 -3
- package/dist/run-host.js.map +1 -1
- package/dist/run-wait-host.d.ts +2 -1
- package/dist/run-wait-host.d.ts.map +1 -1
- package/dist/run-wait-host.js +10 -2
- package/dist/run-wait-host.js.map +1 -1
- package/package.json +7 -4
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The host's two clocks, as ONE injectable object.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately a single source rather than two independent `now`/`wall` parameters. With two, a
|
|
5
|
+
* suite could step the wall while handing the window a clock that by construction never moved, so
|
|
6
|
+
* the clock-step cells passed just as happily against the very wall-clock window they were written
|
|
7
|
+
* to refuse. Measured on this tree: with `now` and `wall` separate, forcing the window back onto
|
|
8
|
+
* `Date.now()` left all 41 cells GREEN. One clock means a test steps THE HOST, and which member the
|
|
9
|
+
* window reads becomes the implementation's own choice, which a cell can therefore grade.
|
|
10
|
+
*/
|
|
11
|
+
export interface HostClock {
|
|
12
|
+
/**
|
|
13
|
+
* Never steps, counts only forward at its own rate, origin arbitrary. Every window is measured
|
|
14
|
+
* on this. Not `Date.now()`, and that is the whole correction above: a wall clock states what
|
|
15
|
+
* time it IS, which an operator or NTP daemon may revise at any moment, while this file asks how
|
|
16
|
+
* much time PASSED and whether this process got to run during it.
|
|
17
|
+
*/
|
|
18
|
+
monotonic(): number;
|
|
19
|
+
/** Wall time. Steps under NTP, operator edits, VM resume. Reported in the window, never measured on. */
|
|
20
|
+
wall(): number;
|
|
21
|
+
}
|
|
22
|
+
/** The real host. */
|
|
23
|
+
export declare const systemClock: HostClock;
|
|
24
|
+
/** How often the observer checks in. Short enough that a window of a few seconds carries enough
|
|
25
|
+
* ticks for a shortfall to mean something, long enough that the observer is not itself load. */
|
|
26
|
+
export declare const LOOP_TICK_MS = 250;
|
|
27
|
+
/** A point in the observer's history. Windows are differences between two of these. */
|
|
28
|
+
export interface LoopLagMark {
|
|
29
|
+
/** Monotonic. The window's length is a difference of these, never of wall readings. */
|
|
30
|
+
readonly at: number;
|
|
31
|
+
/** The WALL reading at the same instant, carried only so a step can be NOTICED. */
|
|
32
|
+
readonly wallAt: number;
|
|
33
|
+
readonly lagMs: number;
|
|
34
|
+
readonly ticks: number;
|
|
35
|
+
}
|
|
36
|
+
/** What the loop did across one window. */
|
|
37
|
+
export interface LoopLagWindow {
|
|
38
|
+
/** Wall time the window covers. */
|
|
39
|
+
readonly elapsedMs: number;
|
|
40
|
+
/** Total lateness the observer's ticks accumulated inside it. */
|
|
41
|
+
readonly lagMs: number;
|
|
42
|
+
/** Ticks the window was long enough to contain. */
|
|
43
|
+
readonly ticksExpected: number;
|
|
44
|
+
/** Ticks that actually ran. */
|
|
45
|
+
readonly ticksObserved: number;
|
|
46
|
+
/**
|
|
47
|
+
* How far the WALL clock disagreed with the monotonic window across this interval.
|
|
48
|
+
*
|
|
49
|
+
* Reported rather than acted on, and that is the point: the verdict does not depend on it, so a
|
|
50
|
+
* step cannot change the answer. It exists so an operator reading a starvation notice under a
|
|
51
|
+
* simultaneous NTP correction can see that both happened, and so the suite can assert that the
|
|
52
|
+
* step occurred and the window did NOT move with it.
|
|
53
|
+
*/
|
|
54
|
+
readonly wallSkewMs: number;
|
|
55
|
+
}
|
|
56
|
+
export interface LoopLagObserver {
|
|
57
|
+
/** Take a point to measure from. */
|
|
58
|
+
mark(): LoopLagMark;
|
|
59
|
+
/** What the loop did between `mark` and now. */
|
|
60
|
+
since(mark: LoopLagMark): LoopLagWindow;
|
|
61
|
+
}
|
|
62
|
+
/** An observer for a test to drive. Pass a `clock` whose `monotonic` STEPS and you are grading
|
|
63
|
+
* exactly the failure mode the real `systemClock` exists to remove. */
|
|
64
|
+
export declare function loopLagObserver(tickMs?: number, clock?: HostClock): LoopLagObserver & {
|
|
65
|
+
stop(): void;
|
|
66
|
+
};
|
|
67
|
+
export declare function loopLag(): LoopLagObserver;
|
|
68
|
+
/** Lag under this is never starvation, whatever share of the window it is: a 40ms window in which
|
|
69
|
+
* the loop was 12ms late is a loop that is running. */
|
|
70
|
+
export declare const STARVED_FLOOR_MS = 1000;
|
|
71
|
+
/** And it must be a real share of the window. `4` = a quarter. A read that took a minute and lost
|
|
72
|
+
* two seconds to lag was served by a loop that was running for 58 of them. */
|
|
73
|
+
export declare const STARVED_SHARE_DIVISOR = 4;
|
|
74
|
+
/** And the ticks must actually be missing. `2` = the loop kept at least half of them. A window that
|
|
75
|
+
* reports large lag while losing no ticks is a clock that moved, not a loop that stopped. */
|
|
76
|
+
export declare const STARVED_TICK_KEPT_DIVISOR = 2;
|
|
77
|
+
/**
|
|
78
|
+
* Is this failure the SHAPE a client-side deadline produces?
|
|
79
|
+
*
|
|
80
|
+
* Read off the error's CLASS, never its message. The NATS client's timeout carries the bare text
|
|
81
|
+
* `timeout`, and so do several unrelated failures in this tree and in other people's, so matching the
|
|
82
|
+
* word would classify a broker's refusal as starvation the moment somebody phrased one that way.
|
|
83
|
+
*
|
|
84
|
+
* Two accepting branches, because the client raises the deadline in two shapes: on its own for a
|
|
85
|
+
* plain API call, and wrapped in a `RequestError` when the deadline ends a request. Each has a
|
|
86
|
+
* refusing cell that differs from it only in the class involved.
|
|
87
|
+
*/
|
|
88
|
+
export declare function isDeadlineShaped(error: unknown): boolean;
|
|
89
|
+
/** Was this process demonstrably not scheduled across the window? */
|
|
90
|
+
export declare function wasStarved(w: LoopLagWindow): boolean;
|
|
91
|
+
/**
|
|
92
|
+
* What a failed pause-plane operation actually was.
|
|
93
|
+
*
|
|
94
|
+
* `fault` is TODAY'S ANSWER AND IT IS UNCHANGED: the caller raises, the interpreter records
|
|
95
|
+
* `L4000`, and a genuine broken host or handler reads exactly as it did before this file existed.
|
|
96
|
+
* The two things that move are the cases where the error is DEADLINE-SHAPED, which was never
|
|
97
|
+
* evidence about the plane's health at all, only about a reply that did not arrive in time:
|
|
98
|
+
*
|
|
99
|
+
* - `starved`, when this process can show its own loop was not running (#1508). The deadline
|
|
100
|
+
* could not fire because nothing could.
|
|
101
|
+
* - `transient`, when the loop WAS running (#1619). The deadline fired honestly and the reply
|
|
102
|
+
* was merely late, which is a property of one request and not of the pause behind it.
|
|
103
|
+
*/
|
|
104
|
+
export type PauseCondition = {
|
|
105
|
+
readonly condition: "starved";
|
|
106
|
+
readonly window: LoopLagWindow;
|
|
107
|
+
} | {
|
|
108
|
+
readonly condition: "transient";
|
|
109
|
+
readonly window: LoopLagWindow;
|
|
110
|
+
} | {
|
|
111
|
+
readonly condition: "fault";
|
|
112
|
+
};
|
|
113
|
+
export declare function classifyPauseFailure(error: unknown, window: LoopLagWindow): PauseCondition;
|
|
114
|
+
/**
|
|
115
|
+
* How many CONSECUTIVE starved attempts a pause operation gets before the host is declared unable
|
|
116
|
+
* to serve it.
|
|
117
|
+
*
|
|
118
|
+
* A BOUND IS MANDATORY and it is the reason this is a count rather than "retry until served": a
|
|
119
|
+
* caller that genuinely cannot be served must fail, not hang, and an unbounded retry on a host that
|
|
120
|
+
* never recovers is a run that is neither alive nor dead. Each attempt costs at least the client's
|
|
121
|
+
* own deadline before it can fail again, so the count is a real wall-clock bound and not a spin.
|
|
122
|
+
*/
|
|
123
|
+
export declare const STARVED_ATTEMPTS = 6;
|
|
124
|
+
/**
|
|
125
|
+
* A macrotask yield between starved attempts.
|
|
126
|
+
*
|
|
127
|
+
* NOT A WIDENED DEADLINE, and the difference is the whole point of this file: the deadline is
|
|
128
|
+
* unchanged, and this is a pause between re-entries of an operation that already elapsed one. It
|
|
129
|
+
* earns its place twice. A retry that re-enters synchronously occupies the very loop whose absence
|
|
130
|
+
* it is reacting to, so it would spin at full CPU during exactly the window the host needs back;
|
|
131
|
+
* and a starved loop cannot run a timer either, so waiting on one is itself a small measurement of
|
|
132
|
+
* whether scheduling has returned.
|
|
133
|
+
*/
|
|
134
|
+
export declare const STARVED_YIELD_MS = 250;
|
|
135
|
+
/**
|
|
136
|
+
* How many CONSECUTIVE late replies a pause operation gets before the plane is declared unable to
|
|
137
|
+
* answer it (#1619).
|
|
138
|
+
*
|
|
139
|
+
* COUNTED SEPARATELY from {@link STARVED_ATTEMPTS}, and not merged with it, because the two are
|
|
140
|
+
* different diagnoses with different remedies and an operator acts on which one they got: one says
|
|
141
|
+
* give this host capacity, the other says the broker is not answering. Counting them apart also
|
|
142
|
+
* keeps the whole call bounded no matter how the two interleave — neither counter is ever reset, so
|
|
143
|
+
* an alternating sequence cannot retry forever between them.
|
|
144
|
+
*/
|
|
145
|
+
export declare const POLL_ATTEMPTS = 6;
|
|
146
|
+
/**
|
|
147
|
+
* The first wait between late replies, DOUBLED each time and capped at {@link POLL_BACKOFF_MAX_MS}.
|
|
148
|
+
*
|
|
149
|
+
* Backoff rather than the flat yield starvation uses, because the two are waiting for different
|
|
150
|
+
* things to recover. A starved loop needs the CPU back, which the next macrotask already measures;
|
|
151
|
+
* a plane answering late is usually a broker under momentary load, and re-issuing at the same
|
|
152
|
+
* cadence adds to exactly the queue that is already behind. Each attempt additionally costs the
|
|
153
|
+
* client's own deadline before it can fail again, so the bound is real wall-clock time either way.
|
|
154
|
+
*/
|
|
155
|
+
export declare const POLL_BACKOFF_MS = 250;
|
|
156
|
+
export declare const POLL_BACKOFF_MAX_MS = 2000;
|
|
157
|
+
/**
|
|
158
|
+
* Perform a pause-plane operation, and do not let this host's own starvation, or one late reply
|
|
159
|
+
* from the plane, be recorded as the effect's failure.
|
|
160
|
+
*
|
|
161
|
+
* THE OPERATION MUST BE IDEMPOTENT, which the pause plane's are by construction: `arm` mints
|
|
162
|
+
* idempotently-if-identical and otherwise attaches, and `settle` reads a one-use fact that is
|
|
163
|
+
* either there or not. Re-entering either after a starved or unanswered attempt observes the same
|
|
164
|
+
* world.
|
|
165
|
+
*
|
|
166
|
+
* A SLEEP PROMISES AT-LEAST, NOT AT-MOST, so a starved one COMPLETES LATE rather than failing: the
|
|
167
|
+
* broker armed a real timer, it fired while this process was off the CPU, and the fact is waiting
|
|
168
|
+
* to be read. Retrying reads it and the step settles `ok`. That is the whole repair for the
|
|
169
|
+
* incident in #1508, the run in it would have completed.
|
|
170
|
+
*
|
|
171
|
+
* AND A PARKED STEP IS NOT ITS POLLS (#1619). While a pause is parked this host issues roughly one
|
|
172
|
+
* 5s plane read per second for the step's whole duration, so the chance that ONE of them gets a
|
|
173
|
+
* late reply grows with how long the step waits — measured, the asks over 7.5 minutes died and the
|
|
174
|
+
* ones under 4.5 did not. A late reply says nothing about the pause, which is durable on the plane
|
|
175
|
+
* and still answerable, so it is re-read rather than raised.
|
|
176
|
+
*
|
|
177
|
+
* WHAT IS NOT SWALLOWED: a failure that is not deadline-shaped is raised on the FIRST attempt with
|
|
178
|
+
* no retry and no inspection of the loop, keeping `L4000` and its message exactly as they are
|
|
179
|
+
* today. And neither retry is unbounded: each condition carries its own count, and exhausting
|
|
180
|
+
* either fails under a code that names what was measured.
|
|
181
|
+
*/
|
|
182
|
+
export declare function servedDespiteStarvation<T>(operation: () => Promise<T>, lag: LoopLagObserver, what: string, onStarved?: (note: string) => void): Promise<T>;
|
|
183
|
+
//# sourceMappingURL=host-starvation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"host-starvation.d.ts","sourceRoot":"","sources":["../src/host-starvation.ts"],"names":[],"mappings":"AA8CA;;;;;;;;;GASG;AACH,MAAM,WAAW,SAAS;IACxB;;;;;OAKG;IACH,SAAS,IAAI,MAAM,CAAC;IACpB,wGAAwG;IACxG,IAAI,IAAI,MAAM,CAAC;CAChB;AAED,qBAAqB;AACrB,eAAO,MAAM,WAAW,EAAE,SAGzB,CAAC;AAMF;iGACiG;AACjG,eAAO,MAAM,YAAY,MAAM,CAAC;AAEhC,uFAAuF;AACvF,MAAM,WAAW,WAAW;IAC1B,uFAAuF;IACvF,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,mFAAmF;IACnF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,2CAA2C;AAC3C,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,iEAAiE;IACjE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,mDAAmD;IACnD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,+BAA+B;IAC/B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;;;;;;;OAOG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,IAAI,IAAI,WAAW,CAAC;IACpB,gDAAgD;IAChD,KAAK,CAAC,IAAI,EAAE,WAAW,GAAG,aAAa,CAAC;CACzC;AAoFD;wEACwE;AACxE,wBAAgB,eAAe,CAC7B,MAAM,GAAE,MAAqB,EAC7B,KAAK,GAAE,SAAuB,GAC7B,eAAe,GAAG;IAAE,IAAI,IAAI,IAAI,CAAA;CAAE,CAEpC;AAUD,wBAAgB,OAAO,IAAI,eAAe,CAGzC;AAMD;wDACwD;AACxD,eAAO,MAAM,gBAAgB,OAAQ,CAAC;AACtC;+EAC+E;AAC/E,eAAO,MAAM,qBAAqB,IAAI,CAAC;AACvC;8FAC8F;AAC9F,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAO3C;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAIxD;AAED,qEAAqE;AACrE,wBAAgB,UAAU,CAAC,CAAC,EAAE,aAAa,GAAG,OAAO,CASpD;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAA;CAAE,GACjE;IAAE,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAA;CAAE,GACnE;IAAE,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAA;CAAE,CAAC;AAEpC,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,GAAG,cAAc,CAI1F;AAMD;;;;;;;;GAQG;AACH,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC;;;;;;;;;GASG;AACH,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAEpC;;;;;;;;;GASG;AACH,eAAO,MAAM,aAAa,IAAI,CAAC;AAE/B;;;;;;;;GAQG;AACH,eAAO,MAAM,eAAe,MAAM,CAAC;AACnC,eAAO,MAAM,mBAAmB,OAAQ,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,uBAAuB,CAAC,CAAC,EAC7C,SAAS,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC3B,GAAG,EAAE,eAAe,EACpB,IAAI,EAAE,MAAM,EACZ,SAAS,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAoC,GAChE,OAAO,CAAC,CAAC,CAAC,CAsDZ"}
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Did this process get scheduled? The evidence a starved host can obtain about itself, and what
|
|
3
|
+
* follows from it.
|
|
4
|
+
*
|
|
5
|
+
* THE DEFECT THIS EXISTS FOR (#1508). A `sleep` reads the pause plane while it waits. Every one of
|
|
6
|
+
* those reads rides a NATS API request with a 5s client-side deadline, and that deadline is a
|
|
7
|
+
* `setTimeout`. When the host is loaded hard enough that this process does not return to its event
|
|
8
|
+
* loop for longer than the deadline, the deadline's own timer cannot fire either, so it fires the
|
|
9
|
+
* instant the loop frees, rejects with the client's bare `timeout`, and does so even though the
|
|
10
|
+
* broker answered long ago and the reply is sitting in the socket buffer. The interpreter flattens
|
|
11
|
+
* that into `{ code: "L4000", kind: "handler-fault" }`, and the run dies with a record naming the
|
|
12
|
+
* effect as the thing that broke. Measured on this tree before the repair: the identical read
|
|
13
|
+
* returned in 1ms on a healthy loop and rejected `TimeoutError: timeout` after 9001ms with the loop
|
|
14
|
+
* blocked, with the broker healthy immediately before and immediately after.
|
|
15
|
+
*
|
|
16
|
+
* A WALL-CLOCK DEADLINE CANNOT TELL THE TWO APART, which is the whole reason this file is not a
|
|
17
|
+
* larger timeout. "The timer did not fire" and "this process was never scheduled" produce the
|
|
18
|
+
* identical observation, an elapsed deadline with no answer, so widening the deadline only moves
|
|
19
|
+
* the load at which the misattribution happens. The distinction has to come from a DIFFERENT
|
|
20
|
+
* measurement, and there is one the process can take about itself: whether its own event loop ran.
|
|
21
|
+
*
|
|
22
|
+
* TWO INDEPENDENT SIGNALS, and the independence is structural rather than hopeful:
|
|
23
|
+
*
|
|
24
|
+
* - LAG: a repeating tick records how late it was against the instant it was scheduled for. A
|
|
25
|
+
* loop that is running answers within a few ms; a loop that is blocked answers with the whole
|
|
26
|
+
* block.
|
|
27
|
+
* - TICK SHORTFALL: how many ticks the window should have contained against how many it did. A
|
|
28
|
+
* blocked loop loses ticks outright.
|
|
29
|
+
*
|
|
30
|
+
* Lag alone would be fooled by a clock that steps forward, which inflates every interval measured
|
|
31
|
+
* against it while the loop was in fact running fine and losing no ticks. The shortfall is what
|
|
32
|
+
* refuses that case, and it can only refuse it if the two are measured against DIFFERENT clocks.
|
|
33
|
+
*
|
|
34
|
+
* SO THE WINDOW IS MONOTONIC, and this is the correction a reviewer measured rather than argued.
|
|
35
|
+
* An earlier draft derived `ticksExpected` from `Date.now()`, the same reading that inflates the
|
|
36
|
+
* lag, so a forward step inflated BOTH while `ticksObserved` could not follow and a perfectly
|
|
37
|
+
* healthy loop read as starved. Measured on this tree before the repair, loop healthy throughout:
|
|
38
|
+
* a +10s step over a 1.2s window gave `{elapsed 11201, lag 10003, expected 44, observed 4}` and
|
|
39
|
+
* classified `starved`, and +60s gave `{elapsed 61201, lag 60000, expected 244, observed 4}`. Worse
|
|
40
|
+
* in the direction that matters here, a BACKWARD step during a genuine 3s block gave
|
|
41
|
+
* `{elapsed 0, lag 0, expected 0, observed 1}` and classified `fault`, which is #1508's own
|
|
42
|
+
* misattribution returning under an NTP correction. Both are gone once the window is measured on a
|
|
43
|
+
* clock that only moves forward at its own rate.
|
|
44
|
+
*/
|
|
45
|
+
import { EffectError } from "@cotal-ai/lang";
|
|
46
|
+
/** The real host. */
|
|
47
|
+
export const systemClock = {
|
|
48
|
+
monotonic: () => performance.now(),
|
|
49
|
+
wall: () => Date.now(),
|
|
50
|
+
};
|
|
51
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
52
|
+
// The measurement
|
|
53
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
54
|
+
/** How often the observer checks in. Short enough that a window of a few seconds carries enough
|
|
55
|
+
* ticks for a shortfall to mean something, long enough that the observer is not itself load. */
|
|
56
|
+
export const LOOP_TICK_MS = 250;
|
|
57
|
+
/**
|
|
58
|
+
* The observer, as a self-rescheduling unrefed tick.
|
|
59
|
+
*
|
|
60
|
+
* UNREFED, because an observer is not a reason for a process to stay alive: a run that has finished
|
|
61
|
+
* must exit, and a refed 250ms timer would hold it open forever. RESCHEDULED FROM INSIDE THE TICK
|
|
62
|
+
* rather than as an interval, because `setInterval` under a blocked loop coalesces its missed fires
|
|
63
|
+
* into one and the count of what DID run stops being readable, and the count is half the evidence.
|
|
64
|
+
*/
|
|
65
|
+
class TickLoopLag {
|
|
66
|
+
tickMs;
|
|
67
|
+
clock;
|
|
68
|
+
lagMs = 0;
|
|
69
|
+
ticks = 0;
|
|
70
|
+
timer;
|
|
71
|
+
constructor(tickMs = LOOP_TICK_MS,
|
|
72
|
+
/** The host's clock. One object, so a suite steps the HOST and not one chosen reading. */
|
|
73
|
+
clock = systemClock) {
|
|
74
|
+
this.tickMs = tickMs;
|
|
75
|
+
this.clock = clock;
|
|
76
|
+
}
|
|
77
|
+
start() {
|
|
78
|
+
if (this.timer === undefined)
|
|
79
|
+
this.schedule();
|
|
80
|
+
return this;
|
|
81
|
+
}
|
|
82
|
+
/** Stop observing. Only the tests need this; the process observer runs for the process. */
|
|
83
|
+
stop() {
|
|
84
|
+
if (this.timer !== undefined)
|
|
85
|
+
clearTimeout(this.timer);
|
|
86
|
+
this.timer = undefined;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* The reading a TICK measures itself against, as its own named function.
|
|
90
|
+
*
|
|
91
|
+
* Extracted so the wall-clock regression has a single code-only line to be restored on. It was
|
|
92
|
+
* previously reachable only by rewriting `due` and `late` together, which spans the comment
|
|
93
|
+
* between them, and this repo refuses a mutation anchor that includes prose, correctly, since a
|
|
94
|
+
* later tidy of that comment would silently disarm the mutation and nothing would go red.
|
|
95
|
+
*/
|
|
96
|
+
tickNow() {
|
|
97
|
+
return this.clock.monotonic();
|
|
98
|
+
}
|
|
99
|
+
schedule() {
|
|
100
|
+
const due = this.tickNow() + this.tickMs;
|
|
101
|
+
const timer = setTimeout(() => {
|
|
102
|
+
// Lateness against the instant this tick was SCHEDULED for, not against the previous tick:
|
|
103
|
+
// measuring tick-to-tick would report a healthy loop's own jitter as lag and would miss a
|
|
104
|
+
// block that straddled exactly one tick.
|
|
105
|
+
const late = this.tickNow() - due;
|
|
106
|
+
if (late > 0)
|
|
107
|
+
this.lagMs += late;
|
|
108
|
+
this.ticks += 1;
|
|
109
|
+
this.schedule();
|
|
110
|
+
}, this.tickMs);
|
|
111
|
+
// `unref` is present on Node's timer and absent on the DOM's; the host here is Node, and the
|
|
112
|
+
// optional call keeps this file loadable under a bundler that types it the other way.
|
|
113
|
+
timer.unref?.();
|
|
114
|
+
this.timer = timer;
|
|
115
|
+
}
|
|
116
|
+
mark() {
|
|
117
|
+
return {
|
|
118
|
+
at: this.clock.monotonic(),
|
|
119
|
+
wallAt: this.clock.wall(),
|
|
120
|
+
lagMs: this.lagMs,
|
|
121
|
+
ticks: this.ticks,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
since(mark) {
|
|
125
|
+
const elapsedMs = Math.max(0, this.clock.monotonic() - mark.at);
|
|
126
|
+
return {
|
|
127
|
+
elapsedMs,
|
|
128
|
+
lagMs: Math.max(0, this.lagMs - mark.lagMs),
|
|
129
|
+
ticksExpected: Math.floor(elapsedMs / this.tickMs),
|
|
130
|
+
ticksObserved: Math.max(0, this.ticks - mark.ticks),
|
|
131
|
+
// Signed, and NOT clamped: a backward step is as much a fact as a forward one, and the
|
|
132
|
+
// operator reading the notice wants to know which way it went.
|
|
133
|
+
wallSkewMs: (this.clock.wall() - mark.wallAt) - elapsedMs,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
/** An observer for a test to drive. Pass a `clock` whose `monotonic` STEPS and you are grading
|
|
138
|
+
* exactly the failure mode the real `systemClock` exists to remove. */
|
|
139
|
+
export function loopLagObserver(tickMs = LOOP_TICK_MS, clock = systemClock) {
|
|
140
|
+
return new TickLoopLag(tickMs, clock).start();
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* The process's observer.
|
|
144
|
+
*
|
|
145
|
+
* ONE PER PROCESS, because event-loop lag is a property of the process and not of a run, a handler
|
|
146
|
+
* or a pause: two observers would measure the same loop twice and cost two timers to do it. Started
|
|
147
|
+
* on first use so a build that never performs an effect never arms it.
|
|
148
|
+
*/
|
|
149
|
+
let processObserver;
|
|
150
|
+
export function loopLag() {
|
|
151
|
+
processObserver ??= new TickLoopLag().start();
|
|
152
|
+
return processObserver;
|
|
153
|
+
}
|
|
154
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
155
|
+
// The classification
|
|
156
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
157
|
+
/** Lag under this is never starvation, whatever share of the window it is: a 40ms window in which
|
|
158
|
+
* the loop was 12ms late is a loop that is running. */
|
|
159
|
+
export const STARVED_FLOOR_MS = 1_000;
|
|
160
|
+
/** And it must be a real share of the window. `4` = a quarter. A read that took a minute and lost
|
|
161
|
+
* two seconds to lag was served by a loop that was running for 58 of them. */
|
|
162
|
+
export const STARVED_SHARE_DIVISOR = 4;
|
|
163
|
+
/** And the ticks must actually be missing. `2` = the loop kept at least half of them. A window that
|
|
164
|
+
* reports large lag while losing no ticks is a clock that moved, not a loop that stopped. */
|
|
165
|
+
export const STARVED_TICK_KEPT_DIVISOR = 2;
|
|
166
|
+
const nameOf = (e) => {
|
|
167
|
+
const n = e?.name;
|
|
168
|
+
return typeof n === "string" ? n : undefined;
|
|
169
|
+
};
|
|
170
|
+
/**
|
|
171
|
+
* Is this failure the SHAPE a client-side deadline produces?
|
|
172
|
+
*
|
|
173
|
+
* Read off the error's CLASS, never its message. The NATS client's timeout carries the bare text
|
|
174
|
+
* `timeout`, and so do several unrelated failures in this tree and in other people's, so matching the
|
|
175
|
+
* word would classify a broker's refusal as starvation the moment somebody phrased one that way.
|
|
176
|
+
*
|
|
177
|
+
* Two accepting branches, because the client raises the deadline in two shapes: on its own for a
|
|
178
|
+
* plain API call, and wrapped in a `RequestError` when the deadline ends a request. Each has a
|
|
179
|
+
* refusing cell that differs from it only in the class involved.
|
|
180
|
+
*/
|
|
181
|
+
export function isDeadlineShaped(error) {
|
|
182
|
+
if (nameOf(error) === "TimeoutError")
|
|
183
|
+
return true;
|
|
184
|
+
if (nameOf(error) === "RequestError" && nameOf(error.cause) === "TimeoutError")
|
|
185
|
+
return true;
|
|
186
|
+
return false;
|
|
187
|
+
}
|
|
188
|
+
/** Was this process demonstrably not scheduled across the window? */
|
|
189
|
+
export function wasStarved(w) {
|
|
190
|
+
if (w.lagMs < STARVED_FLOOR_MS)
|
|
191
|
+
return false;
|
|
192
|
+
if (w.lagMs * STARVED_SHARE_DIVISOR < w.elapsedMs)
|
|
193
|
+
return false;
|
|
194
|
+
// The second signal, and the one a clock jump cannot fake: a loop that ran is a loop whose ticks
|
|
195
|
+
// happened. `ticksExpected === 0` is a window too short to have contained one, which is no
|
|
196
|
+
// evidence of starvation and is refused here rather than read as a total shortfall.
|
|
197
|
+
if (w.ticksExpected === 0)
|
|
198
|
+
return false;
|
|
199
|
+
if (w.ticksObserved * STARVED_TICK_KEPT_DIVISOR >= w.ticksExpected)
|
|
200
|
+
return false;
|
|
201
|
+
return true;
|
|
202
|
+
}
|
|
203
|
+
export function classifyPauseFailure(error, window) {
|
|
204
|
+
if (!isDeadlineShaped(error))
|
|
205
|
+
return { condition: "fault" };
|
|
206
|
+
if (!wasStarved(window))
|
|
207
|
+
return { condition: "transient", window };
|
|
208
|
+
return { condition: "starved", window };
|
|
209
|
+
}
|
|
210
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
211
|
+
// The policy
|
|
212
|
+
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
|
213
|
+
/**
|
|
214
|
+
* How many CONSECUTIVE starved attempts a pause operation gets before the host is declared unable
|
|
215
|
+
* to serve it.
|
|
216
|
+
*
|
|
217
|
+
* A BOUND IS MANDATORY and it is the reason this is a count rather than "retry until served": a
|
|
218
|
+
* caller that genuinely cannot be served must fail, not hang, and an unbounded retry on a host that
|
|
219
|
+
* never recovers is a run that is neither alive nor dead. Each attempt costs at least the client's
|
|
220
|
+
* own deadline before it can fail again, so the count is a real wall-clock bound and not a spin.
|
|
221
|
+
*/
|
|
222
|
+
export const STARVED_ATTEMPTS = 6;
|
|
223
|
+
/**
|
|
224
|
+
* A macrotask yield between starved attempts.
|
|
225
|
+
*
|
|
226
|
+
* NOT A WIDENED DEADLINE, and the difference is the whole point of this file: the deadline is
|
|
227
|
+
* unchanged, and this is a pause between re-entries of an operation that already elapsed one. It
|
|
228
|
+
* earns its place twice. A retry that re-enters synchronously occupies the very loop whose absence
|
|
229
|
+
* it is reacting to, so it would spin at full CPU during exactly the window the host needs back;
|
|
230
|
+
* and a starved loop cannot run a timer either, so waiting on one is itself a small measurement of
|
|
231
|
+
* whether scheduling has returned.
|
|
232
|
+
*/
|
|
233
|
+
export const STARVED_YIELD_MS = 250;
|
|
234
|
+
/**
|
|
235
|
+
* How many CONSECUTIVE late replies a pause operation gets before the plane is declared unable to
|
|
236
|
+
* answer it (#1619).
|
|
237
|
+
*
|
|
238
|
+
* COUNTED SEPARATELY from {@link STARVED_ATTEMPTS}, and not merged with it, because the two are
|
|
239
|
+
* different diagnoses with different remedies and an operator acts on which one they got: one says
|
|
240
|
+
* give this host capacity, the other says the broker is not answering. Counting them apart also
|
|
241
|
+
* keeps the whole call bounded no matter how the two interleave — neither counter is ever reset, so
|
|
242
|
+
* an alternating sequence cannot retry forever between them.
|
|
243
|
+
*/
|
|
244
|
+
export const POLL_ATTEMPTS = 6;
|
|
245
|
+
/**
|
|
246
|
+
* The first wait between late replies, DOUBLED each time and capped at {@link POLL_BACKOFF_MAX_MS}.
|
|
247
|
+
*
|
|
248
|
+
* Backoff rather than the flat yield starvation uses, because the two are waiting for different
|
|
249
|
+
* things to recover. A starved loop needs the CPU back, which the next macrotask already measures;
|
|
250
|
+
* a plane answering late is usually a broker under momentary load, and re-issuing at the same
|
|
251
|
+
* cadence adds to exactly the queue that is already behind. Each attempt additionally costs the
|
|
252
|
+
* client's own deadline before it can fail again, so the bound is real wall-clock time either way.
|
|
253
|
+
*/
|
|
254
|
+
export const POLL_BACKOFF_MS = 250;
|
|
255
|
+
export const POLL_BACKOFF_MAX_MS = 2_000;
|
|
256
|
+
/**
|
|
257
|
+
* Perform a pause-plane operation, and do not let this host's own starvation, or one late reply
|
|
258
|
+
* from the plane, be recorded as the effect's failure.
|
|
259
|
+
*
|
|
260
|
+
* THE OPERATION MUST BE IDEMPOTENT, which the pause plane's are by construction: `arm` mints
|
|
261
|
+
* idempotently-if-identical and otherwise attaches, and `settle` reads a one-use fact that is
|
|
262
|
+
* either there or not. Re-entering either after a starved or unanswered attempt observes the same
|
|
263
|
+
* world.
|
|
264
|
+
*
|
|
265
|
+
* A SLEEP PROMISES AT-LEAST, NOT AT-MOST, so a starved one COMPLETES LATE rather than failing: the
|
|
266
|
+
* broker armed a real timer, it fired while this process was off the CPU, and the fact is waiting
|
|
267
|
+
* to be read. Retrying reads it and the step settles `ok`. That is the whole repair for the
|
|
268
|
+
* incident in #1508, the run in it would have completed.
|
|
269
|
+
*
|
|
270
|
+
* AND A PARKED STEP IS NOT ITS POLLS (#1619). While a pause is parked this host issues roughly one
|
|
271
|
+
* 5s plane read per second for the step's whole duration, so the chance that ONE of them gets a
|
|
272
|
+
* late reply grows with how long the step waits — measured, the asks over 7.5 minutes died and the
|
|
273
|
+
* ones under 4.5 did not. A late reply says nothing about the pause, which is durable on the plane
|
|
274
|
+
* and still answerable, so it is re-read rather than raised.
|
|
275
|
+
*
|
|
276
|
+
* WHAT IS NOT SWALLOWED: a failure that is not deadline-shaped is raised on the FIRST attempt with
|
|
277
|
+
* no retry and no inspection of the loop, keeping `L4000` and its message exactly as they are
|
|
278
|
+
* today. And neither retry is unbounded: each condition carries its own count, and exhausting
|
|
279
|
+
* either fails under a code that names what was measured.
|
|
280
|
+
*/
|
|
281
|
+
export async function servedDespiteStarvation(operation, lag, what, onStarved = (note) => console.error(note)) {
|
|
282
|
+
let last;
|
|
283
|
+
let starved = 0;
|
|
284
|
+
let unanswered = 0;
|
|
285
|
+
for (let attempt = 1;; attempt += 1) {
|
|
286
|
+
const mark = lag.mark();
|
|
287
|
+
try {
|
|
288
|
+
return await operation();
|
|
289
|
+
}
|
|
290
|
+
catch (error) {
|
|
291
|
+
const verdict = classifyPauseFailure(error, lag.since(mark));
|
|
292
|
+
// The genuine fault: out by the same door it used before this wrapper existed, carrying its
|
|
293
|
+
// own error, so the journal records what it always recorded.
|
|
294
|
+
if (verdict.condition === "fault")
|
|
295
|
+
throw error;
|
|
296
|
+
last = verdict.window;
|
|
297
|
+
const evidence = describe(last);
|
|
298
|
+
if (verdict.condition === "transient") {
|
|
299
|
+
unanswered += 1;
|
|
300
|
+
if (unanswered >= POLL_ATTEMPTS) {
|
|
301
|
+
throw new EffectError("L4026", "pause-unanswered", `${what} could not be served: the pause plane did not answer ${unanswered} consecutive reads before their client deadline, while this host's own event loop was running (${evidence}). `
|
|
302
|
+
+ `The pause and its timer are intact on the plane and nothing about the program or the resource it waits on is at fault; `
|
|
303
|
+
+ `the run can be resumed once the broker is answering.`, { attempts: unanswered, lagMs: last.lagMs, elapsedMs: last.elapsedMs, ticksExpected: last.ticksExpected, ticksObserved: last.ticksObserved });
|
|
304
|
+
}
|
|
305
|
+
// SAID OUT LOUD, for the same reason the starved notice is: a run that is merely taking a
|
|
306
|
+
// while looks identical to one whose plane reads keep timing out, and the operator is the
|
|
307
|
+
// person who can act on the difference.
|
|
308
|
+
onStarved(`${what}: attempt ${attempt} reached its client deadline while this host WAS scheduling the run's process (${evidence}); the plane's reply was late, retrying rather than failing the step`);
|
|
309
|
+
// Unrefed, as below: a retry loop is not a reason for a finished process to stay alive.
|
|
310
|
+
await new Promise((r) => setTimeout(r, Math.min(POLL_BACKOFF_MS * 2 ** (unanswered - 1), POLL_BACKOFF_MAX_MS)).unref?.());
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
starved += 1;
|
|
314
|
+
if (starved >= STARVED_ATTEMPTS) {
|
|
315
|
+
throw new EffectError("L4025", "host-starved", `${what} could not be served: this host did not schedule the run's process across ${starved} consecutive attempts (${evidence}). `
|
|
316
|
+
+ `The pause and its timer are intact on the plane and nothing about the program or the resource it waits on is at fault; `
|
|
317
|
+
+ `the run can be resumed once the host has capacity.`, { attempts: starved, lagMs: last.lagMs, elapsedMs: last.elapsedMs, ticksExpected: last.ticksExpected, ticksObserved: last.ticksObserved });
|
|
318
|
+
}
|
|
319
|
+
// SAID OUT LOUD, once per starved attempt. The operator reading a slow run is the person who
|
|
320
|
+
// can act on this, and "the host is overloaded" is not deducible from a run that is merely
|
|
321
|
+
// taking a while.
|
|
322
|
+
onStarved(`${what}: attempt ${attempt} reached its client deadline while this host was not scheduling the run's process (${evidence}); retrying rather than failing the step`);
|
|
323
|
+
// Unrefed: a retry loop is not a reason for a finished process to stay alive.
|
|
324
|
+
await new Promise((r) => setTimeout(r, STARVED_YIELD_MS).unref?.());
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
const describe = (w) => `event-loop lag ${Math.round(w.lagMs)}ms of a ${Math.round(w.elapsedMs)}ms window, ${w.ticksObserved} of ${w.ticksExpected} scheduled ticks observed`;
|
|
329
|
+
//# sourceMappingURL=host-starvation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"host-starvation.js","sourceRoot":"","sources":["../src/host-starvation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAwB7C,qBAAqB;AACrB,MAAM,CAAC,MAAM,WAAW,GAAc;IACpC,SAAS,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;IAClC,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;CACvB,CAAC;AAEF,gGAAgG;AAChG,kBAAkB;AAClB,gGAAgG;AAEhG;iGACiG;AACjG,MAAM,CAAC,MAAM,YAAY,GAAG,GAAG,CAAC;AAwChC;;;;;;;GAOG;AACH,MAAM,WAAW;IAMI;IAEA;IAPX,KAAK,GAAG,CAAC,CAAC;IACV,KAAK,GAAG,CAAC,CAAC;IACV,KAAK,CAA4C;IAEzD,YACmB,SAAiB,YAAY;IAC9C,0FAA0F;IACzE,QAAmB,WAAW;QAF9B,WAAM,GAAN,MAAM,CAAuB;QAE7B,UAAK,GAAL,KAAK,CAAyB;IAC9C,CAAC;IAEJ,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;YAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,2FAA2F;IAC3F,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;YAAE,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;IACzB,CAAC;IAED;;;;;;;OAOG;IACK,OAAO;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;IAChC,CAAC;IAEO,QAAQ;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,2FAA2F;YAC3F,0FAA0F;YAC1F,yCAAyC;YACzC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC;YAClC,IAAI,IAAI,GAAG,CAAC;gBAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;YACjC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChB,6FAA6F;QAC7F,sFAAsF;QACtF,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QAChB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,IAAI;QACF,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAiB;QACrB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,OAAO;YACL,SAAS;YACT,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YAC3C,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC;YAClD,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACnD,uFAAuF;YACvF,+DAA+D;YAC/D,UAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,SAAS;SAC1D,CAAC;IACJ,CAAC;CACF;AAED;wEACwE;AACxE,MAAM,UAAU,eAAe,CAC7B,SAAiB,YAAY,EAC7B,QAAmB,WAAW;IAE9B,OAAO,IAAI,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;AAChD,CAAC;AAED;;;;;;GAMG;AACH,IAAI,eAAwC,CAAC;AAC7C,MAAM,UAAU,OAAO;IACrB,eAAe,KAAK,IAAI,WAAW,EAAE,CAAC,KAAK,EAAE,CAAC;IAC9C,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,gGAAgG;AAChG,qBAAqB;AACrB,gGAAgG;AAEhG;wDACwD;AACxD,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AACtC;+EAC+E;AAC/E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AACvC;8FAC8F;AAC9F,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAE3C,MAAM,MAAM,GAAG,CAAC,CAAU,EAAsB,EAAE;IAChD,MAAM,CAAC,GAAI,CAA2C,EAAE,IAAI,CAAC;IAC7D,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/C,CAAC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC7C,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,cAAc;QAAE,OAAO,IAAI,CAAC;IAClD,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,cAAc,IAAI,MAAM,CAAE,KAA6B,CAAC,KAAK,CAAC,KAAK,cAAc;QAAE,OAAO,IAAI,CAAC;IACrH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,UAAU,CAAC,CAAgB;IACzC,IAAI,CAAC,CAAC,KAAK,GAAG,gBAAgB;QAAE,OAAO,KAAK,CAAC;IAC7C,IAAI,CAAC,CAAC,KAAK,GAAG,qBAAqB,GAAG,CAAC,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IAChE,iGAAiG;IACjG,2FAA2F;IAC3F,oFAAoF;IACpF,IAAI,CAAC,CAAC,aAAa,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,CAAC,aAAa,GAAG,yBAAyB,IAAI,CAAC,CAAC,aAAa;QAAE,OAAO,KAAK,CAAC;IACjF,OAAO,IAAI,CAAC;AACd,CAAC;AAoBD,MAAM,UAAU,oBAAoB,CAAC,KAAc,EAAE,MAAqB;IACxE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;IAC5D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;IACnE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAC1C,CAAC;AAED,gGAAgG;AAChG,aAAa;AACb,gGAAgG;AAEhG;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAEpC;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC;AAE/B;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,CAAC;AACnC,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,SAA2B,EAC3B,GAAoB,EACpB,IAAY,EACZ,YAAoC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;IAEjE,IAAI,IAA+B,CAAC;IACpC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,oBAAoB,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC7D,4FAA4F;YAC5F,6DAA6D;YAC7D,IAAI,OAAO,CAAC,SAAS,KAAK,OAAO;gBAAE,MAAM,KAAK,CAAC;YAC/C,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;YACtB,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,OAAO,CAAC,SAAS,KAAK,WAAW,EAAE,CAAC;gBACtC,UAAU,IAAI,CAAC,CAAC;gBAChB,IAAI,UAAU,IAAI,aAAa,EAAE,CAAC;oBAChC,MAAM,IAAI,WAAW,CACnB,OAAO,EACP,kBAAkB,EAClB,GAAG,IAAI,wDAAwD,UAAU,kGAAkG,QAAQ,KAAK;0BACtL,yHAAyH;0BACzH,sDAAsD,EACxD,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,CAC7I,CAAC;gBACJ,CAAC;gBACD,0FAA0F;gBAC1F,0FAA0F;gBAC1F,wCAAwC;gBACxC,SAAS,CAAC,GAAG,IAAI,aAAa,OAAO,kFAAkF,QAAQ,sEAAsE,CAAC,CAAC;gBACvM,wFAAwF;gBACxF,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC1H,SAAS;YACX,CAAC;YACD,OAAO,IAAI,CAAC,CAAC;YACb,IAAI,OAAO,IAAI,gBAAgB,EAAE,CAAC;gBAChC,MAAM,IAAI,WAAW,CACnB,OAAO,EACP,cAAc,EACd,GAAG,IAAI,6EAA6E,OAAO,0BAA0B,QAAQ,KAAK;sBAChI,yHAAyH;sBACzH,oDAAoD,EACtD,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,CAC1I,CAAC;YACJ,CAAC;YACD,6FAA6F;YAC7F,2FAA2F;YAC3F,kBAAkB;YAClB,SAAS,CAAC,GAAG,IAAI,aAAa,OAAO,sFAAsF,QAAQ,0CAA0C,CAAC,CAAC;YAC/K,8EAA8E;YAC9E,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,CAAgB,EAAU,EAAE,CAC5C,kBAAkB,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,aAAa,OAAO,CAAC,CAAC,aAAa,2BAA2B,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -10,11 +10,11 @@
|
|
|
10
10
|
*/
|
|
11
11
|
export { RunJournalStore, RunJournalUnavailable } from "./journal-store.js";
|
|
12
12
|
export { startRun, driveRun, PauseToken, type RunLease, type DriveRequest, type DriveOutcome, type AdoptingHandler, type SeatAdoptingHandler } from "./run-driver.js";
|
|
13
|
-
export { MeshHandler, EpfSettleWatcher, CheckpointAnswerMissing, waitConsumerName, waitConsumerConfig, rearmOutstandingPauses, outstandingPauseTokens, readSupervise, spawnArgs, type MeshHandlerBinding, type SettleWatcher, } from "./mesh-handler.js";
|
|
13
|
+
export { MeshHandler, EpfSettleWatcher, CheckpointAnswerMissing, waitConsumerName, waitConsumerConfig, rearmOutstandingPauses, outstandingPauseTokens, readSupervise, spawnArgs, canonicalCwd, type MeshHandlerBinding, type SettleWatcher, } from "./mesh-handler.js";
|
|
14
14
|
export { resolveCheckpoint, locateOpenCheckpoint, answerOpenCheckpoint, openCheckpointToken, CheckpointNotOpen, type OpenCheckpoint, type ResolveCheckpointDeps, type ResolveCheckpointRequest, type ResolveCheckpointResult, } from "./resolve-checkpoint.js";
|
|
15
15
|
export { renderRunContext, UnrenderableNotice, type RunContextRender } from "./run-context.js";
|
|
16
16
|
export { migrateRun, commitMigration, migrationSeats, MigrationNotAdmissible, type MigrateRequest, type MigrateReport, type MigrateOrphan, type MigrateOverrides, type MigrateDivergence, type OrphanVerdict, } from "./migrate.js";
|
|
17
17
|
export { planFork, commitFork, ForkNotAdmissible, CutJournal, CutReached, type ForkRequest, type ForkPlan, type ForkRefusal, type ForkCommitResult, } from "./fork.js";
|
|
18
18
|
export { runWorkflow } from "./run-command.js";
|
|
19
|
-
export { cotalLangRunHost } from "./run-host.js";
|
|
19
|
+
export { cotalLangRunHost, journalOutcomeOf } from "./run-host.js";
|
|
20
20
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,KAAK,eAAe,EAAE,KAAK,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtK,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,aAAa,EACb,SAAS,EACT,KAAK,kBAAkB,EACvB,KAAK,aAAa,GACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,wBAAwB,EAC7B,KAAK,uBAAuB,GAC7B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,KAAK,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,eAAe,EACf,cAAc,EACd,sBAAsB,EACtB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,aAAa,GACnB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,QAAQ,EACR,UAAU,EACV,iBAAiB,EACjB,UAAU,EACV,UAAU,EACV,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,WAAW,EAChB,KAAK,gBAAgB,GACtB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,KAAK,eAAe,EAAE,KAAK,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtK,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,aAAa,EACb,SAAS,EACT,YAAY,EACZ,KAAK,kBAAkB,EACvB,KAAK,aAAa,GACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,wBAAwB,EAC7B,KAAK,uBAAuB,GAC7B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,KAAK,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,eAAe,EACf,cAAc,EACd,sBAAsB,EACtB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,aAAa,GACnB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,QAAQ,EACR,UAAU,EACV,iBAAiB,EACjB,UAAU,EACV,UAAU,EACV,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,WAAW,EAChB,KAAK,gBAAgB,GACtB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -10,13 +10,13 @@
|
|
|
10
10
|
*/
|
|
11
11
|
export { RunJournalStore, RunJournalUnavailable } from "./journal-store.js";
|
|
12
12
|
export { startRun, driveRun, PauseToken } from "./run-driver.js";
|
|
13
|
-
export { MeshHandler, EpfSettleWatcher, CheckpointAnswerMissing, waitConsumerName, waitConsumerConfig, rearmOutstandingPauses, outstandingPauseTokens, readSupervise, spawnArgs, } from "./mesh-handler.js";
|
|
13
|
+
export { MeshHandler, EpfSettleWatcher, CheckpointAnswerMissing, waitConsumerName, waitConsumerConfig, rearmOutstandingPauses, outstandingPauseTokens, readSupervise, spawnArgs, canonicalCwd, } from "./mesh-handler.js";
|
|
14
14
|
export { resolveCheckpoint, locateOpenCheckpoint, answerOpenCheckpoint, openCheckpointToken, CheckpointNotOpen, } from "./resolve-checkpoint.js";
|
|
15
15
|
export { renderRunContext, UnrenderableNotice } from "./run-context.js";
|
|
16
16
|
export { migrateRun, commitMigration, migrationSeats, MigrationNotAdmissible, } from "./migrate.js";
|
|
17
17
|
export { planFork, commitFork, ForkNotAdmissible, CutJournal, CutReached, } from "./fork.js";
|
|
18
18
|
export { runWorkflow } from "./run-command.js";
|
|
19
|
-
export { cotalLangRunHost } from "./run-host.js";
|
|
19
|
+
export { cotalLangRunHost, journalOutcomeOf } from "./run-host.js";
|
|
20
20
|
// Self-register `cotal run` — the workflow-run operator surface — and the `run-host` the manager
|
|
21
21
|
// drives runs through (SPEC 14.3). Importing this package from a composition root (bin/run.ts) is
|
|
22
22
|
// what puts the command on the CLI and the host in the manager's reach; library users who import
|
|
@@ -31,19 +31,22 @@ const runCommand = {
|
|
|
31
31
|
kind: "command",
|
|
32
32
|
name: "run",
|
|
33
33
|
group: "Manager",
|
|
34
|
-
summary: "operate workflow runs — start, resume, list, inspect, answer (hosted by the manager)",
|
|
35
|
-
usage: "run <start --file <program> [--timeout <dur>] | resume <runId> [--local --file <program>] | ps [--endpoint <ep>] | journal <runId> [--endpoint <ep>] | answer <runId> <stepKey> [--value <json>] [--artifact <ref>] [--endpoint <ep>] [--local --by <who>]> [--local]",
|
|
34
|
+
summary: "operate workflow runs — start, resume, list, inspect, answer (hosted by the manager), revoke",
|
|
35
|
+
usage: "run <start --file <program> [--timeout <dur>] | resume <runId> [--local --file <program>] | ps [--endpoint <ep>] | journal <runId> [--endpoint <ep>] | answer <runId> <stepKey> [--value <json>] [--artifact <ref>] [--endpoint <ep>] [--local --by <who>] | revoke <runId> --local --by <who> --reason <text>> [--local [--admit-read <channels> --admit-publish <channels>]]",
|
|
36
36
|
flags: [
|
|
37
37
|
...targetFlags,
|
|
38
38
|
{ name: "file", type: "string", short: "f", value: "<program>", description: "cotal-lang program source (start; resume --local when no program is recorded)" },
|
|
39
39
|
{ name: "local", type: "boolean", description: "drive in this process instead of on the manager (bare broker, or a run the manager cannot host)" },
|
|
40
|
+
{ name: "admit-read", type: "string", value: "<channels>", description: "start --local: the channels the run may read, comma-separated patterns or `none` (required; SPEC 14.8)" },
|
|
41
|
+
{ name: "admit-publish", type: "string", value: "<channels>", description: "start --local: the channels the run may post to, comma-separated patterns or `none` (required; SPEC 14.8)" },
|
|
40
42
|
{ name: "endpoint", type: "string", value: "<ep>", description: "endpoint the run record lives under (ps, journal, answer; default: manager)" },
|
|
41
43
|
{ name: "timeout", type: "string", value: "<dur>", description: "default checkpoint timeout for this drive (default: 1h)" },
|
|
42
|
-
{ name: "by", type: "string", value: "<who>", description: "who is answering (answer --local
|
|
44
|
+
{ name: "by", type: "string", value: "<who>", description: "who is answering or revoking (answer --local, revoke; the manager records the caller for a hosted answer)" },
|
|
45
|
+
{ name: "reason", type: "string", value: "<text>", description: "revoke: why the run's admission is revoked, recorded on the marker (required)" },
|
|
43
46
|
{ name: "value", type: "string", value: "<json>", description: "checkpoint answer payload as JSON (answer)" },
|
|
44
47
|
{ name: "artifact", type: "string", value: "<ref>", description: "artifact reference attached to the answer (answer)" },
|
|
45
48
|
],
|
|
46
|
-
positionals: "<start|resume|ps|journal|answer> …",
|
|
49
|
+
positionals: "<start|resume|ps|journal|answer|revoke> …",
|
|
47
50
|
run: (args) => runWorkflowCommand(args),
|
|
48
51
|
};
|
|
49
52
|
registry.register(runCommand);
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAuG,MAAM,iBAAiB,CAAC;AACtK,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,aAAa,EACb,SAAS,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAuG,MAAM,iBAAiB,CAAC;AACtK,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,aAAa,EACb,SAAS,EACT,YAAY,GAGb,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,GAKlB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAyB,MAAM,kBAAkB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,eAAe,EACf,cAAc,EACd,sBAAsB,GAOvB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,QAAQ,EACR,UAAU,EACV,iBAAiB,EACjB,UAAU,EACV,UAAU,GAKX,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEnE,iGAAiG;AACjG,kGAAkG;AAClG,iGAAiG;AACjG,gGAAgG;AAChG,iBAAiB;AACjB,OAAO,EAAE,QAAQ,EAAgB,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,WAAW,IAAI,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,gBAAgB,IAAI,OAAO,EAAE,MAAM,eAAe,CAAC;AAE5D,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AAE3B,MAAM,UAAU,GAAY;IAC1B,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,KAAK;IACX,KAAK,EAAE,SAAS;IAChB,OAAO,EAAE,8FAA8F;IACvG,KAAK,EACH,gXAAgX;IAClX,KAAK,EAAE;QACL,GAAG,WAAW;QACd,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,+EAA+E,EAAE;QAC9J,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,iGAAiG,EAAE;QAClJ,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,wGAAwG,EAAE;QAClL,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,2GAA2G,EAAE;QACxL,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,6EAA6E,EAAE;QAC/I,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,yDAAyD,EAAE;QAC3H,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,2GAA2G,EAAE;QACxK,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,+EAA+E,EAAE;QACjJ,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,4CAA4C,EAAE;QAC7G,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,oDAAoD,EAAE;KACxH;IACD,WAAW,EAAE,2CAA2C;IACxD,GAAG,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC;CACxC,CAAC;AAEF,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC"}
|