@substrat-run/kernel 0.136.0 → 0.138.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capability.d.ts +2 -0
- package/dist/capability.d.ts.map +1 -1
- package/dist/capability.js +2 -0
- package/dist/capability.js.map +1 -1
- package/dist/delivery-refusal.d.ts +21 -0
- package/dist/delivery-refusal.d.ts.map +1 -0
- package/dist/delivery-refusal.js +21 -0
- package/dist/delivery-refusal.js.map +1 -0
- package/dist/index.d.ts +23 -17
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +18 -12
- package/dist/index.js.map +1 -1
- package/dist/invocation-log.d.ts +25 -1
- package/dist/invocation-log.d.ts.map +1 -1
- package/dist/invocation-log.js +45 -6
- package/dist/invocation-log.js.map +1 -1
- package/dist/job-run.d.ts +370 -47
- package/dist/job-run.d.ts.map +1 -1
- package/dist/job-run.js +472 -81
- package/dist/job-run.js.map +1 -1
- package/dist/lifecycle-flow.d.ts.map +1 -1
- package/dist/lifecycle-flow.js +2 -1
- package/dist/lifecycle-flow.js.map +1 -1
- package/dist/membership-executor.d.ts +97 -0
- package/dist/membership-executor.d.ts.map +1 -0
- package/dist/membership-executor.js +246 -0
- package/dist/membership-executor.js.map +1 -0
- package/dist/membership-fence.d.ts +57 -0
- package/dist/membership-fence.d.ts.map +1 -0
- package/dist/membership-fence.js +90 -0
- package/dist/membership-fence.js.map +1 -0
- package/dist/operation-series.d.ts +58 -0
- package/dist/operation-series.d.ts.map +1 -0
- package/dist/operation-series.js +63 -0
- package/dist/operation-series.js.map +1 -0
- package/dist/peer.d.ts +13 -8
- package/dist/peer.d.ts.map +1 -1
- package/dist/peer.js +13 -14
- package/dist/peer.js.map +1 -1
- package/dist/permission-eval.d.ts +48 -1
- package/dist/permission-eval.d.ts.map +1 -1
- package/dist/permission-eval.js +137 -18
- package/dist/permission-eval.js.map +1 -1
- package/dist/read-only-sql.d.ts +2 -26
- package/dist/read-only-sql.d.ts.map +1 -1
- package/dist/read-only-sql.js +13 -9
- package/dist/read-only-sql.js.map +1 -1
- package/dist/refusal-query.d.ts +4 -1
- package/dist/refusal-query.d.ts.map +1 -1
- package/dist/refusal-query.js +6 -4
- package/dist/refusal-query.js.map +1 -1
- package/dist/refusals.d.ts +80 -8
- package/dist/refusals.d.ts.map +1 -1
- package/dist/refusals.js +141 -22
- package/dist/refusals.js.map +1 -1
- package/dist/scope-copy.d.ts +18 -0
- package/dist/scope-copy.d.ts.map +1 -1
- package/dist/scope-copy.js +18 -0
- package/dist/scope-copy.js.map +1 -1
- package/dist/scope-host.d.ts +266 -15
- package/dist/scope-host.d.ts.map +1 -1
- package/dist/scope-host.js +41 -0
- package/dist/scope-host.js.map +1 -1
- package/dist/spine-restore.d.ts +7 -2
- package/dist/spine-restore.d.ts.map +1 -1
- package/dist/spine-restore.js +13 -5
- package/dist/spine-restore.js.map +1 -1
- package/dist/system-switch-record.d.ts +290 -74
- package/dist/system-switch-record.d.ts.map +1 -1
- package/dist/system-switch-record.js +353 -101
- package/dist/system-switch-record.js.map +1 -1
- package/dist/system-switch.d.ts +146 -20
- package/dist/system-switch.d.ts.map +1 -1
- package/dist/system-switch.js +135 -27
- package/dist/system-switch.js.map +1 -1
- package/dist/timeline.d.ts +23 -0
- package/dist/timeline.d.ts.map +1 -1
- package/dist/timeline.js +30 -0
- package/dist/timeline.js.map +1 -1
- package/dist/ulid.d.ts +5 -0
- package/dist/ulid.d.ts.map +1 -1
- package/dist/ulid.js +9 -0
- package/dist/ulid.js.map +1 -1
- package/dist/vertical-events.d.ts +7 -1
- package/dist/vertical-events.d.ts.map +1 -1
- package/dist/vertical-events.js +14 -0
- package/dist/vertical-events.js.map +1 -1
- package/package.json +4 -4
package/dist/job-run.d.ts
CHANGED
|
@@ -60,23 +60,76 @@ import { type ExecutorRetryPolicy, type ScopeStub } from './scope-host.js';
|
|
|
60
60
|
* `_substrat_sweep_runs` is a RECEIPT and carries such a constraint; a cursor is not a
|
|
61
61
|
* receipt, and the two must not be re-merged (#1571, #1572).
|
|
62
62
|
*
|
|
63
|
-
* ## One
|
|
64
|
-
*
|
|
65
|
-
* Coalescing stops duplicate RUNS.
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* and
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
63
|
+
* ## One pass per run at a time — a lease (#2034)
|
|
64
|
+
*
|
|
65
|
+
* Coalescing stops duplicate RUNS. What stops two drives overlapping on one scope from
|
|
66
|
+
* running the same run's handler together is the CLAIM: before a pass is invoked, one
|
|
67
|
+
* compare-and-set (`JOB_RUN_CLAIM_SQL`) checks that the run is still `running` and due
|
|
68
|
+
* and, in the same statement, writes a lease — a `lease_owner` minted for that one pass,
|
|
69
|
+
* and `next_attempt_at` pushed out to the lease's expiry. Both adapters serialize a
|
|
70
|
+
* scope's writes (the DO's input gate, the pure host's turn queue), so exactly one
|
|
71
|
+
* claim can win; the loser's statement finds the run no longer due and changes nothing.
|
|
72
|
+
* The lease's expiry IS `next_attempt_at`, deliberately: a leased run is simply not
|
|
73
|
+
* due, so the due read, its index and its `JOB_RUN_DUE_AT` order need nothing new.
|
|
74
|
+
*
|
|
75
|
+
* **One clock: the store's** (#2042 r4). Every lease time — the claim's due test and
|
|
76
|
+
* expiry, the BEGIN test, each renewal — is computed by the STORE as its statement runs
|
|
77
|
+
* (the DO's clock, the pure host's injected one), never by the drive. The drive only
|
|
78
|
+
* measures how long it has itself been waiting, on its own monotonic clock, from just
|
|
79
|
+
* before it sent the claim: the time left on its lease is `leaseMs` minus that, which
|
|
80
|
+
* counts the whole round trip and so can only err short. A skew between the drive's
|
|
81
|
+
* clock and the store's cannot move a lease either way.
|
|
82
|
+
*
|
|
83
|
+
* ## The commitment point: BEGIN
|
|
84
|
+
*
|
|
85
|
+
* A claim does not yet run anything. The drive first checks its lease still has more
|
|
86
|
+
* than `JOB_LEASE_ENTRY_MARGIN` of it left, then BEGINS the pass with a second
|
|
87
|
+
* compare-and-set (`JOB_RUN_BEGIN_SQL`): it holds only while the claim still owns the
|
|
88
|
+
* lease with more than the margin left by the store's clock, and it stamps
|
|
89
|
+
* `lease_began_at`. **The drive invokes the handler if and only if BEGIN wrote — and it
|
|
90
|
+
* does not second-guess BEGIN's reply.** BEGIN is the commitment: from that write on,
|
|
91
|
+
* the pass counts as having begun, whether or not its handler gets far.
|
|
92
|
+
*
|
|
93
|
+
* Why no check after BEGIN, and why no further write could replace one: whatever the
|
|
94
|
+
* drive does after an acknowledged write, the acknowledgement can arrive late. A check
|
|
95
|
+
* on BEGIN's reply that declined to invoke would leave a durable stamp for a pass that
|
|
96
|
+
* never ran — and a takeover would charge it, failing a `maxAttempts: 1` run with no
|
|
97
|
+
* invocation at all. Recording "the handler really started" in another write only moves
|
|
98
|
+
* the same race onto THAT write's reply. Some last write has to be the point the drive
|
|
99
|
+
* acts on unconditionally; it is BEGIN.
|
|
100
|
+
*
|
|
101
|
+
* **What that leaves, by construction, is at-least-once:** if BEGIN's reply takes longer
|
|
102
|
+
* than the lease it left — more than the margin, a quarter of the lease, minutes at the
|
|
103
|
+
* default — the lease can expire before the handler starts, another drive can take the
|
|
104
|
+
* run over, and both run until the stale pass reaches a step boundary or its outcome,
|
|
105
|
+
* where it learns and stops. The same holds for a pass that spends longer than the lease
|
|
106
|
+
* between two steps. Like a step body (see `JobPassContext.step`), that stretch is
|
|
107
|
+
* at-least-once, and the job's `leaseMs` is what keeps it from happening.
|
|
108
|
+
*
|
|
109
|
+
* Everything the pass writes after BEGIN — a step's ledger row, the outcome patch, the
|
|
110
|
+
* commit — is conditional on still holding the lease, so a holder that lost it changes
|
|
111
|
+
* nothing. Each step boundary RENEWS the lease (by the store's clock), so a pass longer
|
|
112
|
+
* than one lease is not taken over while it is still making progress.
|
|
113
|
+
*
|
|
114
|
+
* **A lease that expires without a patch is recovered.** The run is due again at the
|
|
115
|
+
* expiry, and the next claim takes it over. A takeover charges an attempt if and only if
|
|
116
|
+
* the lease had BEGUN (`lease_began_at`): that pass started and never reported — a crash
|
|
117
|
+
* between BEGIN and the handler included, which nothing can tell from a crash inside it —
|
|
118
|
+
* so `attempts` stays honest and a pass that keeps dying exhausts the job's policy and
|
|
119
|
+
* ends `failed` rather than looping forever. A claim that never began ran nothing, and
|
|
120
|
+
* costs nothing. The expiry has to outlast the longest stretch a pass spends between two
|
|
121
|
+
* steps, which is the job's own number: `registerJob`'s `leaseMs`, default `JOB_LEASE_MS`.
|
|
122
|
+
*
|
|
123
|
+
* ## Admission misses
|
|
124
|
+
*
|
|
125
|
+
* A claim that does not BEGIN — its reply came back with too little of the lease left, or
|
|
126
|
+
* BEGIN refused — is an ADMISSION MISS. No handler ran, so no attempt is charged; but the
|
|
127
|
+
* run counts CONSECUTIVE misses (`admission_misses`, cleared only by BEGIN), and ONE
|
|
128
|
+
* conditional store transaction (`JOB_RUN_MISS_SQL`) releases the lease, adds the miss to
|
|
129
|
+
* the count it finds, and sets the run's backoff (`admissionBackoffMs`) or, at
|
|
130
|
+
* `JOB_ADMISSION_MISS_MAX`, fails it with `JOB_LEASE_TOO_SHORT_NOTE`. The drive reports
|
|
131
|
+
* and logs a warning naming the job, its `leaseMs` and the delay it saw. No outcome patch
|
|
132
|
+
* ever writes the count, so no write built from an older snapshot can overwrite it.
|
|
80
133
|
*
|
|
81
134
|
* ## What this is NOT
|
|
82
135
|
*
|
|
@@ -94,10 +147,10 @@ import { type ExecutorRetryPolicy, type ScopeStub } from './scope-host.js';
|
|
|
94
147
|
* and the hosted store the same shape rather than merely the same intention.
|
|
95
148
|
* Spine — kernel-written, `_substrat_*`, never a module migration.
|
|
96
149
|
*/
|
|
97
|
-
export declare const JOB_RUN_DDL = "\n CREATE TABLE IF NOT EXISTS _substrat_job_runs (\n id TEXT PRIMARY KEY,\n -- The coalescing key: one LIVE run per (module_id, job, instance). Not a UNIQUE\n -- index, deliberately -- see this file's header. instance names WHAT is being\n -- walked (a source id, a mapping version); a job with one walk per scope uses\n -- the 'default' the input schema fills in.\n module_id TEXT NOT NULL,\n job TEXT NOT NULL,\n instance TEXT NOT NULL,\n -- What start() was handed, as JSON. Held to the queue-safety rule\n -- (assertQueueSafe): ids and configuration, never bytes, class instances or\n -- functions, because this value has to survive a queue message unchanged.\n payload TEXT NOT NULL,\n -- Declared subject for erasing external output without guessing from its text.\n subject_id TEXT,\n -- 'running' | 'done' | 'failed'. A killed run stays 'running' and is picked up\n -- again by the next drive -- which is exactly what makes it restartable.\n status TEXT NOT NULL,\n -- What the last COMMITTED pass handed forward, as JSON. NULL = no pass has\n -- committed yet, which is a fact ('start from the beginning'), not missing data.\n cursor TEXT,\n -- The run's counter bag (JSON object of numbers), merged on each commit. An\n -- uncommitted pass's counts are discarded with the rest of the pass.\n counters TEXT NOT NULL DEFAULT '{}',\n -- CONSECUTIVE failed passes. Reset to 0 whenever a pass commits, so this reads\n -- as \"how stuck is it now\", not \"how much work has it done\".\n attempts INTEGER NOT NULL DEFAULT 0,\n -- The error the last failed pass left. Retained after the run goes 'failed' --\n -- the record is the evidence, so it must still say why.\n last_error TEXT,\n started_at TEXT NOT NULL,\n updated_at TEXT NOT NULL,\n -- When the next pass may run, on the executor's own backoff curve. NULL = now\n -- (or terminal), which is why the due read tests IS NULL as well as <= now.\n next_attempt_at TEXT,\n -- When the run reached 'done' or 'failed'. NULL while it is still running.\n ended_at TEXT\n );\n -- The drive's read: WHERE status = 'running' AND (next_attempt_at IS NULL OR <= ?)\n -- ORDER BY id. Leading with status makes the live runs a seekable range over a\n -- table that RETAINS every finished run, so the cost tracks how much is in flight\n -- rather than how much the scope has ever imported.\n CREATE INDEX IF NOT EXISTS _substrat_job_runs_due ON _substrat_job_runs (status, next_attempt_at, id);\n -- Coalescing's read, and the operator read's filter.\n CREATE INDEX IF NOT EXISTS _substrat_job_runs_key ON _substrat_job_runs (module_id, job, instance, id);\n -- The step ledger of the pass currently in flight. Rows are written as each step\n -- COMPLETES and dropped when the pass COMMITS, so this holds one pass's worth of\n -- steps and never grows with the length of the walk.\n --\n -- A FAILED run keeps its last pass's rows, deliberately: they are the difference\n -- between \"it got nowhere\" and \"it got three quarters of the way and then the\n -- provider went down\", which the run row's single last_error cannot say. Still\n -- bounded -- one pass's worth per failed run -- because a restart is a NEW run\n -- with a new id and therefore its own ledger.\n CREATE TABLE IF NOT EXISTS _substrat_job_steps (\n run_id TEXT NOT NULL,\n step TEXT NOT NULL,\n -- The step's return value as JSON. NOT NULL is what MEANS completed: a step that\n -- threw leaves the row with a NULL result and a raised attempts count, so the\n -- next pass runs it again rather than reading a success that never happened. A\n -- step returning nothing stores the JSON text 'null', which is not SQL NULL.\n result TEXT,\n attempts INTEGER NOT NULL DEFAULT 0,\n last_error TEXT,\n recorded_at TEXT NOT NULL,\n PRIMARY KEY (run_id, step)\n );\n";
|
|
150
|
+
export declare const JOB_RUN_DDL = "\n CREATE TABLE IF NOT EXISTS _substrat_job_runs (\n id TEXT PRIMARY KEY,\n -- The coalescing key: one LIVE run per (module_id, job, instance). Not a UNIQUE\n -- index, deliberately -- see this file's header. instance names WHAT is being\n -- walked (a source id, a mapping version); a job with one walk per scope uses\n -- the 'default' the input schema fills in.\n module_id TEXT NOT NULL,\n job TEXT NOT NULL,\n instance TEXT NOT NULL,\n -- What start() was handed, as JSON. Held to the queue-safety rule\n -- (assertQueueSafe): ids and configuration, never bytes, class instances or\n -- functions, because this value has to survive a queue message unchanged.\n payload TEXT NOT NULL,\n -- Declared subject for erasing external output without guessing from its text.\n subject_id TEXT,\n -- 'running' | 'done' | 'failed'. A killed run stays 'running' and is picked up\n -- again by the next drive -- which is exactly what makes it restartable.\n status TEXT NOT NULL,\n -- What the last COMMITTED pass handed forward, as JSON. NULL = no pass has\n -- committed yet, which is a fact ('start from the beginning'), not missing data.\n cursor TEXT,\n -- The run's counter bag (JSON object of numbers), merged on each commit. An\n -- uncommitted pass's counts are discarded with the rest of the pass.\n counters TEXT NOT NULL DEFAULT '{}',\n -- CONSECUTIVE failed passes. Reset to 0 whenever a pass commits, so this reads\n -- as \"how stuck is it now\", not \"how much work has it done\".\n attempts INTEGER NOT NULL DEFAULT 0,\n -- The error the last failed pass left. Retained after the run goes 'failed' --\n -- the record is the evidence, so it must still say why.\n last_error TEXT,\n started_at TEXT NOT NULL,\n updated_at TEXT NOT NULL,\n -- When the next pass may run, on the executor's own backoff curve. NULL = now\n -- (or terminal), which is why the due read tests IS NULL as well as <= now. While\n -- a pass holds the run (lease_owner set) it is the lease's expiry (#2034).\n next_attempt_at TEXT,\n -- When the run reached 'done' or 'failed'. NULL while it is still running.\n ended_at TEXT,\n -- #2034: the pass holding the run, minted per claim; NULL = nobody. Every write a\n -- pass makes is conditional on it, and the pass's outcome clears it.\n lease_owner TEXT,\n -- #2034 (#2042 r2, r4): when the holder BEGAN its pass (JOB_RUN_BEGIN_SQL), the point\n -- after which its handler runs; NULL = it has not, and a claim that never began is\n -- taken over without costing an attempt.\n lease_began_at TEXT,\n -- #2042 r3: CONSECUTIVE claims that did not begin their pass (their answer came back\n -- with too little of the lease left). NULL = none; cleared by BEGIN, written only by\n -- JOB_RUN_MISS_SQL. Not attempts: no handler ran. At JOB_ADMISSION_MISS_MAX the run\n -- fails, its lease too short for where it runs.\n admission_misses INTEGER\n );\n -- The drive's read: WHERE status = 'running' AND (next_attempt_at IS NULL OR <= ?)\n -- ORDER BY id. Leading with status makes the live runs a seekable range over a\n -- table that RETAINS every finished run, so the cost tracks how much is in flight\n -- rather than how much the scope has ever imported.\n CREATE INDEX IF NOT EXISTS _substrat_job_runs_due ON _substrat_job_runs (status, next_attempt_at, id);\n -- #1834: the drive's ORDER -- by when each run became due (JOB_RUN_DUE_AT), then id -- so a run\n -- that waited (a deferral, a backoff) queues behind work that was due before it, instead of\n -- heading every drive by its age. The expression is spelled exactly as the query spells it.\n CREATE INDEX IF NOT EXISTS _substrat_job_runs_due_at ON _substrat_job_runs (status, COALESCE(next_attempt_at, started_at), id);\n -- Coalescing's read, and the operator read's filter.\n CREATE INDEX IF NOT EXISTS _substrat_job_runs_key ON _substrat_job_runs (module_id, job, instance, id);\n -- The step ledger of the pass currently in flight. Rows are written as each step\n -- COMPLETES and dropped when the pass COMMITS, so this holds one pass's worth of\n -- steps and never grows with the length of the walk.\n --\n -- A FAILED run keeps its last pass's rows, deliberately: they are the difference\n -- between \"it got nowhere\" and \"it got three quarters of the way and then the\n -- provider went down\", which the run row's single last_error cannot say. Still\n -- bounded -- one pass's worth per failed run -- because a restart is a NEW run\n -- with a new id and therefore its own ledger.\n CREATE TABLE IF NOT EXISTS _substrat_job_steps (\n run_id TEXT NOT NULL,\n step TEXT NOT NULL,\n -- The step's return value as JSON. NOT NULL is what MEANS completed: a step that\n -- threw leaves the row with a NULL result and a raised attempts count, so the\n -- next pass runs it again rather than reading a success that never happened. A\n -- step returning nothing stores the JSON text 'null', which is not SQL NULL.\n result TEXT,\n attempts INTEGER NOT NULL DEFAULT 0,\n last_error TEXT,\n recorded_at TEXT NOT NULL,\n PRIMARY KEY (run_id, step)\n );\n";
|
|
98
151
|
/**
|
|
99
152
|
* The write every pass outcome lands through, on both adapters — a compare-and-set on
|
|
100
|
-
* `status = 'running'` (#1632).
|
|
153
|
+
* `status = 'running'` (#1632) and on the pass's lease (#2034), which it releases.
|
|
101
154
|
*
|
|
102
155
|
* A pass runs for as long as its handler takes, outside any lock, so a subject erasure
|
|
103
156
|
* can settle the run `failed` and tombstone its payload, cursor and step memos while the
|
|
@@ -107,12 +160,91 @@ export declare const JOB_RUN_DDL = "\n CREATE TABLE IF NOT EXISTS _substrat_job
|
|
|
107
160
|
* nothing else in the kernel moves a run out of that state except the pass itself and
|
|
108
161
|
* the erasure. Same shape, same reason, as `settlePlatformRequest`'s CAS on `pending`.
|
|
109
162
|
*
|
|
163
|
+
* The lease half: a pass whose lease another drive took over after it expired is no
|
|
164
|
+
* longer the run's, and its outcome — written over the new holder's work — is refused.
|
|
165
|
+
* `IS ?`, not `= ?`, so a caller passing NULL (a coordinator from before leases) still
|
|
166
|
+
* patches the unleased rows it drives.
|
|
167
|
+
*
|
|
168
|
+
* It never writes `admission_misses` (#2042 r4): only BEGIN clears it and only
|
|
169
|
+
* `JOB_RUN_MISS_SQL` adds to it.
|
|
170
|
+
*
|
|
110
171
|
* Params: status, cursor, counters, attempts, last_error, updated_at, next_attempt_at,
|
|
111
|
-
* ended_at, id.
|
|
172
|
+
* ended_at, id, lease_owner.
|
|
173
|
+
*/
|
|
174
|
+
export declare const JOB_RUN_PATCH_SQL = "UPDATE _substrat_job_runs\n SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,\n updated_at = ?, next_attempt_at = ?, ended_at = ?,\n lease_owner = NULL, lease_began_at = NULL\n WHERE id = ? AND status = 'running' AND lease_owner IS ?";
|
|
175
|
+
/**
|
|
176
|
+
* #2034: the claim — the ONE statement that decides which drive runs a due run's pass.
|
|
177
|
+
*
|
|
178
|
+
* Succeeds (one row changed) only where the run is still `running` and due, so of two
|
|
179
|
+
* drives that both picked it, one wins and the other changes nothing. The winner's
|
|
180
|
+
* lease is written in the same statement: `lease_owner`, and `next_attempt_at` pushed
|
|
181
|
+
* out to the lease's expiry, which is what takes the run out of the due read. Both the
|
|
182
|
+
* due test and the expiry are the STORE's time as it runs (#2042 r4): the adapter binds
|
|
183
|
+
* its own clock's now, never the drive's.
|
|
184
|
+
*
|
|
185
|
+
* A run due while it still carries a BEGUN lease (`lease_began_at`) is a pass that
|
|
186
|
+
* started and never reported, and taking it over counts that pass as failed: `attempts`
|
|
187
|
+
* + 1 and the note as `last_error`. A lease whose claim never began ran nothing and is
|
|
188
|
+
* taken over for free. SQLite evaluates every `SET` against the row as it was, so the
|
|
189
|
+
* `CASE`s read the previous lease.
|
|
190
|
+
*
|
|
191
|
+
* Returns the claimed row, as the claim left it (`RETURNING *`); no row = the claim lost.
|
|
192
|
+
*
|
|
193
|
+
* Params: lease_owner, next_attempt_at (store now + leaseMs), updated_at (store now),
|
|
194
|
+
* last_error (the takeover note), id, store now.
|
|
195
|
+
*/
|
|
196
|
+
export declare const JOB_RUN_CLAIM_SQL = "UPDATE _substrat_job_runs\n SET lease_owner = ?, next_attempt_at = ?, updated_at = ?, lease_began_at = NULL,\n attempts = attempts + CASE WHEN lease_began_at IS NULL THEN 0 ELSE 1 END,\n last_error = CASE WHEN lease_began_at IS NULL THEN last_error ELSE ? END\n WHERE id = ? AND status = 'running' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)\n RETURNING *";
|
|
197
|
+
/**
|
|
198
|
+
* #2034 (#2042 r4): BEGIN a claimed pass — the commitment point (see the file header). A
|
|
199
|
+
* compare-and-set in the store's clock: it stamps `lease_began_at` and clears
|
|
200
|
+
* `admission_misses` only while `lease_owner` is still this claim's and the lease runs
|
|
201
|
+
* past the store's now plus the margin. The drive invokes the handler if and only if this
|
|
202
|
+
* wrote, and does not second-guess its reply.
|
|
203
|
+
*
|
|
204
|
+
* Params: lease_began_at (store now), id, lease_owner, store now + margin.
|
|
112
205
|
*/
|
|
113
|
-
export declare const
|
|
206
|
+
export declare const JOB_RUN_BEGIN_SQL = "UPDATE _substrat_job_runs SET lease_began_at = ?, admission_misses = NULL\n WHERE id = ? AND status = 'running' AND lease_owner IS ? AND next_attempt_at > ?";
|
|
114
207
|
/**
|
|
115
|
-
*
|
|
208
|
+
* #2042 r3, r4: an ADMISSION MISS, first statement — release a claim that did not begin and add
|
|
209
|
+
* one to the count it finds (`COALESCE(admission_misses, 0) + 1`), never writing a count read
|
|
210
|
+
* earlier. Only while `owner` still holds the run and has not begun. Returns the new count.
|
|
211
|
+
* Both adapters run it and `JOB_RUN_MISS_SETTLE_SQL` in ONE transaction.
|
|
212
|
+
*
|
|
213
|
+
* Params: updated_at (store now), id, lease_owner.
|
|
214
|
+
*/
|
|
215
|
+
export declare const JOB_RUN_MISS_SQL = "UPDATE _substrat_job_runs\n SET admission_misses = COALESCE(admission_misses, 0) + 1, updated_at = ?,\n lease_owner = NULL, lease_began_at = NULL\n WHERE id = ? AND status = 'running' AND lease_owner IS ? AND lease_began_at IS NULL\n RETURNING admission_misses";
|
|
216
|
+
/**
|
|
217
|
+
* #2042 r3, r4: an ADMISSION MISS, second statement, in the same transaction — the backoff or the
|
|
218
|
+
* failure that the NEW count calls for (`admissionMissOutcome`). Conditional on that count still
|
|
219
|
+
* being on the row, unleased.
|
|
220
|
+
*
|
|
221
|
+
* Params: status, next_attempt_at, ended_at, last_error (NULL keeps it), id, admission_misses.
|
|
222
|
+
*/
|
|
223
|
+
export declare const JOB_RUN_MISS_SETTLE_SQL = "UPDATE _substrat_job_runs\n SET status = ?, next_attempt_at = ?, ended_at = ?, last_error = COALESCE(?, last_error)\n WHERE id = ? AND status = 'running' AND lease_owner IS NULL AND admission_misses = ?";
|
|
224
|
+
/**
|
|
225
|
+
* #2042 r3: what a run's `misses`-th consecutive admission miss does to it, at the store's `now`:
|
|
226
|
+
* a backoff (`admissionBackoffMs`), or at `JOB_ADMISSION_MISS_MAX` a failure carrying `note`.
|
|
227
|
+
* The params of `JOB_RUN_MISS_SETTLE_SQL` before its id; shared so the adapters cannot differ.
|
|
228
|
+
*/
|
|
229
|
+
export declare function admissionMissOutcome(misses: number, now: string, note: string): {
|
|
230
|
+
status: 'running' | 'failed';
|
|
231
|
+
nextAttemptAt: string | null;
|
|
232
|
+
endedAt: string | null;
|
|
233
|
+
lastError: string | null;
|
|
234
|
+
};
|
|
235
|
+
/**
|
|
236
|
+
* #2034: renew a pass's lease at a step boundary — only while the pass still holds it, to the
|
|
237
|
+
* store's now plus the lease (#2042 r4: the store's clock, as everywhere a lease is timed).
|
|
238
|
+
* One row changed = still held, and the lease now runs to the new expiry; none = the
|
|
239
|
+
* pass lost the run (taken over, or settled by an erasure) and must stop.
|
|
240
|
+
*
|
|
241
|
+
* Params: next_attempt_at (the new expiry), id, lease_owner.
|
|
242
|
+
*/
|
|
243
|
+
export declare const JOB_RUN_RENEW_SQL = "UPDATE _substrat_job_runs SET next_attempt_at = ?\n WHERE id = ? AND status = 'running' AND lease_owner IS ?";
|
|
244
|
+
/**
|
|
245
|
+
* Record one step attempt — only while its run is still `running` (#1632) and the pass
|
|
246
|
+
* still holds its lease (#2034). Both adapters run it after `JOB_RUN_RENEW_SQL`, in one
|
|
247
|
+
* transaction, and only when the renewal held.
|
|
116
248
|
*
|
|
117
249
|
* The step half of `JOB_RUN_PATCH_SQL`'s CAS: a stale pass's step, finishing after an
|
|
118
250
|
* erasure settled the run, would otherwise write a fresh result carrying the person
|
|
@@ -120,9 +252,9 @@ export declare const JOB_RUN_PATCH_SQL = "UPDATE _substrat_job_runs\n SET st
|
|
|
120
252
|
* `VALUES` so the guard and the write are one statement; the `WHERE` also resolves
|
|
121
253
|
* SQLite's parse ambiguity between a SELECT's trailing clause and `ON CONFLICT`.
|
|
122
254
|
*
|
|
123
|
-
* Params: run_id, step, result, attempts, last_error, recorded_at, run_id.
|
|
255
|
+
* Params: run_id, step, result, attempts, last_error, recorded_at, run_id, lease_owner.
|
|
124
256
|
*/
|
|
125
|
-
export declare const JOB_STEP_RECORD_SQL = "INSERT INTO _substrat_job_steps (run_id, step, result, attempts, last_error, recorded_at)\n SELECT ?, ?, ?, ?, ?, ?\n WHERE EXISTS (SELECT 1 FROM _substrat_job_runs WHERE id = ? AND status = 'running')\n ON CONFLICT (run_id, step) DO UPDATE SET result = excluded.result,\n attempts = excluded.attempts,\n last_error = excluded.last_error,\n recorded_at = excluded.recorded_at";
|
|
257
|
+
export declare const JOB_STEP_RECORD_SQL = "INSERT INTO _substrat_job_steps (run_id, step, result, attempts, last_error, recorded_at)\n SELECT ?, ?, ?, ?, ?, ?\n WHERE EXISTS (SELECT 1 FROM _substrat_job_runs WHERE id = ? AND status = 'running' AND lease_owner IS ?)\n ON CONFLICT (run_id, step) DO UPDATE SET result = excluded.result,\n attempts = excluded.attempts,\n last_error = excluded.last_error,\n recorded_at = excluded.recorded_at";
|
|
126
258
|
/** Where a run is. A killed run is `running` — that is what makes it resumable. */
|
|
127
259
|
export type JobRunStatus = 'running' | 'done' | 'failed';
|
|
128
260
|
/** The coalescing key: one LIVE run per triple. */
|
|
@@ -149,8 +281,16 @@ export interface JobRun extends JobRunKey {
|
|
|
149
281
|
lastError: string | null;
|
|
150
282
|
startedAt: string;
|
|
151
283
|
updatedAt: string;
|
|
284
|
+
/** While `leaseOwner` is set, when that lease expires and the run is due again. */
|
|
152
285
|
nextAttemptAt: string | null;
|
|
153
286
|
endedAt: string | null;
|
|
287
|
+
/** #2034: the pass holding the run right now, or null. Opaque; minted per claim. */
|
|
288
|
+
leaseOwner: string | null;
|
|
289
|
+
/**
|
|
290
|
+
* #2042 r3: consecutive claims that did not begin their pass in time — 0 once one does. Not
|
|
291
|
+
* attempts: no handler ran. A run that keeps missing fails with `JOB_LEASE_TOO_SHORT_NOTE`.
|
|
292
|
+
*/
|
|
293
|
+
admissionMisses: number;
|
|
154
294
|
/**
|
|
155
295
|
* Why this row could not be read whole, or null — which is the ordinary case and
|
|
156
296
|
* what every run written by this driver carries.
|
|
@@ -204,6 +344,42 @@ export declare const JOB_RUN_LIST_MAX = 500;
|
|
|
204
344
|
* the hosted read cannot answer the same filter differently.
|
|
205
345
|
*/
|
|
206
346
|
export declare function jobRunListLimit(limit: number | undefined): number;
|
|
347
|
+
/**
|
|
348
|
+
* #2034: how long a pass's lease lasts by default — from its claim, and again from each
|
|
349
|
+
* step boundary, which renews it. A job whose pass can spend longer than this between
|
|
350
|
+
* two steps registers its own `leaseMs`; otherwise its run is taken over while still
|
|
351
|
+
* working. Fifteen minutes is the longest a hosted alarm or cron invocation runs, so a
|
|
352
|
+
* hosted pass that has gone longer without a step has been stopped anyway.
|
|
353
|
+
*/
|
|
354
|
+
export declare const JOB_LEASE_MS: number;
|
|
355
|
+
/** #2034: the shortest `leaseMs` a job may register — below it, the BEGIN margin is noise. */
|
|
356
|
+
export declare const JOB_LEASE_MIN_MS = 100;
|
|
357
|
+
/**
|
|
358
|
+
* #2034: the share of its lease a claim must still have left to BEGIN its pass — by the drive's own
|
|
359
|
+
* count when the claim's reply arrives, and by the store's clock as BEGIN runs. Time spent between
|
|
360
|
+
* the claim's write and its reply — a slow round trip, a paused isolate — is time another drive can
|
|
361
|
+
* spend taking the run over once the lease has expired. It is also the bound on the at-least-once
|
|
362
|
+
* window: a double run needs BEGIN's reply to take longer than this share of the lease.
|
|
363
|
+
*/
|
|
364
|
+
export declare const JOB_LEASE_ENTRY_MARGIN = 0.25;
|
|
365
|
+
/**
|
|
366
|
+
* #2042 r3: consecutive admission misses after which a run fails rather than being claimed again.
|
|
367
|
+
* A claim that does not begin in time costs no attempt, so without a bound a run whose lease is too
|
|
368
|
+
* short for where it runs would be claimed, missed and released forever, with nothing reporting it.
|
|
369
|
+
*/
|
|
370
|
+
export declare const JOB_ADMISSION_MISS_MAX = 10;
|
|
371
|
+
/** #2042 r3: the wait after a first admission miss; it doubles per miss, up to `JOB_ADMISSION_BACKOFF_MAX_MS`. */
|
|
372
|
+
export declare const JOB_ADMISSION_BACKOFF_BASE_MS = 1000;
|
|
373
|
+
/** #2042 r3: the longest wait between two admission attempts. */
|
|
374
|
+
export declare const JOB_ADMISSION_BACKOFF_MAX_MS: number;
|
|
375
|
+
/** #2042 r3: how long a run waits after its `misses`-th consecutive admission miss. */
|
|
376
|
+
export declare function admissionBackoffMs(misses: number): number;
|
|
377
|
+
/** #2042 r3: the start of `last_error` on a run failed for missing its admission `JOB_ADMISSION_MISS_MAX` times. */
|
|
378
|
+
export declare const JOB_LEASE_TOO_SHORT_NOTE = "lease too short for this environment";
|
|
379
|
+
/** #2034: `last_error` of a run whose expired lease a later claim took over. */
|
|
380
|
+
export declare const JOB_LEASE_EXPIRED_NOTE = "interrupted: the pass holding this run stopped reporting, and its lease expired";
|
|
381
|
+
/** #2034: a job's `leaseMs`, refused at registration unless it is an integer of at least `JOB_LEASE_MIN_MS`. */
|
|
382
|
+
export declare function assertLeaseMs(leaseMs: number | undefined): void;
|
|
207
383
|
/** Runs one `runDueJobs` call picks up by default. */
|
|
208
384
|
export declare const JOB_DRIVE_LIMIT = 50;
|
|
209
385
|
/**
|
|
@@ -282,6 +458,13 @@ export interface JobPassContext {
|
|
|
282
458
|
* scope the way a schedule does, through `pass.scope()`.
|
|
283
459
|
*/
|
|
284
460
|
export type JobHandler = (pass: JobPassContext) => JobPassResult | void | Promise<JobPassResult | void>;
|
|
461
|
+
/** A job as `registerJob` recorded it: its handler, its default step policy and its lease (#2034). */
|
|
462
|
+
export interface JobRegistration {
|
|
463
|
+
readonly handler: JobHandler;
|
|
464
|
+
readonly retry?: ExecutorRetryPolicy;
|
|
465
|
+
/** Default `JOB_LEASE_MS`. */
|
|
466
|
+
readonly leaseMs?: number;
|
|
467
|
+
}
|
|
285
468
|
/** What `runDueJobs` did in one call. */
|
|
286
469
|
export interface JobDriveReport {
|
|
287
470
|
/** Runs this call picked up — due, and `running`. */
|
|
@@ -294,11 +477,33 @@ export interface JobDriveReport {
|
|
|
294
477
|
retrying: number;
|
|
295
478
|
/** Runs whose step exhausted its retries this call. Terminal: `failed`, with the error on the record. */
|
|
296
479
|
failed: number;
|
|
480
|
+
/**
|
|
481
|
+
* #2034: passes whose outcome was refused because the run was no longer theirs — their
|
|
482
|
+
* lease expired and another drive took the run over, or it was settled meanwhile (an
|
|
483
|
+
* erasure) — and claims whose answer came back with too little of the lease left to start
|
|
484
|
+
* a pass safely (`JOB_LEASE_ENTRY_MARGIN`), which ran nothing. Nothing the pass produced was
|
|
485
|
+
* written; whoever holds the run reports it.
|
|
486
|
+
*/
|
|
487
|
+
superseded: number;
|
|
488
|
+
/**
|
|
489
|
+
* Runs whose pass the host's system door told to wait (#1834): no attempt was counted and no error
|
|
490
|
+
* recorded, as a switched-off schedule is `skipped` without its cadence moving. The run is due
|
|
491
|
+
* again after `JOB_DEFER_MS`, so a run that keeps waiting never takes the turn of one behind it.
|
|
492
|
+
*/
|
|
493
|
+
deferred: number;
|
|
297
494
|
/** Per-run failures, the same shape `ScheduleRunReport.errors` has: what failed and why. */
|
|
298
495
|
errors: {
|
|
299
496
|
runId: string;
|
|
300
497
|
error: string;
|
|
301
498
|
}[];
|
|
499
|
+
/**
|
|
500
|
+
* #2042 r3: what the drive saw but did not count as a failure — a claim that did not begin its
|
|
501
|
+
* pass in time, with the job's lease and the delay observed. Also logged as a structured line.
|
|
502
|
+
*/
|
|
503
|
+
warnings: {
|
|
504
|
+
runId: string;
|
|
505
|
+
warning: string;
|
|
506
|
+
}[];
|
|
302
507
|
}
|
|
303
508
|
/** The `_substrat_job_runs` row, as both adapters' SQL returns it. */
|
|
304
509
|
export interface JobRunRow {
|
|
@@ -318,6 +523,26 @@ export interface JobRunRow {
|
|
|
318
523
|
readonly updated_at: string;
|
|
319
524
|
readonly next_attempt_at: string | null;
|
|
320
525
|
readonly ended_at: string | null;
|
|
526
|
+
/** #2034. Optional only for an older scope DO answering before the lease column. */
|
|
527
|
+
readonly lease_owner?: string | null;
|
|
528
|
+
/** #2034 (#2042 r2, r4): when the holder BEGAN its pass. Optional for the same reason. */
|
|
529
|
+
readonly lease_began_at?: string | null;
|
|
530
|
+
/** #2042 r3: consecutive claims that did not begin. Optional for the same reason. */
|
|
531
|
+
readonly admission_misses?: number | null;
|
|
532
|
+
}
|
|
533
|
+
/**
|
|
534
|
+
* #2034: a won claim — the run as the claim left it, and whether it took over an expired lease
|
|
535
|
+
* whose pass had BEGUN (which the claim charged as a failed attempt).
|
|
536
|
+
*/
|
|
537
|
+
export interface JobRunClaim {
|
|
538
|
+
readonly run: JobRunRow;
|
|
539
|
+
readonly takeover: boolean;
|
|
540
|
+
}
|
|
541
|
+
/** #1834: what a drive's snapshot holds of one due run — enough to pick it, never to act on it. */
|
|
542
|
+
export interface JobDueKey {
|
|
543
|
+
readonly id: string;
|
|
544
|
+
readonly module_id: string;
|
|
545
|
+
readonly job: string;
|
|
321
546
|
}
|
|
322
547
|
/** The `_substrat_job_steps` row, as both adapters' SQL returns it. */
|
|
323
548
|
export interface JobStepRow {
|
|
@@ -369,20 +594,45 @@ export interface JobRunStore {
|
|
|
369
594
|
* DO side, where the input gate makes one round trip indivisible.
|
|
370
595
|
*/
|
|
371
596
|
startOrJoin(key: JobRunKey, row: JobRunRow): Promise<JobRunRow>;
|
|
372
|
-
/** One run by id, whatever its status. */
|
|
373
|
-
get(id: string): Promise<JobRunRow | null>;
|
|
374
597
|
/**
|
|
375
|
-
* `running` runs whose `next_attempt_at` has passed
|
|
376
|
-
*
|
|
598
|
+
* #1834: ONE snapshot of the keys of up to `max` `running` runs whose `next_attempt_at` has passed
|
|
599
|
+
* (or is NULL), in the order they became due (`JOB_RUN_DUE_AT`, then id), read in ONE query.
|
|
377
600
|
*
|
|
378
|
-
*
|
|
379
|
-
*
|
|
380
|
-
*
|
|
381
|
-
*
|
|
601
|
+
* Keys, not rows: the driver CLAIMS each one (`claim`) just before it runs it, so what it acts on
|
|
602
|
+
* is the row as the claim left it, never the snapshot. And one read, not pages: a cursor across
|
|
603
|
+
* separate reads met rows that a concurrent drive moved in between, and a drive then saw a run
|
|
604
|
+
* twice, or never.
|
|
382
605
|
*/
|
|
383
|
-
|
|
606
|
+
dueKeys(now: string, max: number): Promise<JobDueKey[]>;
|
|
607
|
+
/**
|
|
608
|
+
* #2034: claim run `id` for one pass — `JOB_RUN_CLAIM_SQL`, which returns the row as it left it —
|
|
609
|
+
* or null when the run is no longer `running` and due (another drive holds it, or it moved).
|
|
610
|
+
*
|
|
611
|
+
* The due test and the expiry (`leaseMs` on) are the STORE's now as the statement runs (#2042
|
|
612
|
+
* r4), never a time the drive computed. ONE operation, for the reason `startOrJoin` is one:
|
|
613
|
+
* `takeover` is whether the row carried a BEGUN lease before the claim, which only a read in
|
|
614
|
+
* the same transaction can say.
|
|
615
|
+
*/
|
|
616
|
+
claim(id: string, owner: string, leaseMs: number): Promise<JobRunClaim | null>;
|
|
617
|
+
/**
|
|
618
|
+
* #2034 (#2042 r4): BEGIN `owner`'s claimed pass (`JOB_RUN_BEGIN_SQL`) — the commitment point.
|
|
619
|
+
* True only while it still holds the lease and the lease runs more than `marginMs` past the
|
|
620
|
+
* store's now as the statement runs. The drive invokes its handler if and only if this is true.
|
|
621
|
+
*/
|
|
622
|
+
begin(id: string, owner: string, marginMs: number): Promise<boolean>;
|
|
623
|
+
/**
|
|
624
|
+
* #2042 r3, r4: an admission miss — `JOB_RUN_MISS_SQL` then `JOB_RUN_MISS_SETTLE_SQL` with
|
|
625
|
+
* `admissionMissOutcome`, in ONE transaction, at the store's now. Null = `owner` no longer held
|
|
626
|
+
* the run (or had begun): nothing was written. Otherwise the new consecutive count, and
|
|
627
|
+
* whether it failed the run.
|
|
628
|
+
*/
|
|
629
|
+
miss(id: string, owner: string, note: string): Promise<{
|
|
630
|
+
misses: number;
|
|
631
|
+
failed: boolean;
|
|
632
|
+
} | null>;
|
|
384
633
|
list(filter: JobRunFilter): Promise<JobRunRow[]>;
|
|
385
|
-
|
|
634
|
+
/** A pass outcome, only while `owner` holds the run (`JOB_RUN_PATCH_SQL`). False = refused. */
|
|
635
|
+
patch(id: string, patch: JobRunPatch, owner: string): Promise<boolean>;
|
|
386
636
|
/**
|
|
387
637
|
* A COMMITTED pass: write the run's new state and drop its step ledger together,
|
|
388
638
|
* indivisibly.
|
|
@@ -394,13 +644,51 @@ export interface JobRunStore {
|
|
|
394
644
|
* cursor — reads that stale memo and skips work it never did. The first cut of
|
|
395
645
|
* this file ordered the two calls carefully and explained in a comment why the
|
|
396
646
|
* gap was harmless. The comment was wrong; only atomicity makes it true.
|
|
647
|
+
*
|
|
648
|
+
* Only while `owner` holds the run, and the ledger is dropped only when the patch applied: a
|
|
649
|
+
* stale holder's commit would otherwise empty the ledger of the pass that took the run over.
|
|
650
|
+
* False = refused, and nothing was written.
|
|
651
|
+
*/
|
|
652
|
+
commitPass(id: string, patch: JobRunPatch, owner: string): Promise<boolean>;
|
|
653
|
+
/**
|
|
654
|
+
* #2034: a step boundary — renew `owner`'s lease to the store's now plus `leaseMs`
|
|
655
|
+
* (`JOB_RUN_RENEW_SQL`) and read the step's ledger row, in one operation. `held: false` = the
|
|
656
|
+
* pass lost the run; `row` is then null and the pass must stop.
|
|
657
|
+
*/
|
|
658
|
+
beginStep(runId: string, name: string, owner: string, leaseMs: number): Promise<{
|
|
659
|
+
held: boolean;
|
|
660
|
+
row: JobStepRow | null;
|
|
661
|
+
}>;
|
|
662
|
+
/**
|
|
663
|
+
* Record one step attempt, renewing `owner`'s lease to the store's now plus `leaseMs` in the
|
|
664
|
+
* same transaction — both only while `owner` holds the run. False = the pass lost it, and
|
|
665
|
+
* nothing was written.
|
|
397
666
|
*/
|
|
398
|
-
|
|
399
|
-
step(runId: string, name: string): Promise<JobStepRow | null>;
|
|
400
|
-
recordStep(runId: string, name: string, result: string | null, attempts: number, lastError: string | null, at: string): Promise<void>;
|
|
667
|
+
recordStep(runId: string, name: string, result: string | null, attempts: number, lastError: string | null, at: string, owner: string, leaseMs: number): Promise<boolean>;
|
|
401
668
|
}
|
|
402
669
|
/** `conflict` reason: two `step()` calls under one name in one pass. */
|
|
403
670
|
export declare const JOB_STEP_REUSED = "job_step_reused";
|
|
671
|
+
/**
|
|
672
|
+
* #1834: the reason a host's system door gives a refusal that means "not now", never "no": the
|
|
673
|
+
* module is held off on this scope until its switch is applied again, or the scope kept restarting
|
|
674
|
+
* under the door. It is for a caller READING the refusal. It decides nothing: anyone can throw an
|
|
675
|
+
* error with this reason, so the job driver never defers on it (`deferral` below is how a host
|
|
676
|
+
* says which refusals are its own).
|
|
677
|
+
*/
|
|
678
|
+
export declare const SYSTEM_DOOR_WAIT = "system_door_wait";
|
|
679
|
+
/**
|
|
680
|
+
* #1834: when a run became due — its `next_attempt_at`, or when it started if it has none — and the
|
|
681
|
+
* key the drive orders by. Ordering by id instead let an older run that keeps waiting (a deferral, a
|
|
682
|
+
* backoff) head every drive the moment it was due again, and take the turn of a run that had been
|
|
683
|
+
* due for longer. Spelled exactly as the `_substrat_job_runs_due_at` index spells it.
|
|
684
|
+
*/
|
|
685
|
+
export declare const JOB_RUN_DUE_AT = "COALESCE(next_attempt_at, started_at)";
|
|
686
|
+
/**
|
|
687
|
+
* #1834: how long a deferred run waits before it is due again. Not an attempt: nothing counts and
|
|
688
|
+
* there is no backoff. It only stops a waiting run from heading the due order on every drive, where
|
|
689
|
+
* it would take the turn of a run behind it that could make progress.
|
|
690
|
+
*/
|
|
691
|
+
export declare const JOB_DEFER_MS = 60000;
|
|
404
692
|
/**
|
|
405
693
|
* Refuse anything that cannot survive a queue message, naming the path.
|
|
406
694
|
*
|
|
@@ -457,8 +745,11 @@ export declare function jobRunOf(row: JobRunRow): JobRun;
|
|
|
457
745
|
export declare function startJobRun(store: JobRunStore, input: StartJobRunInput, mintId: () => string, now: () => string): Promise<JobRunRow>;
|
|
458
746
|
/** What one pass did, as the drive loop reads it. */
|
|
459
747
|
export interface JobPassOutcome {
|
|
460
|
-
status: 'advanced' | 'completed' | 'retrying' | 'failed';
|
|
461
|
-
/**
|
|
748
|
+
status: 'advanced' | 'completed' | 'retrying' | 'failed' | 'deferred' | 'superseded';
|
|
749
|
+
/**
|
|
750
|
+
* The error left on the record. Present exactly on `retrying` and `failed`, and on `deferred`,
|
|
751
|
+
* where it is the door's refusal and NOT on the record: a deferred pass records no error.
|
|
752
|
+
*/
|
|
462
753
|
error?: string;
|
|
463
754
|
}
|
|
464
755
|
/**
|
|
@@ -480,13 +771,29 @@ export interface JobPassOutcome {
|
|
|
480
771
|
*/
|
|
481
772
|
export declare function runJobPass(options: {
|
|
482
773
|
store: JobRunStore;
|
|
774
|
+
/** The run as this pass's claim left it. */
|
|
483
775
|
run: JobRunRow;
|
|
776
|
+
/** #2034: the lease this pass's claim wrote. Every write below is conditional on it. */
|
|
777
|
+
owner: string;
|
|
778
|
+
/** #2034: how long each renewal at a step boundary extends the lease. Default `JOB_LEASE_MS`. */
|
|
779
|
+
leaseMs?: number;
|
|
484
780
|
handler: JobHandler;
|
|
485
781
|
/** The job's own retry policy — a step may narrow it, none may widen past its own. */
|
|
486
782
|
retry?: ExecutorRetryPolicy;
|
|
487
783
|
now: () => string;
|
|
488
|
-
/**
|
|
489
|
-
|
|
784
|
+
/**
|
|
785
|
+
* Opens the system door for this run's module. Called at most once per pass, with this pass's
|
|
786
|
+
* token: a fresh object per pass, which the host ties its door's refusals to.
|
|
787
|
+
*/
|
|
788
|
+
openScope: (pass: object) => Promise<ScopeStub>;
|
|
789
|
+
/**
|
|
790
|
+
* #1834: did the host's own system door, on THIS pass (`pass`, the token `openScope` was handed),
|
|
791
|
+
* throw this value to say "not now"? The HOST answers, by the identity of what its door threw for
|
|
792
|
+
* this pass, never by an error's shape: a step or an operation can throw any public shape, and a
|
|
793
|
+
* pass that could defer itself would wait forever without spending a retry. The host consumes
|
|
794
|
+
* the mark as it answers yes. Absent: nothing defers.
|
|
795
|
+
*/
|
|
796
|
+
deferral?: (err: unknown, pass: object) => boolean;
|
|
490
797
|
}): Promise<JobPassOutcome>;
|
|
491
798
|
/**
|
|
492
799
|
* Advance every due run on one scope — the driver both adapters expose as
|
|
@@ -503,9 +810,21 @@ export declare function runJobPass(options: {
|
|
|
503
810
|
* when the budget was applied to the query instead, a scope holding `limit` such rows
|
|
504
811
|
* at the head of the due order returned the same unrunnable batch on every call —
|
|
505
812
|
* nothing newer was ever reached, and the report said `attempted: 0` forever with no
|
|
506
|
-
* indication why. So the read
|
|
813
|
+
* indication why. So the read looks past them, bounded by `JOB_DRIVE_SCAN_MAX` rows
|
|
507
814
|
* examined so one scope full of orphans cannot turn a tick into a table scan.
|
|
508
815
|
*
|
|
816
|
+
* **Selection is ONE snapshot per drive** (#1834). The drive reads the keys of up to
|
|
817
|
+
* `JOB_DRIVE_SCAN_MAX` due runs in a single query, picks up to `limit` runnable ones
|
|
818
|
+
* (each id once), and CLAIMS each one just before it runs it (#2034), skipping one the
|
|
819
|
+
* claim finds no longer `running` or no longer due. A cursor across several reads met rows
|
|
820
|
+
* that another writer had moved in between, and ran one twice or skipped it. A run that
|
|
821
|
+
* moves, or becomes due, after the snapshot is the NEXT drive's business, never this one's.
|
|
822
|
+
*
|
|
823
|
+
* The claim is a reservation, not a read: of two drives overlapping on one scope that both
|
|
824
|
+
* picked a run, exactly one wins it, and only the winner invokes the handler (see the
|
|
825
|
+
* file header). A second pass in the same drive (`maxPasses`) claims again, with a fresh
|
|
826
|
+
* owner, so it holds a lease of its own rather than one the first pass released.
|
|
827
|
+
*
|
|
509
828
|
* The starvation was spotted while re-reading this file, judged unlikely and left
|
|
510
829
|
* alone — and then found independently by a reviewer. The judgement may even have
|
|
511
830
|
* been right; recording it only in the author's head was not, because a decision
|
|
@@ -515,13 +834,17 @@ export declare function runJobPass(options: {
|
|
|
515
834
|
export declare function runDueJobRuns(options: {
|
|
516
835
|
store: JobRunStore;
|
|
517
836
|
/** job name → its handler and policy, as `registerJob` recorded them. */
|
|
518
|
-
handlerFor: (run: JobRunRow) =>
|
|
519
|
-
handler: JobHandler;
|
|
520
|
-
retry?: ExecutorRetryPolicy;
|
|
521
|
-
} | undefined;
|
|
837
|
+
handlerFor: (run: Pick<JobRunRow, 'module_id' | 'job'>) => JobRegistration | undefined;
|
|
522
838
|
now: () => string;
|
|
523
|
-
openScope: (run: JobRunRow) => Promise<ScopeStub>;
|
|
839
|
+
openScope: (run: JobRunRow, pass: object) => Promise<ScopeStub>;
|
|
840
|
+
/** #1834: the host's own system-door refusals on one pass, by identity (`runJobPass`). */
|
|
841
|
+
deferral?: (err: unknown, pass: object) => boolean;
|
|
524
842
|
maxPasses?: number;
|
|
525
843
|
limit?: number;
|
|
844
|
+
/**
|
|
845
|
+
* #2042 r4: this drive's own monotonic clock, in milliseconds — what it measures its wait for the
|
|
846
|
+
* store against. Never compared with the store's clock. Default `performance.now()`.
|
|
847
|
+
*/
|
|
848
|
+
monotonic?: () => number;
|
|
526
849
|
}): Promise<JobDriveReport>;
|
|
527
850
|
//# sourceMappingURL=job-run.d.ts.map
|
package/dist/job-run.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"job-run.d.ts","sourceRoot":"","sources":["../src/job-run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgC,KAAK,aAAa,EAAE,KAAK,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAC1G,OAAO,EAAiD,KAAK,mBAAmB,EAAE,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"job-run.d.ts","sourceRoot":"","sources":["../src/job-run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgC,KAAK,aAAa,EAAE,KAAK,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAC1G,OAAO,EAAiD,KAAK,mBAAmB,EAAE,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG1H;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyIG;AAEH;;;;;;;;GAQG;AACH,eAAO,MAAM,WAAW,+nKAqFvB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,eAAO,MAAM,iBAAiB,0RAI8B,CAAC;AAE7D;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,iBAAiB,0YAKf,CAAC;AAEhB;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,mKACsD,CAAC;AAErF;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,yRAIC,CAAC;AAE/B;;;;;;GAMG;AACH,eAAO,MAAM,uBAAuB,qNAEoD,CAAC;AAEzF;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,GACX;IAAE,MAAM,EAAE,SAAS,GAAG,QAAQ,CAAC;IAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAIlH;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,iBAAiB,mHAC8B,CAAC;AAE7D;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,mBAAmB,6hBAM+C,CAAC;AAEhF,mFAAmF;AACnF,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEzD,mDAAmD;AACnD,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,QAAQ,CAAC;IACnB,oDAAoD;IACpD,GAAG,EAAE,MAAM,CAAC;IACZ,4FAA4F;IAC5F,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,uDAAuD;AACvD,MAAM,WAAW,MAAO,SAAQ,SAAS;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,YAAY,CAAC;IACrB,qCAAqC;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,kEAAkE;IAClE,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,iFAAiF;IACjF,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,mFAAmF;IACnF,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,oFAAoF;IACpF,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B;;;OAGG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;;OAeG;IACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,oCAAoC;AACpC,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,QAAQ,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,iFAAiF;IACjF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,yFAAyF;IACzF,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,yEAAyE;AACzE,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oCAAoC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,kDAAkD;AAClD,eAAO,MAAM,kBAAkB,KAAK,CAAC;AAErC,uFAAuF;AACvF,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAEpC;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAGjE;AAED;;;;;;GAMG;AACH,eAAO,MAAM,YAAY,QAAc,CAAC;AAExC,8FAA8F;AAC9F,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAEpC;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,OAAO,CAAC;AAE3C;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,KAAK,CAAC;AAEzC,kHAAkH;AAClH,eAAO,MAAM,6BAA6B,OAAQ,CAAC;AAEnD,iEAAiE;AACjE,eAAO,MAAM,4BAA4B,QAAa,CAAC;AAEvD,uFAAuF;AACvF,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,oHAAoH;AACpH,eAAO,MAAM,wBAAwB,yCAAyC,CAAC;AAE/E,gFAAgF;AAChF,eAAO,MAAM,sBAAsB,oFACgD,CAAC;AAEpF,gHAAgH;AAChH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAS/D;AAED,sDAAsD;AACtD,eAAO,MAAM,eAAe,KAAK,CAAC;AAElC;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAEtC,gDAAgD;AAChD,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,qFAAqF;IACrF,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B,wCAAwC;IACxC,QAAQ,CAAC,GAAG,EAAE,SAAS,GAAG;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IACzC,8CAA8C;IAC9C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,6EAA6E;IAC7E,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzF,8EAA8E;IAC9E,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC;;;;;OAKG;IACH,KAAK,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,cAAc,KAAK,aAAa,GAAG,IAAI,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;AAExG,sGAAsG;AACtG,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,mBAAmB,CAAC;IACrC,8BAA8B;IAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,yCAAyC;AACzC,MAAM,WAAW,cAAc;IAC7B,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;IAClB,mGAAmG;IACnG,QAAQ,EAAE,MAAM,CAAC;IACjB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,6FAA6F;IAC7F,QAAQ,EAAE,MAAM,CAAC;IACjB,yGAAyG;IACzG,MAAM,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,4FAA4F;IAC5F,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC3C;;;OAGG;IACH,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAChD;AAED,sEAAsE;AACtE,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,kFAAkF;IAClF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,oFAAoF;IACpF,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,0FAA0F;IAC1F,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,qFAAqF;IACrF,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3C;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAC;IACxB,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC5B;AAED,mGAAmG;AACnG,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED,uEAAuE;AACvE,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,4FAA4F;AAC5F,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,WAAW;IAC1B;;;;;;;;;;;;;;;;OAgBG;IACH,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAChE;;;;;;;;OAQG;IACH,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACxD;;;;;;;;OAQG;IACH,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IAC/E;;;;OAIG;IACH,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACrE;;;;;OAKG;IACH,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IACnG,IAAI,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACjD,+FAA+F;IAC/F,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACvE;;;;;;;;;;;;;;;OAeG;IACH,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5E;;;;OAIG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;IAC3H;;;;OAIG;IACH,UAAU,CACR,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,OAAO,CAAC,CAAC;CACrB;AAED,wEAAwE;AACxE,eAAO,MAAM,eAAe,oBAAoB,CAAC;AAEjD;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,qBAAqB,CAAC;AAEnD;;;;;GAKG;AACH,eAAO,MAAM,cAAc,0CAA0C,CAAC;AAgBtE;;;;GAIG;AACH,eAAO,MAAM,YAAY,QAAS,CAAC;AAoBnC;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAuClE;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAoC/C;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,gBAAgB,EACvB,MAAM,EAAE,MAAM,MAAM,EACpB,GAAG,EAAE,MAAM,MAAM,GAChB,OAAO,CAAC,SAAS,CAAC,CAuCpB;AAqDD,qDAAqD;AACrD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,YAAY,CAAC;IACrF;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,UAAU,CAAC,OAAO,EAAE;IACxC,KAAK,EAAE,WAAW,CAAC;IACnB,4CAA4C;IAC5C,GAAG,EAAE,SAAS,CAAC;IACf,wFAAwF;IACxF,KAAK,EAAE,MAAM,CAAC;IACd,iGAAiG;IACjG,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC;IACpB,sFAAsF;IACtF,KAAK,CAAC,EAAE,mBAAmB,CAAC;IAC5B,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB;;;OAGG;IACH,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,CAAC,CAAC;IAChD;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;CACpD,GAAG,OAAO,CAAC,cAAc,CAAC,CA2L1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE;IAC3C,KAAK,EAAE,WAAW,CAAC;IACnB,yEAAyE;IACzE,UAAU,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,WAAW,GAAG,KAAK,CAAC,KAAK,eAAe,GAAG,SAAS,CAAC;IACvF,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB,SAAS,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,CAAC,CAAC;IAChE,0FAA0F;IAC1F,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,cAAc,CAAC,CAuI1B"}
|