@ancientpantheon/khronoton-core 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +8 -0
- package/dist/blockchain/stoachain.d.ts +1 -1
- package/dist/blockchain/stoachain.js +16 -1
- package/dist/{chunk-GIGLXIMC.js → chunk-A5YODPKQ.js} +820 -23
- package/dist/{chunk-KQZYMPZS.js → chunk-WKI52IPY.js} +1 -1
- package/dist/{context-CApk-LP4.d.ts → context-P-rCtBbJ.d.ts} +1 -1
- package/dist/handlers/execute.d.ts +12 -4
- package/dist/handlers/execute.js +42 -5
- package/dist/handlers/index.d.ts +2 -1
- package/dist/handlers/index.js +4 -1
- package/dist/handlers/read.d.ts +11 -0
- package/dist/handlers/read.js +65 -1
- package/dist/handlers/steps.d.ts +3 -0
- package/dist/handlers/steps.js +152 -0
- package/dist/hooks/index.d.ts +3 -3
- package/dist/hooks/index.js +2 -2
- package/dist/provider/index.d.ts +3 -3
- package/dist/provider/index.js +33 -2
- package/dist/{seams-DUtxDw_U.d.ts → seams-Co8Tq4ka.d.ts} +24 -0
- package/dist/server/executor.d.ts +14 -1
- package/dist/server/executor.js +144 -26
- package/dist/server/index.d.ts +12 -2
- package/dist/server/index.js +38 -1
- package/dist/server/loop.d.ts +17 -13
- package/dist/server/loop.js +19 -12
- package/dist/server/planners/evict.d.ts +20 -0
- package/dist/server/planners/evict.js +143 -0
- package/dist/server/planners/inject.d.ts +22 -0
- package/dist/server/planners/inject.js +120 -0
- package/dist/server/planners/registry.d.ts +118 -0
- package/dist/server/planners/registry.js +94 -0
- package/dist/server/planners/stoa-unwrap.d.ts +23 -0
- package/dist/server/planners/stoa-unwrap.js +204 -0
- package/dist/server/planners/stoa-wrap.d.ts +26 -0
- package/dist/server/planners/stoa-wrap.js +229 -0
- package/dist/server/planners/unstall.d.ts +25 -0
- package/dist/server/planners/unstall.js +135 -0
- package/dist/server/pure/run-advance.d.ts +76 -0
- package/dist/server/pure/run-advance.js +94 -0
- package/dist/server/pure/split-plan.d.ts +144 -0
- package/dist/server/pure/split-plan.js +362 -0
- package/dist/server/pure/yields.d.ts +57 -0
- package/dist/server/pure/yields.js +101 -0
- package/dist/server/resolvers.js +48 -6
- package/dist/server/runs-driver.d.ts +92 -0
- package/dist/server/runs-driver.js +582 -0
- package/dist/server/schema.d.ts +6 -2
- package/dist/server/schema.js +89 -1
- package/dist/server/seams.d.ts +28 -0
- package/dist/server/seams.js +4 -0
- package/dist/server/store/errors.d.ts +38 -0
- package/dist/server/store/errors.js +61 -0
- package/dist/server/store/index.d.ts +7 -2
- package/dist/server/store/index.js +8 -2
- package/dist/server/store/mappers.d.ts +11 -0
- package/dist/server/store/mappers.js +20 -0
- package/dist/server/store/runs.d.ts +233 -0
- package/dist/server/store/runs.js +341 -0
- package/dist/server/store/steps.d.ts +54 -0
- package/dist/server/store/steps.js +74 -0
- package/dist/server/store/yields.d.ts +46 -0
- package/dist/server/store/yields.js +73 -0
- package/dist/server/tick.d.ts +7 -0
- package/dist/server/tick.js +50 -1
- package/dist/server/types.d.ts +42 -3
- package/dist/ui/index.js +2 -2
- package/package.json +1 -1
|
@@ -0,0 +1,582 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The non-blocking chained-run driver — executes Phase 2's pure `advanceRun`
|
|
3
|
+
* decision function against the real store/chain, per
|
|
4
|
+
* `docs/work/khronoton-chained-runs-phase-3/design.md` §D/§F.
|
|
5
|
+
*
|
|
6
|
+
* `driveRunOnce` is a SINGLE-RUN, loop-until-no-progress pass: it repeatedly
|
|
7
|
+
* (1) reads the run + its current step + that step's legs, (2) calls the pure
|
|
8
|
+
* `advanceRun` to decide what happens next, and (3) executes the returned
|
|
9
|
+
* `RunAction`s against the store/executor — re-reading fresh state and
|
|
10
|
+
* calling `advanceRun` again after every mutating action, stopping only once
|
|
11
|
+
* a pass makes NO further store write (a `poll` that landed nothing new, a
|
|
12
|
+
* `submit-leg` that failed, or the defensive `wait`), the run/step vanishes,
|
|
13
|
+
* or an action throws.
|
|
14
|
+
*
|
|
15
|
+
* This internal looping is deliberate, not incidental: `advanceRun` itself
|
|
16
|
+
* documents its own terminal-run check (step 1) as "defends against a driver
|
|
17
|
+
* that calls advanceRun on a run it should have already stopped touching" —
|
|
18
|
+
* i.e. the pure function already anticipates being re-invoked, within one
|
|
19
|
+
* driver pass, on a run this SAME pass just terminalized. It is also the only
|
|
20
|
+
* way to satisfy design.md §B's requirement that a freshly created run's step
|
|
21
|
+
* 0 gets "planned-and-submitted" — i.e. both `replan-step` (build leg 0) AND
|
|
22
|
+
* `submit-leg` (submit it) — within the SAME tick pass that created the run,
|
|
23
|
+
* rather than waiting a full `tickIntervalMs` for the next
|
|
24
|
+
* `advanceInFlightRunsOnce` pass to notice the freshly-built leg.
|
|
25
|
+
*
|
|
26
|
+
* `advanceInFlightRunsOnce` is the scheduler-facing sweep: every in-flight
|
|
27
|
+
* run, one `driveRunOnce` each, same per-row isolation posture
|
|
28
|
+
* `codexCronotonTickOnce` already applies (one run's failure never blocks the
|
|
29
|
+
* others).
|
|
30
|
+
*
|
|
31
|
+
* Deliberately does NOT import anything from `./tick.js` or `./loop.js` —
|
|
32
|
+
* both of those import FROM this module in a later wave of this same phase;
|
|
33
|
+
* importing `TickCtx` here instead of declaring the structurally-compatible
|
|
34
|
+
* `RunsDriverCtx` below would create a circular import.
|
|
35
|
+
*/
|
|
36
|
+
import { DEFAULT_MAX_REPLAN_ATTEMPTS, RUN_STEP_DEADLINE_MS } from "./seams.js";
|
|
37
|
+
import { advanceRunStep, bumpRunAttempt, getRun, insertRunLeg, listInFlightRuns, listRunLegs, claimLegForSubmit, releaseLegClaim, markLegConfirmed, markLegFailed, markLegSubmitted, markRunTerminal, touchRunDeadline, } from "./store/runs.js";
|
|
38
|
+
import { listStepsForCronoton } from "./store/steps.js";
|
|
39
|
+
import { RunLegAlreadyExistsError } from "./store/errors.js";
|
|
40
|
+
import { stepRowToDefinition } from "./store/mappers.js";
|
|
41
|
+
import { getYieldPool, upsertYield } from "./store/yields.js";
|
|
42
|
+
import { executeCodexTransaction } from "./executor.js";
|
|
43
|
+
import { resolvePlannerFor, normalizePlannerResult } from "./planners/registry.js";
|
|
44
|
+
import { resolveSingleStepPayload } from "./pure/yields.js";
|
|
45
|
+
import { advanceRun } from "./pure/run-advance.js";
|
|
46
|
+
/**
|
|
47
|
+
* Thrown by `driveRunOnce` when a `mode: 'planner'` step names a
|
|
48
|
+
* `planner_name` that `resolvePlannerFor` cannot resolve — e.g. a typo, or a
|
|
49
|
+
* planner that genuinely hasn't been registered yet. The `StepPlanner`
|
|
50
|
+
* registry itself (`planners/registry.ts`) and its real planners
|
|
51
|
+
* (`ouronet.aqp-evict`/`ouronet.aqp-inject`/`ouronet.aqp-unstall`) are fully
|
|
52
|
+
* built; this error reports an unresolved NAME against that registry, not a
|
|
53
|
+
* missing registry mechanism.
|
|
54
|
+
*/
|
|
55
|
+
export class PlannerNotRegisteredError extends Error {
|
|
56
|
+
constructor(plannerName) {
|
|
57
|
+
super(`planner '${plannerName}' is not registered in the StepPlanner registry`);
|
|
58
|
+
this.name = "PlannerNotRegisteredError";
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/** This pass's per-step barrier deadline, `now + (config override ?? the module default)`. */
|
|
62
|
+
function computeStepDeadline(now, ctx) {
|
|
63
|
+
return new Date(now.getTime() + (ctx.config?.runStepDeadlineMs ?? RUN_STEP_DEADLINE_MS)).toISOString();
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Parse a leg's stored `definition_json` and check its MINIMAL shape before
|
|
67
|
+
* it is trusted enough to sign and submit as a real transaction. Today's
|
|
68
|
+
* only writer (`handleReplanStep` → `stepRowToDefinition`) is type-checked
|
|
69
|
+
* and always produces a well-formed value, so this never actually fires on
|
|
70
|
+
* any path Phase 3 exercises — it exists because `insertRunLeg` is public
|
|
71
|
+
* store API (not module-private), and Phase 4's planner registry will be
|
|
72
|
+
* the next writer of `definition_json`. A shallow check (not a full schema
|
|
73
|
+
* validator): non-empty `pactCode`, an array `signers`, an object
|
|
74
|
+
* `gasPayer`/`config`. Throws a plain `Error` (caught by the caller) rather
|
|
75
|
+
* than returning a sentinel, since every other parse failure in this layer
|
|
76
|
+
* (a corrupt `schedule_config_json`, for example — see `store/claim.ts`) is
|
|
77
|
+
* handled the same way: catch at the call site, never let `JSON.parse`'s
|
|
78
|
+
* own exception or a malformed shape reach the signer.
|
|
79
|
+
*/
|
|
80
|
+
function parseLegDefinition(definitionJson) {
|
|
81
|
+
const parsed = JSON.parse(definitionJson);
|
|
82
|
+
if (typeof parsed !== "object" || parsed === null) {
|
|
83
|
+
throw new Error("leg definition is not an object");
|
|
84
|
+
}
|
|
85
|
+
const def = parsed;
|
|
86
|
+
if (typeof def.pactCode !== "string" || def.pactCode.length === 0) {
|
|
87
|
+
throw new Error("leg definition is missing a non-empty pactCode");
|
|
88
|
+
}
|
|
89
|
+
if (!Array.isArray(def.signers)) {
|
|
90
|
+
throw new Error("leg definition is missing a signers array");
|
|
91
|
+
}
|
|
92
|
+
if (typeof def.gasPayer !== "object" || def.gasPayer === null) {
|
|
93
|
+
throw new Error("leg definition is missing a gasPayer object");
|
|
94
|
+
}
|
|
95
|
+
if (typeof def.config !== "object" || def.config === null) {
|
|
96
|
+
throw new Error("leg definition is missing a config object");
|
|
97
|
+
}
|
|
98
|
+
return parsed;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* `submit-leg { legIndex }`: submit the already-`pending` leg at this index
|
|
102
|
+
* (built by an earlier `replan-step` pass; `submit-leg` never builds a leg
|
|
103
|
+
* itself). Returns `true` iff the leg flipped to `submitted` (a store write
|
|
104
|
+
* happened) — a submit failure leaves the leg `pending` and returns `false`,
|
|
105
|
+
* so the caller's loop stops rather than busy-retrying within this same pass;
|
|
106
|
+
* the *next* `driveRunOnce` pass naturally re-selects the same still-pending
|
|
107
|
+
* leg (design.md §F — no retry-counting write exists or is needed).
|
|
108
|
+
*
|
|
109
|
+
* CONCURRENCY (Phase 5 fix): before the chain submission
|
|
110
|
+
* (`executeCodexTransaction`) is awaited, this now calls
|
|
111
|
+
* `claimLegForSubmit` — a real claim-before-submit step, mirroring
|
|
112
|
+
* `claim.ts`'s `claimDueCodexCronoton` idiom. Two concurrent callers racing
|
|
113
|
+
* the same `pending` leg (e.g. `executeNow`'s chained-row branch invoking
|
|
114
|
+
* `driveRunOnce` directly from an HTTP handler, racing the background
|
|
115
|
+
* loop's own `advanceInFlightRunsOnce` sweep — both are public exports, and
|
|
116
|
+
* only the LOOP's own pass sits behind `loop.ts`'s re-entrancy guard) can
|
|
117
|
+
* never both win the claim, so at most one of them ever reaches the real
|
|
118
|
+
* network call. A lost claim (the leg was already claimed/submitted by a
|
|
119
|
+
* concurrent winner, or moved on since `stepLegs` was read) returns `false`
|
|
120
|
+
* immediately — exactly like "wait", never a second submit. A submit
|
|
121
|
+
* attempt that does not land releases the claim (`releaseLegClaim`) so a
|
|
122
|
+
* LATER pass can still retry the still-`pending` leg, preserving the
|
|
123
|
+
* existing no-retry-cap posture (design.md §F) unchanged.
|
|
124
|
+
*/
|
|
125
|
+
async function handleSubmitLeg(legIndex, run, stepLegs, now, ctx) {
|
|
126
|
+
const leg = stepLegs.find((l) => l.leg_index === legIndex);
|
|
127
|
+
if (!leg) {
|
|
128
|
+
// Defensive — should never happen, since every submit-leg action targets
|
|
129
|
+
// a pending leg advanceRun itself selected from the very legs array this
|
|
130
|
+
// function passed it.
|
|
131
|
+
await ctx.onAudit({
|
|
132
|
+
action: "codex_cronoton_run_leg.not_found",
|
|
133
|
+
result: "failure",
|
|
134
|
+
targetKind: "codex_cronoton_run",
|
|
135
|
+
targetId: run.id,
|
|
136
|
+
detail: { codexCronotonId: run.codex_cronoton_id, stepIndex: run.step_index, legIndex },
|
|
137
|
+
});
|
|
138
|
+
return false;
|
|
139
|
+
}
|
|
140
|
+
let definition;
|
|
141
|
+
try {
|
|
142
|
+
definition = parseLegDefinition(leg.definition_json);
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
// Malformed/invalid stored definition — never hand this to the signer.
|
|
146
|
+
// Leaves the leg `pending` (same posture as a submit failure below): no
|
|
147
|
+
// request key was ever derived, so nothing unsafe happened, and this is
|
|
148
|
+
// deterministic (retrying will fail the same way every pass) — but
|
|
149
|
+
// Phase 3 has no retry-cap mechanism (see design.md's Out of scope), so
|
|
150
|
+
// an operator noticing the repeated audit event is the backstop today.
|
|
151
|
+
await ctx.onAudit({
|
|
152
|
+
action: "codex_cronoton_run_leg.invalid_definition",
|
|
153
|
+
result: "failure",
|
|
154
|
+
targetKind: "codex_cronoton_run_leg",
|
|
155
|
+
targetId: leg.id,
|
|
156
|
+
detail: {
|
|
157
|
+
runId: run.id,
|
|
158
|
+
stepIndex: run.step_index,
|
|
159
|
+
legIndex,
|
|
160
|
+
error: err instanceof Error ? err.message : String(err),
|
|
161
|
+
},
|
|
162
|
+
});
|
|
163
|
+
return false;
|
|
164
|
+
}
|
|
165
|
+
const claimed = claimLegForSubmit(leg.id, now.toISOString(), { db: ctx.db });
|
|
166
|
+
if (!claimed) {
|
|
167
|
+
// Lost the race: a concurrent caller already claimed (or submitted, or
|
|
168
|
+
// otherwise moved on) this exact leg since `stepLegs` was read. Back off
|
|
169
|
+
// silently — this pass makes no progress on this leg, exactly like
|
|
170
|
+
// "wait" — never a second submit attempt.
|
|
171
|
+
return false;
|
|
172
|
+
}
|
|
173
|
+
const result = await executeCodexTransaction(definition, "submit", {
|
|
174
|
+
runtime: ctx.runtime,
|
|
175
|
+
resolver: ctx.resolver,
|
|
176
|
+
config: ctx.config,
|
|
177
|
+
});
|
|
178
|
+
if (result.ok && result.requestKey && result.chainId) {
|
|
179
|
+
markLegSubmitted(leg.id, { requestKey: result.requestKey, chainId: result.chainId }, { db: ctx.db });
|
|
180
|
+
return true;
|
|
181
|
+
}
|
|
182
|
+
// Release the claim so a LATER pass can still retry this still-`pending`
|
|
183
|
+
// leg — the existing no-retry-cap posture (design.md §F) is unchanged; only
|
|
184
|
+
// the CONCURRENT-double-submit window this claim closes is new.
|
|
185
|
+
releaseLegClaim(leg.id, { db: ctx.db });
|
|
186
|
+
await ctx.onAudit({
|
|
187
|
+
action: "codex_cronoton_run_leg.submit_failed",
|
|
188
|
+
result: "failure",
|
|
189
|
+
targetKind: "codex_cronoton_run_leg",
|
|
190
|
+
targetId: leg.id,
|
|
191
|
+
detail: { runId: run.id, stepIndex: run.step_index, legIndex, error: result.error },
|
|
192
|
+
});
|
|
193
|
+
return false;
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* `poll {}`: one `pollRequestKeys` call per distinct chain for every
|
|
197
|
+
* `submitted` leg of the current step (Phase 0's "one request per chain per
|
|
198
|
+
* tick" contract). Returns `true` iff at least one leg landed (confirmed or
|
|
199
|
+
* failed) this pass.
|
|
200
|
+
*/
|
|
201
|
+
async function handlePoll(stepLegs, step, ctx) {
|
|
202
|
+
const submitted = stepLegs.filter((l) => l.status === "submitted" && l.request_key != null && l.chain_id != null);
|
|
203
|
+
if (submitted.length === 0)
|
|
204
|
+
return false;
|
|
205
|
+
const byChain = new Map();
|
|
206
|
+
for (const leg of submitted) {
|
|
207
|
+
const group = byChain.get(leg.chain_id) ?? [];
|
|
208
|
+
group.push(leg);
|
|
209
|
+
byChain.set(leg.chain_id, group);
|
|
210
|
+
}
|
|
211
|
+
let mutated = false;
|
|
212
|
+
for (const [chainId, legsForChain] of byChain) {
|
|
213
|
+
const client = ctx.runtime.createClient(ctx.runtime.getPactUrl(chainId));
|
|
214
|
+
const polled = (await client.pollRequestKeys?.(legsForChain.map((l) => l.request_key), chainId)) ?? {};
|
|
215
|
+
for (const leg of legsForChain) {
|
|
216
|
+
const landed = polled[leg.request_key];
|
|
217
|
+
if (!landed)
|
|
218
|
+
continue; // still pending — absent key means "not landed yet"
|
|
219
|
+
if (landed.result.status === "success") {
|
|
220
|
+
markLegConfirmed(leg.id, { db: ctx.db });
|
|
221
|
+
if (step.mode === "single" && step.yield_as) {
|
|
222
|
+
// The only way a plain hand-written Pact call (no JS-side planner
|
|
223
|
+
// runs for it) can contribute a yield: capture the real on-chain
|
|
224
|
+
// return value the moment this step's one leg CONFIRMS. `data` is
|
|
225
|
+
// not declared on `ListenResult.result`'s own narrow type (only
|
|
226
|
+
// `status`/`error` are), since most callers never need it — cast
|
|
227
|
+
// locally rather than widening that shared seam type for this one
|
|
228
|
+
// reader.
|
|
229
|
+
upsertYield({
|
|
230
|
+
runId: leg.run_id,
|
|
231
|
+
name: step.yield_as,
|
|
232
|
+
value: landed.result.data,
|
|
233
|
+
stepIndex: step.step_index,
|
|
234
|
+
}, { db: ctx.db });
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
markLegFailed(leg.id, { errorMessage: landed.result.error?.message ?? "transaction failed on-chain" }, { db: ctx.db });
|
|
239
|
+
}
|
|
240
|
+
mutated = true;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return mutated;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* `advance-step {}`: the barrier for the current step is met. Terminal
|
|
247
|
+
* (`succeeded`) if this was the last step, else move `step_index` forward and
|
|
248
|
+
* reset `attempt`/stamp a fresh deadline for the next step. Always a store
|
|
249
|
+
* write.
|
|
250
|
+
*/
|
|
251
|
+
function handleAdvanceStep(run, now, ctx) {
|
|
252
|
+
if (run.step_index + 1 >= run.step_count) {
|
|
253
|
+
markRunTerminal(run.id, "succeeded", { db: ctx.db });
|
|
254
|
+
}
|
|
255
|
+
else {
|
|
256
|
+
advanceRunStep(run.id, { fromStepIndex: run.step_index, toStepIndex: run.step_index + 1, deadlineAt: computeStepDeadline(now, ctx) }, { db: ctx.db });
|
|
257
|
+
}
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* `replan-step { reason }`: as of Phase 4.1, `driveRunOnce` sets
|
|
262
|
+
* `canReplan: step.mode === "planner"`, so this action is reached in TWO
|
|
263
|
+
* distinct situations, both handled here — a step's very first entry
|
|
264
|
+
* (`advanceRun`'s "no legs yet" branch, no prior legs for this step) and a
|
|
265
|
+
* genuine re-plan after a leg failure (`replanOrFail`, prior legs for this
|
|
266
|
+
* step already exist). The `isReplan` check below distinguishes them: a
|
|
267
|
+
* `mode: 'planner'` step looks its `planner_name` up in Phase 4's
|
|
268
|
+
* `StepPlanner` registry and calls it to produce this pass's legs (zero legs
|
|
269
|
+
* is treated exactly like a cleared barrier, matching `handleAdvanceStep`'s
|
|
270
|
+
* own terminal-vs-advance branch verbatim rather than duplicating it) — on a
|
|
271
|
+
* genuine re-plan, the new legs land at a bumped `attempt` generation and a
|
|
272
|
+
* fresh `leg_index` range (never colliding with the prior generation's now-
|
|
273
|
+
* failed legs), and the run's `attempt` counter and step deadline are
|
|
274
|
+
* advanced directly rather than through `advanceRunStep` (whose own
|
|
275
|
+
* attempt-reset side effect would otherwise silently undo the bump — see
|
|
276
|
+
* design.md §A and `store/runs.ts`'s `touchRunDeadline` doc comment); a
|
|
277
|
+
* `mode: 'single'` step builds its one fixed leg from the step row and stamps
|
|
278
|
+
* this step's first deadline (single-mode steps never set `canReplan`, so
|
|
279
|
+
* `isReplan` is always `false` on this branch). Always a store write (or a
|
|
280
|
+
* throw).
|
|
281
|
+
*
|
|
282
|
+
* CRITICAL, cross-phase-review-found fix (NARROWLY SCOPED — a first
|
|
283
|
+
* version of this fix wrapped EVERY call unconditionally and was itself
|
|
284
|
+
* found, on the SAME review's own terminal pass, to introduce two new
|
|
285
|
+
* regressions; both are explained below, not just the original bug):
|
|
286
|
+
*
|
|
287
|
+
* The original bug: a step whose CURRENT generation has NEVER had a
|
|
288
|
+
* single leg (`stepHasAnyLegs` below, `false`) is the ONLY situation
|
|
289
|
+
* `advanceRun`'s "no legs yet" decision (step 2, pure/run-advance.ts) can
|
|
290
|
+
* reach — and that decision is checked UNCONDITIONALLY, before its OWN
|
|
291
|
+
* deadline-exceeded check (step 4), REGARDLESS of whether `deadline_at`
|
|
292
|
+
* happens to already be set. So if the planner lookup/call (or, for
|
|
293
|
+
* `mode:'single'`, `stepRowToDefinition`) throws EVERY time before ever
|
|
294
|
+
* successfully inserting this step's first leg, `legs.length` stays `0`
|
|
295
|
+
* forever and `advanceRun` can NEVER reach its own deadline/attempt-budget
|
|
296
|
+
* recovery machinery — not just on a run's absolute first step, but on
|
|
297
|
+
* ANY step whose planner never manages one successful plan. Nothing else
|
|
298
|
+
* in this codebase can terminalize such a run (no exposed handler, no
|
|
299
|
+
* cron); left unguarded, it permanently occupies its cronoton's self-block
|
|
300
|
+
* slot (`schema.ts`'s partial unique index).
|
|
301
|
+
*
|
|
302
|
+
* Regression #1 (HIGH) an unconditional catch introduced: a GENUINE
|
|
303
|
+
* re-plan (`stepHasAnyLegs === true` — this step already built at least
|
|
304
|
+
* one PRIOR leg; `replanOrFail`'s own `maxReplanAttempts` budget already
|
|
305
|
+
* decided this attempt was worth making) throwing must NOT be
|
|
306
|
+
* terminalized here — that would silently zero out the retry budget
|
|
307
|
+
* `maxReplanAttempts`/`replanOrFail` exist to grant. Letting it propagate
|
|
308
|
+
* exactly as before this fix (isolated by `advanceInFlightRunsOnce`,
|
|
309
|
+
* retried or genuinely exhausted by the NEXT pass's own deadline/attempt
|
|
310
|
+
* check) is correct; only the `stepHasAnyLegs === false` case ever needed
|
|
311
|
+
* protection, because only it has no OTHER recovery path at all.
|
|
312
|
+
*
|
|
313
|
+
* Regression #2 (CRITICAL) an unconditional catch introduced: TWO
|
|
314
|
+
* concurrent `driveRunOnce` callers (e.g. `executeNow`'s HTTP-triggered
|
|
315
|
+
* call racing the background loop's own sweep — `store/runs.ts`'s own doc
|
|
316
|
+
* comment names this exact scenario) can both reach a step's first entry
|
|
317
|
+
* and race on `insertRunLeg`; the loser gets `RunLegAlreadyExistsError`
|
|
318
|
+
* while the WINNER is concurrently making real progress. Terminalizing
|
|
319
|
+
* the LOSER's run would incorrectly fail a run that is actually
|
|
320
|
+
* succeeding via the winner's write — `RunLegAlreadyExistsError` is
|
|
321
|
+
* therefore never terminalized here either, regardless of
|
|
322
|
+
* `stepHasAnyLegs`.
|
|
323
|
+
*/
|
|
324
|
+
async function handleReplanStep(run, step, allRunLegs, now, ctx) {
|
|
325
|
+
const stepHasAnyLegs = allRunLegs.some((l) => l.step_index === run.step_index);
|
|
326
|
+
try {
|
|
327
|
+
return await handleReplanStepInner(run, step, allRunLegs, now, ctx);
|
|
328
|
+
}
|
|
329
|
+
catch (err) {
|
|
330
|
+
if (stepHasAnyLegs || err instanceof RunLegAlreadyExistsError) {
|
|
331
|
+
// Regression #1 / #2 guards (see doc comment above) — let it
|
|
332
|
+
// propagate exactly as before this fix, isolated by
|
|
333
|
+
// `advanceInFlightRunsOnce`, never terminalized here.
|
|
334
|
+
throw err;
|
|
335
|
+
}
|
|
336
|
+
// This step's CURRENT generation has never had a single leg — the
|
|
337
|
+
// ONLY case with no other recovery path at all (see doc comment).
|
|
338
|
+
const reason = `replan-step threw: ${err instanceof Error ? err.message : String(err)}`;
|
|
339
|
+
await handleFailRun(run, reason, ctx);
|
|
340
|
+
throw err;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
async function handleReplanStepInner(run, step, allRunLegs, now, ctx) {
|
|
344
|
+
if (step.mode === "planner") {
|
|
345
|
+
// Resolve any `{"$yield": "<name>"}` placeholder in this step's
|
|
346
|
+
// `planner_params_json` against the run's yield pool BEFORE the planner
|
|
347
|
+
// ever runs — a step with no `$yield` references anywhere resolves
|
|
348
|
+
// against this SAME pool (always `{}` for such a step) byte-identical
|
|
349
|
+
// to before this feature existed.
|
|
350
|
+
const pool = getYieldPool(run.id, { db: ctx.db });
|
|
351
|
+
const resolved = resolvePlannerFor(step, ctx, pool);
|
|
352
|
+
if (!resolved) {
|
|
353
|
+
throw new PlannerNotRegisteredError(step.planner_name ?? "(unnamed)");
|
|
354
|
+
}
|
|
355
|
+
const rawResult = await resolved.planner(resolved.resolvedStep, resolved.plannerCtx);
|
|
356
|
+
const normalized = normalizePlannerResult(rawResult);
|
|
357
|
+
const legs = normalized.legs;
|
|
358
|
+
if (legs.length === 0) {
|
|
359
|
+
// The planner decided this step needs no work this pass (e.g. nothing
|
|
360
|
+
// owed) — treat exactly like a cleared barrier, matching
|
|
361
|
+
// `handleAdvanceStep`'s own terminal-vs-advance branch verbatim rather
|
|
362
|
+
// than duplicating it. A review finding caught that an EARLIER version
|
|
363
|
+
// returned here before ever reaching the yield-write below, silently
|
|
364
|
+
// discarding a yield a planner reported alongside zero legs (e.g. "I
|
|
365
|
+
// confirmed there's nothing to do, and here's the value that proves
|
|
366
|
+
// it") — this step's barrier is cleared either way, so its yield (if
|
|
367
|
+
// any) is just as real and must still be persisted.
|
|
368
|
+
if (normalized.yield !== undefined && step.yield_as) {
|
|
369
|
+
upsertYield({ runId: run.id, name: step.yield_as, value: normalized.yield, stepIndex: run.step_index }, { db: ctx.db });
|
|
370
|
+
}
|
|
371
|
+
return handleAdvanceStep(run, now, ctx);
|
|
372
|
+
}
|
|
373
|
+
// Every generation of this step's legs, NOT attempt-filtered — unlike
|
|
374
|
+
// `driveRunOnce`'s own `stepLegs` (which only ever sees the CURRENT
|
|
375
|
+
// generation for the barrier/submit/poll decision), a re-plan needs to
|
|
376
|
+
// see every PRIOR generation's `leg_index` values too, to compute a new
|
|
377
|
+
// one that can never collide with the unique `(run_id, step_index,
|
|
378
|
+
// leg_index)` index — see design.md §A. Filtered from `allRunLegs`,
|
|
379
|
+
// the SAME query `driveRunOnce` already issued this outer iteration —
|
|
380
|
+
// no second `listRunLegs` round-trip for the same run (a confirmed LOW
|
|
381
|
+
// performance finding this fix closes).
|
|
382
|
+
const allStepLegs = allRunLegs.filter((l) => l.step_index === run.step_index);
|
|
383
|
+
const isReplan = allStepLegs.length > 0;
|
|
384
|
+
const newAttempt = isReplan ? run.attempt + 1 : run.attempt;
|
|
385
|
+
const startLegIndex = isReplan ? Math.max(...allStepLegs.map((l) => l.leg_index)) + 1 : 0;
|
|
386
|
+
// Leg inserts run BEFORE the attempt bump / deadline touch / yield
|
|
387
|
+
// write, not after — a review finding caught that writing those first
|
|
388
|
+
// let a concurrent loser (two `driveRunOnce` callers racing this exact
|
|
389
|
+
// step generation, per this function's own doc comment above) commit
|
|
390
|
+
// its own `bumpRunAttempt`/`touchRunDeadline`/`upsertYield` calls even
|
|
391
|
+
// though `insertRunLeg` below then throws `RunLegAlreadyExistsError`
|
|
392
|
+
// for THAT caller specifically, meaning only the OTHER (winning)
|
|
393
|
+
// caller's legs actually get submitted. A SECOND review pass (after the
|
|
394
|
+
// yield-ordering fix alone had already landed) caught that
|
|
395
|
+
// `bumpRunAttempt`/`touchRunDeadline` had been left in their ORIGINAL,
|
|
396
|
+
// still-unguarded position — a losing racer was still silently burning
|
|
397
|
+
// through `maxReplanAttempts` roughly twice as fast as the configured
|
|
398
|
+
// budget, racing ahead of `insertRunLeg`'s own loss detection.
|
|
399
|
+
// Inserting legs FIRST means a losing caller never reaches ANY of the
|
|
400
|
+
// three writes below — each one is only ever persisted by whichever
|
|
401
|
+
// caller's legs genuinely won the race.
|
|
402
|
+
legs.forEach((leg, i) => {
|
|
403
|
+
insertRunLeg({
|
|
404
|
+
runId: run.id,
|
|
405
|
+
stepIndex: run.step_index,
|
|
406
|
+
legIndex: startLegIndex + i,
|
|
407
|
+
definitionJson: JSON.stringify(leg),
|
|
408
|
+
attempt: newAttempt,
|
|
409
|
+
}, { db: ctx.db });
|
|
410
|
+
});
|
|
411
|
+
if (isReplan) {
|
|
412
|
+
// `bumpRunAttempt` + `touchRunDeadline`, NEVER `advanceRunStep`, for a
|
|
413
|
+
// genuine re-plan: `advanceRunStep` unconditionally resets `attempt`
|
|
414
|
+
// back to 0 as part of its contract, which would silently undo the
|
|
415
|
+
// bump on every re-plan and turn `maxReplanAttempts` into dead config
|
|
416
|
+
// (an infinite retry loop with no error ever surfacing) — see
|
|
417
|
+
// `store/runs.ts`'s `touchRunDeadline` doc and design.md §A.
|
|
418
|
+
bumpRunAttempt(run.id, { db: ctx.db });
|
|
419
|
+
touchRunDeadline(run.id, computeStepDeadline(now, ctx), { db: ctx.db });
|
|
420
|
+
}
|
|
421
|
+
if (normalized.yield !== undefined && step.yield_as) {
|
|
422
|
+
// A planner that yields on a step with no `yield_as` set simply has
|
|
423
|
+
// that value discarded here — the leg-building itself still
|
|
424
|
+
// succeeded, so this is not an error, just nothing to carry forward.
|
|
425
|
+
upsertYield({ runId: run.id, name: step.yield_as, value: normalized.yield, stepIndex: run.step_index }, { db: ctx.db });
|
|
426
|
+
}
|
|
427
|
+
if (!isReplan) {
|
|
428
|
+
// First entry (no legs existed yet) — byte-identical in final effect
|
|
429
|
+
// to before: stamp this step's first deadline (and the no-op
|
|
430
|
+
// `attempt = 0` reset `advanceRunStep` always performs, matching the
|
|
431
|
+
// `attempt: 0` every leg above was just inserted with).
|
|
432
|
+
advanceRunStep(run.id, { fromStepIndex: run.step_index, toStepIndex: run.step_index, deadlineAt: computeStepDeadline(now, ctx) }, { db: ctx.db });
|
|
433
|
+
}
|
|
434
|
+
return true;
|
|
435
|
+
}
|
|
436
|
+
// Same yield-reference resolution as the planner branch above, applied to
|
|
437
|
+
// the stored `payload_json` instead of `planner_params_json` — via the
|
|
438
|
+
// SAME shared `resolveSingleStepPayload` helper `handlers/steps.ts` uses,
|
|
439
|
+
// not a parallel reimplementation (a review finding caught the two call
|
|
440
|
+
// sites independently re-deriving the identical sequence).
|
|
441
|
+
const pool = getYieldPool(run.id, { db: ctx.db });
|
|
442
|
+
const resolvedStep = resolveSingleStepPayload(step, pool);
|
|
443
|
+
const definition = stepRowToDefinition(resolvedStep);
|
|
444
|
+
insertRunLeg({ runId: run.id, stepIndex: run.step_index, legIndex: 0, definitionJson: JSON.stringify(definition) }, { db: ctx.db });
|
|
445
|
+
advanceRunStep(run.id, { fromStepIndex: run.step_index, toStepIndex: run.step_index, deadlineAt: computeStepDeadline(now, ctx) }, { db: ctx.db });
|
|
446
|
+
return true;
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* `fail-run { reason }`: terminalize the run as `failed` and audit the
|
|
450
|
+
* reason (not persisted on the run row itself — no such column exists; see
|
|
451
|
+
* design.md's "Out of scope"). Always a store write.
|
|
452
|
+
*/
|
|
453
|
+
async function handleFailRun(run, reason, ctx) {
|
|
454
|
+
markRunTerminal(run.id, "failed", { db: ctx.db });
|
|
455
|
+
await ctx.onAudit({
|
|
456
|
+
action: "codex_cronoton_run.failed",
|
|
457
|
+
result: "failure",
|
|
458
|
+
targetKind: "codex_cronoton_run",
|
|
459
|
+
targetId: run.id,
|
|
460
|
+
detail: { codexCronotonId: run.codex_cronoton_id, stepIndex: run.step_index, reason },
|
|
461
|
+
});
|
|
462
|
+
return true;
|
|
463
|
+
}
|
|
464
|
+
/** Execute one `RunAction`, returning `true` iff it performed a store write. */
|
|
465
|
+
async function executeAction(action, run, step, stepLegs, allRunLegs, now, ctx) {
|
|
466
|
+
switch (action.kind) {
|
|
467
|
+
case "submit-leg":
|
|
468
|
+
return handleSubmitLeg(action.legIndex, run, stepLegs, now, ctx);
|
|
469
|
+
case "poll":
|
|
470
|
+
return handlePoll(stepLegs, step, ctx);
|
|
471
|
+
case "advance-step":
|
|
472
|
+
return handleAdvanceStep(run, now, ctx);
|
|
473
|
+
case "replan-step":
|
|
474
|
+
return handleReplanStep(run, step, allRunLegs, now, ctx);
|
|
475
|
+
case "fail-run":
|
|
476
|
+
return handleFailRun(run, action.reason, ctx);
|
|
477
|
+
case "wait":
|
|
478
|
+
return false;
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
/**
|
|
482
|
+
* Drive one run forward as far as it will synchronously go this pass — see
|
|
483
|
+
* the module doc above for why this loops internally rather than making a
|
|
484
|
+
* single `advanceRun` call. Returns once the run/current step vanishes (both
|
|
485
|
+
* defensive no-ops) or a pass makes no further progress.
|
|
486
|
+
*/
|
|
487
|
+
export async function driveRunOnce(runId, now, ctx) {
|
|
488
|
+
const initialRun = getRun(runId, { db: ctx.db });
|
|
489
|
+
if (!initialRun)
|
|
490
|
+
return; // the run vanished between listInFlightRuns and this call
|
|
491
|
+
// Fetched ONCE for the whole call, not per inner-loop pass: a run's
|
|
492
|
+
// parent cronoton_id is immutable for the run's lifetime, so its static
|
|
493
|
+
// step definitions can never change mid-call either — a confirmed LOW
|
|
494
|
+
// performance finding (this file's own `allRunLegs` hoist two lines below
|
|
495
|
+
// already recognized and closed exactly this class of redundant re-query
|
|
496
|
+
// for legs; the same re-query was still happening here for steps).
|
|
497
|
+
const steps = listStepsForCronoton(initialRun.codex_cronoton_id, { db: ctx.db });
|
|
498
|
+
for (;;) {
|
|
499
|
+
const run = getRun(runId, { db: ctx.db });
|
|
500
|
+
if (!run)
|
|
501
|
+
return; // the run vanished since the last pass
|
|
502
|
+
const step = steps.find((s) => s.step_index === run.step_index);
|
|
503
|
+
if (!step) {
|
|
504
|
+
// A run/step-definition mismatch (e.g. steps deleted out from under a
|
|
505
|
+
// live run) must fail loudly, not loop forever.
|
|
506
|
+
markRunTerminal(runId, "failed", { db: ctx.db });
|
|
507
|
+
await ctx.onAudit({
|
|
508
|
+
action: "codex_cronoton_run.step_missing",
|
|
509
|
+
result: "failure",
|
|
510
|
+
targetKind: "codex_cronoton_run",
|
|
511
|
+
targetId: runId,
|
|
512
|
+
detail: { codexCronotonId: run.codex_cronoton_id, stepIndex: run.step_index },
|
|
513
|
+
});
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
// Fetched ONCE per outer iteration — `handleReplanStep` used to issue
|
|
517
|
+
// its own independent `listRunLegs` query for the same run (a confirmed
|
|
518
|
+
// LOW performance finding from a cross-phase review); it now reuses
|
|
519
|
+
// this same result (filtered differently: by step_index only, across
|
|
520
|
+
// every attempt generation, not just the current one — see its own
|
|
521
|
+
// leg-index-collision-avoidance doc comment).
|
|
522
|
+
const allRunLegs = listRunLegs(runId, { db: ctx.db });
|
|
523
|
+
const stepLegs = allRunLegs.filter((l) => l.step_index === run.step_index && l.attempt === run.attempt);
|
|
524
|
+
const legs = stepLegs.map((l) => ({ legIndex: l.leg_index, status: l.status }));
|
|
525
|
+
const policy = {
|
|
526
|
+
stepMode: step.mode === "single" ? "serial" : "parallel",
|
|
527
|
+
canReplan: step.mode === "planner",
|
|
528
|
+
maxReplanAttempts: ctx.config?.maxReplanAttempts ?? DEFAULT_MAX_REPLAN_ATTEMPTS,
|
|
529
|
+
};
|
|
530
|
+
const runState = {
|
|
531
|
+
status: run.status,
|
|
532
|
+
stepIndex: run.step_index,
|
|
533
|
+
stepCount: run.step_count,
|
|
534
|
+
deadlineAt: run.deadline_at,
|
|
535
|
+
attempt: run.attempt,
|
|
536
|
+
};
|
|
537
|
+
const actions = advanceRun(runState, legs, now, policy);
|
|
538
|
+
let progressed = false;
|
|
539
|
+
for (const action of actions) {
|
|
540
|
+
// for...of with await, not Promise.all: a parallel step's multiple
|
|
541
|
+
// submit-leg actions must submit one at a time, never concurrently.
|
|
542
|
+
const mutated = await executeAction(action, run, step, stepLegs, allRunLegs, now, ctx);
|
|
543
|
+
if (mutated)
|
|
544
|
+
progressed = true;
|
|
545
|
+
}
|
|
546
|
+
if (!progressed)
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* Sweep every in-flight run, driving each forward one `driveRunOnce` pass.
|
|
552
|
+
* Per-run `try`/`catch` isolation — one run's thrown error (including
|
|
553
|
+
* `PlannerNotRegisteredError`) never blocks the others — mirrors
|
|
554
|
+
* `codexCronotonTickOnce`'s own per-row isolation posture.
|
|
555
|
+
*
|
|
556
|
+
* `excludeRunIds`, when given, skips those runs entirely this sweep. This
|
|
557
|
+
* exists so `loop.ts` can pass the run ids `codexCronotonTickOnce` already
|
|
558
|
+
* drove inline this SAME pass (design.md §B's "plan-and-submit step 0
|
|
559
|
+
* within one tick pass," built by calling `driveRunOnce` directly from the
|
|
560
|
+
* tick) — without it, a freshly created run would be driven twice in one
|
|
561
|
+
* loop callback: once inline by the tick, once again by this sweep moments
|
|
562
|
+
* later, doubling `pollRequestKeys` traffic for no benefit (the inline call
|
|
563
|
+
* already drove it as far as it would synchronously go).
|
|
564
|
+
*/
|
|
565
|
+
export async function advanceInFlightRunsOnce(now, ctx, excludeRunIds) {
|
|
566
|
+
const drivenIds = [];
|
|
567
|
+
const erroredIds = [];
|
|
568
|
+
const runs = listInFlightRuns({ db: ctx.db });
|
|
569
|
+
for (const run of runs) {
|
|
570
|
+
if (excludeRunIds?.has(run.id))
|
|
571
|
+
continue;
|
|
572
|
+
try {
|
|
573
|
+
await driveRunOnce(run.id, now, ctx);
|
|
574
|
+
drivenIds.push(run.id);
|
|
575
|
+
}
|
|
576
|
+
catch (err) {
|
|
577
|
+
console.error(`khronoton runs-driver: run ${run.id} errored while advancing`, err);
|
|
578
|
+
erroredIds.push(run.id);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
return { drivenIds, erroredIds };
|
|
582
|
+
}
|
package/dist/server/schema.d.ts
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The consolidated DDL installer for the
|
|
2
|
+
* The consolidated DDL installer for the seven codex-cronoton tables: the
|
|
3
|
+
* original three (`codex_cronotons`, `codex_cronoton_fires`,
|
|
4
|
+
* `codex_cronoton_manual_batches`) plus the chained-runs tables
|
|
5
|
+
* (`codex_cronoton_steps`, `codex_cronoton_runs`, `codex_cronoton_run_legs`)
|
|
6
|
+
* plus the step-to-step yield carry table (`codex_cronoton_run_yields`).
|
|
3
7
|
*
|
|
4
8
|
* This folds the Hub's incremental migration chain (base tables + a dozen later
|
|
5
9
|
* ALTERs) into the FINAL column shapes, emitted in a single idempotent call.
|
|
@@ -21,7 +25,7 @@
|
|
|
21
25
|
*/
|
|
22
26
|
import type { Database } from "./seams.js";
|
|
23
27
|
/**
|
|
24
|
-
* Installs the
|
|
28
|
+
* Installs the seven codex-cronoton tables and their indexes on the injected DB
|
|
25
29
|
* handle in a single idempotent call. Safe to re-run: every statement is guarded
|
|
26
30
|
* with `IF NOT EXISTS`.
|
|
27
31
|
*
|