@ancientpantheon/khronoton-core 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CHANGELOG.md +20 -0
  2. package/README.md +4 -0
  3. package/dist/blockchain/stoachain.d.ts +1 -1
  4. package/dist/blockchain/stoachain.js +16 -1
  5. package/dist/{chunk-KQZYMPZS.js → chunk-7H5TM3CS.js} +1 -1
  6. package/dist/{chunk-GIGLXIMC.js → chunk-IAK4UEAA.js} +710 -23
  7. package/dist/{context-CApk-LP4.d.ts → context-P-rCtBbJ.d.ts} +1 -1
  8. package/dist/handlers/execute.d.ts +12 -4
  9. package/dist/handlers/execute.js +42 -5
  10. package/dist/handlers/index.d.ts +2 -1
  11. package/dist/handlers/index.js +4 -1
  12. package/dist/handlers/read.d.ts +11 -0
  13. package/dist/handlers/read.js +65 -1
  14. package/dist/handlers/steps.d.ts +3 -0
  15. package/dist/handlers/steps.js +115 -0
  16. package/dist/hooks/index.d.ts +3 -3
  17. package/dist/hooks/index.js +2 -2
  18. package/dist/provider/index.d.ts +3 -3
  19. package/dist/provider/index.js +33 -2
  20. package/dist/{seams-DUtxDw_U.d.ts → seams-Co8Tq4ka.d.ts} +24 -0
  21. package/dist/server/executor.d.ts +14 -1
  22. package/dist/server/executor.js +144 -26
  23. package/dist/server/index.d.ts +11 -2
  24. package/dist/server/index.js +29 -1
  25. package/dist/server/loop.d.ts +17 -13
  26. package/dist/server/loop.js +19 -12
  27. package/dist/server/planners/evict.d.ts +20 -0
  28. package/dist/server/planners/evict.js +143 -0
  29. package/dist/server/planners/inject.d.ts +22 -0
  30. package/dist/server/planners/inject.js +120 -0
  31. package/dist/server/planners/registry.d.ts +80 -0
  32. package/dist/server/planners/registry.js +50 -0
  33. package/dist/server/planners/stoa-unwrap.d.ts +23 -0
  34. package/dist/server/planners/stoa-unwrap.js +204 -0
  35. package/dist/server/planners/stoa-wrap.d.ts +22 -0
  36. package/dist/server/planners/stoa-wrap.js +201 -0
  37. package/dist/server/planners/unstall.d.ts +25 -0
  38. package/dist/server/planners/unstall.js +135 -0
  39. package/dist/server/pure/run-advance.d.ts +76 -0
  40. package/dist/server/pure/run-advance.js +94 -0
  41. package/dist/server/pure/split-plan.d.ts +144 -0
  42. package/dist/server/pure/split-plan.js +362 -0
  43. package/dist/server/resolvers.js +48 -6
  44. package/dist/server/runs-driver.d.ts +92 -0
  45. package/dist/server/runs-driver.js +511 -0
  46. package/dist/server/schema.d.ts +5 -2
  47. package/dist/server/schema.js +73 -1
  48. package/dist/server/seams.d.ts +28 -0
  49. package/dist/server/seams.js +4 -0
  50. package/dist/server/store/errors.d.ts +38 -0
  51. package/dist/server/store/errors.js +61 -0
  52. package/dist/server/store/index.d.ts +6 -2
  53. package/dist/server/store/index.js +6 -2
  54. package/dist/server/store/mappers.d.ts +11 -0
  55. package/dist/server/store/mappers.js +20 -0
  56. package/dist/server/store/runs.d.ts +233 -0
  57. package/dist/server/store/runs.js +341 -0
  58. package/dist/server/store/steps.d.ts +42 -0
  59. package/dist/server/store/steps.js +51 -0
  60. package/dist/server/tick.d.ts +7 -0
  61. package/dist/server/tick.js +50 -1
  62. package/dist/server/types.d.ts +42 -3
  63. package/dist/ui/index.js +2 -2
  64. package/package.json +1 -1
@@ -0,0 +1,511 @@
1
+ /**
2
+ * The non-blocking chained-run driver — executes Phase 2's pure `advanceRun`
3
+ * decision function against the real store/chain, per
4
+ * `docs/work/khronoton-chained-runs-phase-3/design.md` §D/§F.
5
+ *
6
+ * `driveRunOnce` is a SINGLE-RUN, loop-until-no-progress pass: it repeatedly
7
+ * (1) reads the run + its current step + that step's legs, (2) calls the pure
8
+ * `advanceRun` to decide what happens next, and (3) executes the returned
9
+ * `RunAction`s against the store/executor — re-reading fresh state and
10
+ * calling `advanceRun` again after every mutating action, stopping only once
11
+ * a pass makes NO further store write (a `poll` that landed nothing new, a
12
+ * `submit-leg` that failed, or the defensive `wait`), the run/step vanishes,
13
+ * or an action throws.
14
+ *
15
+ * This internal looping is deliberate, not incidental: `advanceRun` itself
16
+ * documents its own terminal-run check (step 1) as "defends against a driver
17
+ * that calls advanceRun on a run it should have already stopped touching" —
18
+ * i.e. the pure function already anticipates being re-invoked, within one
19
+ * driver pass, on a run this SAME pass just terminalized. It is also the only
20
+ * way to satisfy design.md §B's requirement that a freshly created run's step
21
+ * 0 gets "planned-and-submitted" — i.e. both `replan-step` (build leg 0) AND
22
+ * `submit-leg` (submit it) — within the SAME tick pass that created the run,
23
+ * rather than waiting a full `tickIntervalMs` for the next
24
+ * `advanceInFlightRunsOnce` pass to notice the freshly-built leg.
25
+ *
26
+ * `advanceInFlightRunsOnce` is the scheduler-facing sweep: every in-flight
27
+ * run, one `driveRunOnce` each, same per-row isolation posture
28
+ * `codexCronotonTickOnce` already applies (one run's failure never blocks the
29
+ * others).
30
+ *
31
+ * Deliberately does NOT import anything from `./tick.js` or `./loop.js` —
32
+ * both of those import FROM this module in a later wave of this same phase;
33
+ * importing `TickCtx` here instead of declaring the structurally-compatible
34
+ * `RunsDriverCtx` below would create a circular import.
35
+ */
36
+ import { DEFAULT_MAX_REPLAN_ATTEMPTS, RUN_STEP_DEADLINE_MS } from "./seams.js";
37
+ import { advanceRunStep, bumpRunAttempt, getRun, insertRunLeg, listInFlightRuns, listRunLegs, claimLegForSubmit, releaseLegClaim, markLegConfirmed, markLegFailed, markLegSubmitted, markRunTerminal, touchRunDeadline, } from "./store/runs.js";
38
+ import { listStepsForCronoton } from "./store/steps.js";
39
+ import { RunLegAlreadyExistsError } from "./store/errors.js";
40
+ import { stepRowToDefinition } from "./store/mappers.js";
41
+ import { executeCodexTransaction } from "./executor.js";
42
+ import { resolvePlannerFor } from "./planners/registry.js";
43
+ import { advanceRun } from "./pure/run-advance.js";
44
+ /**
45
+ * Thrown by `driveRunOnce` when a `mode: 'planner'` step names a
46
+ * `planner_name` that `resolvePlannerFor` cannot resolve — e.g. a typo, or a
47
+ * planner that genuinely hasn't been registered yet. The `StepPlanner`
48
+ * registry itself (`planners/registry.ts`) and its real planners
49
+ * (`ouronet.aqp-evict`/`ouronet.aqp-inject`/`ouronet.aqp-unstall`) are fully
50
+ * built; this error reports an unresolved NAME against that registry, not a
51
+ * missing registry mechanism.
52
+ */
53
+ export class PlannerNotRegisteredError extends Error {
54
+ constructor(plannerName) {
55
+ super(`planner '${plannerName}' is not registered in the StepPlanner registry`);
56
+ this.name = "PlannerNotRegisteredError";
57
+ }
58
+ }
59
+ /** This pass's per-step barrier deadline, `now + (config override ?? the module default)`. */
60
+ function computeStepDeadline(now, ctx) {
61
+ return new Date(now.getTime() + (ctx.config?.runStepDeadlineMs ?? RUN_STEP_DEADLINE_MS)).toISOString();
62
+ }
63
+ /**
64
+ * Parse a leg's stored `definition_json` and check its MINIMAL shape before
65
+ * it is trusted enough to sign and submit as a real transaction. Today's
66
+ * only writer (`handleReplanStep` → `stepRowToDefinition`) is type-checked
67
+ * and always produces a well-formed value, so this never actually fires on
68
+ * any path Phase 3 exercises — it exists because `insertRunLeg` is public
69
+ * store API (not module-private), and Phase 4's planner registry will be
70
+ * the next writer of `definition_json`. A shallow check (not a full schema
71
+ * validator): non-empty `pactCode`, an array `signers`, an object
72
+ * `gasPayer`/`config`. Throws a plain `Error` (caught by the caller) rather
73
+ * than returning a sentinel, since every other parse failure in this layer
74
+ * (a corrupt `schedule_config_json`, for example — see `store/claim.ts`) is
75
+ * handled the same way: catch at the call site, never let `JSON.parse`'s
76
+ * own exception or a malformed shape reach the signer.
77
+ */
78
+ function parseLegDefinition(definitionJson) {
79
+ const parsed = JSON.parse(definitionJson);
80
+ if (typeof parsed !== "object" || parsed === null) {
81
+ throw new Error("leg definition is not an object");
82
+ }
83
+ const def = parsed;
84
+ if (typeof def.pactCode !== "string" || def.pactCode.length === 0) {
85
+ throw new Error("leg definition is missing a non-empty pactCode");
86
+ }
87
+ if (!Array.isArray(def.signers)) {
88
+ throw new Error("leg definition is missing a signers array");
89
+ }
90
+ if (typeof def.gasPayer !== "object" || def.gasPayer === null) {
91
+ throw new Error("leg definition is missing a gasPayer object");
92
+ }
93
+ if (typeof def.config !== "object" || def.config === null) {
94
+ throw new Error("leg definition is missing a config object");
95
+ }
96
+ return parsed;
97
+ }
98
+ /**
99
+ * `submit-leg { legIndex }`: submit the already-`pending` leg at this index
100
+ * (built by an earlier `replan-step` pass; `submit-leg` never builds a leg
101
+ * itself). Returns `true` iff the leg flipped to `submitted` (a store write
102
+ * happened) — a submit failure leaves the leg `pending` and returns `false`,
103
+ * so the caller's loop stops rather than busy-retrying within this same pass;
104
+ * the *next* `driveRunOnce` pass naturally re-selects the same still-pending
105
+ * leg (design.md §F — no retry-counting write exists or is needed).
106
+ *
107
+ * CONCURRENCY (Phase 5 fix): before the chain submission
108
+ * (`executeCodexTransaction`) is awaited, this now calls
109
+ * `claimLegForSubmit` — a real claim-before-submit step, mirroring
110
+ * `claim.ts`'s `claimDueCodexCronoton` idiom. Two concurrent callers racing
111
+ * the same `pending` leg (e.g. `executeNow`'s chained-row branch invoking
112
+ * `driveRunOnce` directly from an HTTP handler, racing the background
113
+ * loop's own `advanceInFlightRunsOnce` sweep — both are public exports, and
114
+ * only the LOOP's own pass sits behind `loop.ts`'s re-entrancy guard) can
115
+ * never both win the claim, so at most one of them ever reaches the real
116
+ * network call. A lost claim (the leg was already claimed/submitted by a
117
+ * concurrent winner, or moved on since `stepLegs` was read) returns `false`
118
+ * immediately — exactly like "wait", never a second submit. A submit
119
+ * attempt that does not land releases the claim (`releaseLegClaim`) so a
120
+ * LATER pass can still retry the still-`pending` leg, preserving the
121
+ * existing no-retry-cap posture (design.md §F) unchanged.
122
+ */
123
+ async function handleSubmitLeg(legIndex, run, stepLegs, now, ctx) {
124
+ const leg = stepLegs.find((l) => l.leg_index === legIndex);
125
+ if (!leg) {
126
+ // Defensive — should never happen, since every submit-leg action targets
127
+ // a pending leg advanceRun itself selected from the very legs array this
128
+ // function passed it.
129
+ await ctx.onAudit({
130
+ action: "codex_cronoton_run_leg.not_found",
131
+ result: "failure",
132
+ targetKind: "codex_cronoton_run",
133
+ targetId: run.id,
134
+ detail: { codexCronotonId: run.codex_cronoton_id, stepIndex: run.step_index, legIndex },
135
+ });
136
+ return false;
137
+ }
138
+ let definition;
139
+ try {
140
+ definition = parseLegDefinition(leg.definition_json);
141
+ }
142
+ catch (err) {
143
+ // Malformed/invalid stored definition — never hand this to the signer.
144
+ // Leaves the leg `pending` (same posture as a submit failure below): no
145
+ // request key was ever derived, so nothing unsafe happened, and this is
146
+ // deterministic (retrying will fail the same way every pass) — but
147
+ // Phase 3 has no retry-cap mechanism (see design.md's Out of scope), so
148
+ // an operator noticing the repeated audit event is the backstop today.
149
+ await ctx.onAudit({
150
+ action: "codex_cronoton_run_leg.invalid_definition",
151
+ result: "failure",
152
+ targetKind: "codex_cronoton_run_leg",
153
+ targetId: leg.id,
154
+ detail: {
155
+ runId: run.id,
156
+ stepIndex: run.step_index,
157
+ legIndex,
158
+ error: err instanceof Error ? err.message : String(err),
159
+ },
160
+ });
161
+ return false;
162
+ }
163
+ const claimed = claimLegForSubmit(leg.id, now.toISOString(), { db: ctx.db });
164
+ if (!claimed) {
165
+ // Lost the race: a concurrent caller already claimed (or submitted, or
166
+ // otherwise moved on) this exact leg since `stepLegs` was read. Back off
167
+ // silently — this pass makes no progress on this leg, exactly like
168
+ // "wait" — never a second submit attempt.
169
+ return false;
170
+ }
171
+ const result = await executeCodexTransaction(definition, "submit", {
172
+ runtime: ctx.runtime,
173
+ resolver: ctx.resolver,
174
+ config: ctx.config,
175
+ });
176
+ if (result.ok && result.requestKey && result.chainId) {
177
+ markLegSubmitted(leg.id, { requestKey: result.requestKey, chainId: result.chainId }, { db: ctx.db });
178
+ return true;
179
+ }
180
+ // Release the claim so a LATER pass can still retry this still-`pending`
181
+ // leg — the existing no-retry-cap posture (design.md §F) is unchanged; only
182
+ // the CONCURRENT-double-submit window this claim closes is new.
183
+ releaseLegClaim(leg.id, { db: ctx.db });
184
+ await ctx.onAudit({
185
+ action: "codex_cronoton_run_leg.submit_failed",
186
+ result: "failure",
187
+ targetKind: "codex_cronoton_run_leg",
188
+ targetId: leg.id,
189
+ detail: { runId: run.id, stepIndex: run.step_index, legIndex, error: result.error },
190
+ });
191
+ return false;
192
+ }
193
+ /**
194
+ * `poll {}`: one `pollRequestKeys` call per distinct chain for every
195
+ * `submitted` leg of the current step (Phase 0's "one request per chain per
196
+ * tick" contract). Returns `true` iff at least one leg landed (confirmed or
197
+ * failed) this pass.
198
+ */
199
+ async function handlePoll(stepLegs, ctx) {
200
+ const submitted = stepLegs.filter((l) => l.status === "submitted" && l.request_key != null && l.chain_id != null);
201
+ if (submitted.length === 0)
202
+ return false;
203
+ const byChain = new Map();
204
+ for (const leg of submitted) {
205
+ const group = byChain.get(leg.chain_id) ?? [];
206
+ group.push(leg);
207
+ byChain.set(leg.chain_id, group);
208
+ }
209
+ let mutated = false;
210
+ for (const [chainId, legsForChain] of byChain) {
211
+ const client = ctx.runtime.createClient(ctx.runtime.getPactUrl(chainId));
212
+ const polled = (await client.pollRequestKeys?.(legsForChain.map((l) => l.request_key), chainId)) ?? {};
213
+ for (const leg of legsForChain) {
214
+ const landed = polled[leg.request_key];
215
+ if (!landed)
216
+ continue; // still pending — absent key means "not landed yet"
217
+ if (landed.result.status === "success") {
218
+ markLegConfirmed(leg.id, { db: ctx.db });
219
+ }
220
+ else {
221
+ markLegFailed(leg.id, { errorMessage: landed.result.error?.message ?? "transaction failed on-chain" }, { db: ctx.db });
222
+ }
223
+ mutated = true;
224
+ }
225
+ }
226
+ return mutated;
227
+ }
228
+ /**
229
+ * `advance-step {}`: the barrier for the current step is met. Terminal
230
+ * (`succeeded`) if this was the last step, else move `step_index` forward and
231
+ * reset `attempt`/stamp a fresh deadline for the next step. Always a store
232
+ * write.
233
+ */
234
+ function handleAdvanceStep(run, now, ctx) {
235
+ if (run.step_index + 1 >= run.step_count) {
236
+ markRunTerminal(run.id, "succeeded", { db: ctx.db });
237
+ }
238
+ else {
239
+ advanceRunStep(run.id, { fromStepIndex: run.step_index, toStepIndex: run.step_index + 1, deadlineAt: computeStepDeadline(now, ctx) }, { db: ctx.db });
240
+ }
241
+ return true;
242
+ }
243
+ /**
244
+ * `replan-step { reason }`: as of Phase 4.1, `driveRunOnce` sets
245
+ * `canReplan: step.mode === "planner"`, so this action is reached in TWO
246
+ * distinct situations, both handled here — a step's very first entry
247
+ * (`advanceRun`'s "no legs yet" branch, no prior legs for this step) and a
248
+ * genuine re-plan after a leg failure (`replanOrFail`, prior legs for this
249
+ * step already exist). The `isReplan` check below distinguishes them: a
250
+ * `mode: 'planner'` step looks its `planner_name` up in Phase 4's
251
+ * `StepPlanner` registry and calls it to produce this pass's legs (zero legs
252
+ * is treated exactly like a cleared barrier, matching `handleAdvanceStep`'s
253
+ * own terminal-vs-advance branch verbatim rather than duplicating it) — on a
254
+ * genuine re-plan, the new legs land at a bumped `attempt` generation and a
255
+ * fresh `leg_index` range (never colliding with the prior generation's now-
256
+ * failed legs), and the run's `attempt` counter and step deadline are
257
+ * advanced directly rather than through `advanceRunStep` (whose own
258
+ * attempt-reset side effect would otherwise silently undo the bump — see
259
+ * design.md §A and `store/runs.ts`'s `touchRunDeadline` doc comment); a
260
+ * `mode: 'single'` step builds its one fixed leg from the step row and stamps
261
+ * this step's first deadline (single-mode steps never set `canReplan`, so
262
+ * `isReplan` is always `false` on this branch). Always a store write (or a
263
+ * throw).
264
+ *
265
+ * CRITICAL, cross-phase-review-found fix (NARROWLY SCOPED — a first
266
+ * version of this fix wrapped EVERY call unconditionally and was itself
267
+ * found, on the SAME review's own terminal pass, to introduce two new
268
+ * regressions; both are explained below, not just the original bug):
269
+ *
270
+ * The original bug: a step whose CURRENT generation has NEVER had a
271
+ * single leg (`stepHasAnyLegs` below, `false`) is the ONLY situation
272
+ * `advanceRun`'s "no legs yet" decision (step 2, pure/run-advance.ts) can
273
+ * reach — and that decision is checked UNCONDITIONALLY, before its OWN
274
+ * deadline-exceeded check (step 4), REGARDLESS of whether `deadline_at`
275
+ * happens to already be set. So if the planner lookup/call (or, for
276
+ * `mode:'single'`, `stepRowToDefinition`) throws EVERY time before ever
277
+ * successfully inserting this step's first leg, `legs.length` stays `0`
278
+ * forever and `advanceRun` can NEVER reach its own deadline/attempt-budget
279
+ * recovery machinery — not just on a run's absolute first step, but on
280
+ * ANY step whose planner never manages one successful plan. Nothing else
281
+ * in this codebase can terminalize such a run (no exposed handler, no
282
+ * cron); left unguarded, it permanently occupies its cronoton's self-block
283
+ * slot (`schema.ts`'s partial unique index).
284
+ *
285
+ * Regression #1 (HIGH) an unconditional catch introduced: a GENUINE
286
+ * re-plan (`stepHasAnyLegs === true` — this step already built at least
287
+ * one PRIOR leg; `replanOrFail`'s own `maxReplanAttempts` budget already
288
+ * decided this attempt was worth making) throwing must NOT be
289
+ * terminalized here — that would silently zero out the retry budget
290
+ * `maxReplanAttempts`/`replanOrFail` exist to grant. Letting it propagate
291
+ * exactly as before this fix (isolated by `advanceInFlightRunsOnce`,
292
+ * retried or genuinely exhausted by the NEXT pass's own deadline/attempt
293
+ * check) is correct; only the `stepHasAnyLegs === false` case ever needed
294
+ * protection, because only it has no OTHER recovery path at all.
295
+ *
296
+ * Regression #2 (CRITICAL) an unconditional catch introduced: TWO
297
+ * concurrent `driveRunOnce` callers (e.g. `executeNow`'s HTTP-triggered
298
+ * call racing the background loop's own sweep — `store/runs.ts`'s own doc
299
+ * comment names this exact scenario) can both reach a step's first entry
300
+ * and race on `insertRunLeg`; the loser gets `RunLegAlreadyExistsError`
301
+ * while the WINNER is concurrently making real progress. Terminalizing
302
+ * the LOSER's run would incorrectly fail a run that is actually
303
+ * succeeding via the winner's write — `RunLegAlreadyExistsError` is
304
+ * therefore never terminalized here either, regardless of
305
+ * `stepHasAnyLegs`.
306
+ */
307
+ async function handleReplanStep(run, step, allRunLegs, now, ctx) {
308
+ const stepHasAnyLegs = allRunLegs.some((l) => l.step_index === run.step_index);
309
+ try {
310
+ return await handleReplanStepInner(run, step, allRunLegs, now, ctx);
311
+ }
312
+ catch (err) {
313
+ if (stepHasAnyLegs || err instanceof RunLegAlreadyExistsError) {
314
+ // Regression #1 / #2 guards (see doc comment above) — let it
315
+ // propagate exactly as before this fix, isolated by
316
+ // `advanceInFlightRunsOnce`, never terminalized here.
317
+ throw err;
318
+ }
319
+ // This step's CURRENT generation has never had a single leg — the
320
+ // ONLY case with no other recovery path at all (see doc comment).
321
+ const reason = `replan-step threw: ${err instanceof Error ? err.message : String(err)}`;
322
+ await handleFailRun(run, reason, ctx);
323
+ throw err;
324
+ }
325
+ }
326
+ async function handleReplanStepInner(run, step, allRunLegs, now, ctx) {
327
+ if (step.mode === "planner") {
328
+ const resolved = resolvePlannerFor(step, ctx);
329
+ if (!resolved) {
330
+ throw new PlannerNotRegisteredError(step.planner_name ?? "(unnamed)");
331
+ }
332
+ const legs = await resolved.planner(step, resolved.plannerCtx);
333
+ if (legs.length === 0) {
334
+ // The planner decided this step needs no work this pass (e.g. nothing
335
+ // owed) — treat exactly like a cleared barrier, matching
336
+ // `handleAdvanceStep`'s own terminal-vs-advance branch verbatim rather
337
+ // than duplicating it.
338
+ return handleAdvanceStep(run, now, ctx);
339
+ }
340
+ // Every generation of this step's legs, NOT attempt-filtered — unlike
341
+ // `driveRunOnce`'s own `stepLegs` (which only ever sees the CURRENT
342
+ // generation for the barrier/submit/poll decision), a re-plan needs to
343
+ // see every PRIOR generation's `leg_index` values too, to compute a new
344
+ // one that can never collide with the unique `(run_id, step_index,
345
+ // leg_index)` index — see design.md §A. Filtered from `allRunLegs`,
346
+ // the SAME query `driveRunOnce` already issued this outer iteration —
347
+ // no second `listRunLegs` round-trip for the same run (a confirmed LOW
348
+ // performance finding this fix closes).
349
+ const allStepLegs = allRunLegs.filter((l) => l.step_index === run.step_index);
350
+ const isReplan = allStepLegs.length > 0;
351
+ const newAttempt = isReplan ? run.attempt + 1 : run.attempt;
352
+ const startLegIndex = isReplan ? Math.max(...allStepLegs.map((l) => l.leg_index)) + 1 : 0;
353
+ if (isReplan) {
354
+ // `bumpRunAttempt` + `touchRunDeadline`, NEVER `advanceRunStep`, for a
355
+ // genuine re-plan: `advanceRunStep` unconditionally resets `attempt`
356
+ // back to 0 as part of its contract, which would silently undo the
357
+ // bump on every re-plan and turn `maxReplanAttempts` into dead config
358
+ // (an infinite retry loop with no error ever surfacing) — see
359
+ // `store/runs.ts`'s `touchRunDeadline` doc and design.md §A.
360
+ bumpRunAttempt(run.id, { db: ctx.db });
361
+ touchRunDeadline(run.id, computeStepDeadline(now, ctx), { db: ctx.db });
362
+ }
363
+ legs.forEach((leg, i) => {
364
+ insertRunLeg({
365
+ runId: run.id,
366
+ stepIndex: run.step_index,
367
+ legIndex: startLegIndex + i,
368
+ definitionJson: JSON.stringify(leg),
369
+ attempt: newAttempt,
370
+ }, { db: ctx.db });
371
+ });
372
+ if (!isReplan) {
373
+ // First entry (no legs existed yet) — byte-identical in final effect
374
+ // to before: stamp this step's first deadline (and the no-op
375
+ // `attempt = 0` reset `advanceRunStep` always performs, matching the
376
+ // `attempt: 0` every leg above was just inserted with).
377
+ advanceRunStep(run.id, { fromStepIndex: run.step_index, toStepIndex: run.step_index, deadlineAt: computeStepDeadline(now, ctx) }, { db: ctx.db });
378
+ }
379
+ return true;
380
+ }
381
+ const definition = stepRowToDefinition(step);
382
+ insertRunLeg({ runId: run.id, stepIndex: run.step_index, legIndex: 0, definitionJson: JSON.stringify(definition) }, { db: ctx.db });
383
+ advanceRunStep(run.id, { fromStepIndex: run.step_index, toStepIndex: run.step_index, deadlineAt: computeStepDeadline(now, ctx) }, { db: ctx.db });
384
+ return true;
385
+ }
386
+ /**
387
+ * `fail-run { reason }`: terminalize the run as `failed` and audit the
388
+ * reason (not persisted on the run row itself — no such column exists; see
389
+ * design.md's "Out of scope"). Always a store write.
390
+ */
391
+ async function handleFailRun(run, reason, ctx) {
392
+ markRunTerminal(run.id, "failed", { db: ctx.db });
393
+ await ctx.onAudit({
394
+ action: "codex_cronoton_run.failed",
395
+ result: "failure",
396
+ targetKind: "codex_cronoton_run",
397
+ targetId: run.id,
398
+ detail: { codexCronotonId: run.codex_cronoton_id, stepIndex: run.step_index, reason },
399
+ });
400
+ return true;
401
+ }
402
+ /** Execute one `RunAction`, returning `true` iff it performed a store write. */
403
+ async function executeAction(action, run, step, stepLegs, allRunLegs, now, ctx) {
404
+ switch (action.kind) {
405
+ case "submit-leg":
406
+ return handleSubmitLeg(action.legIndex, run, stepLegs, now, ctx);
407
+ case "poll":
408
+ return handlePoll(stepLegs, ctx);
409
+ case "advance-step":
410
+ return handleAdvanceStep(run, now, ctx);
411
+ case "replan-step":
412
+ return handleReplanStep(run, step, allRunLegs, now, ctx);
413
+ case "fail-run":
414
+ return handleFailRun(run, action.reason, ctx);
415
+ case "wait":
416
+ return false;
417
+ }
418
+ }
419
+ /**
420
+ * Drive one run forward as far as it will synchronously go this pass — see
421
+ * the module doc above for why this loops internally rather than making a
422
+ * single `advanceRun` call. Returns once the run/current step vanishes (both
423
+ * defensive no-ops) or a pass makes no further progress.
424
+ */
425
+ export async function driveRunOnce(runId, now, ctx) {
426
+ for (;;) {
427
+ const run = getRun(runId, { db: ctx.db });
428
+ if (!run)
429
+ return; // the run vanished between listInFlightRuns and this call
430
+ const steps = listStepsForCronoton(run.codex_cronoton_id, { db: ctx.db });
431
+ const step = steps.find((s) => s.step_index === run.step_index);
432
+ if (!step) {
433
+ // A run/step-definition mismatch (e.g. steps deleted out from under a
434
+ // live run) must fail loudly, not loop forever.
435
+ markRunTerminal(runId, "failed", { db: ctx.db });
436
+ await ctx.onAudit({
437
+ action: "codex_cronoton_run.step_missing",
438
+ result: "failure",
439
+ targetKind: "codex_cronoton_run",
440
+ targetId: runId,
441
+ detail: { codexCronotonId: run.codex_cronoton_id, stepIndex: run.step_index },
442
+ });
443
+ return;
444
+ }
445
+ // Fetched ONCE per outer iteration — `handleReplanStep` used to issue
446
+ // its own independent `listRunLegs` query for the same run (a confirmed
447
+ // LOW performance finding from a cross-phase review); it now reuses
448
+ // this same result (filtered differently: by step_index only, across
449
+ // every attempt generation, not just the current one — see its own
450
+ // leg-index-collision-avoidance doc comment).
451
+ const allRunLegs = listRunLegs(runId, { db: ctx.db });
452
+ const stepLegs = allRunLegs.filter((l) => l.step_index === run.step_index && l.attempt === run.attempt);
453
+ const legs = stepLegs.map((l) => ({ legIndex: l.leg_index, status: l.status }));
454
+ const policy = {
455
+ stepMode: step.mode === "single" ? "serial" : "parallel",
456
+ canReplan: step.mode === "planner",
457
+ maxReplanAttempts: ctx.config?.maxReplanAttempts ?? DEFAULT_MAX_REPLAN_ATTEMPTS,
458
+ };
459
+ const runState = {
460
+ status: run.status,
461
+ stepIndex: run.step_index,
462
+ stepCount: run.step_count,
463
+ deadlineAt: run.deadline_at,
464
+ attempt: run.attempt,
465
+ };
466
+ const actions = advanceRun(runState, legs, now, policy);
467
+ let progressed = false;
468
+ for (const action of actions) {
469
+ // for...of with await, not Promise.all: a parallel step's multiple
470
+ // submit-leg actions must submit one at a time, never concurrently.
471
+ const mutated = await executeAction(action, run, step, stepLegs, allRunLegs, now, ctx);
472
+ if (mutated)
473
+ progressed = true;
474
+ }
475
+ if (!progressed)
476
+ return;
477
+ }
478
+ }
479
+ /**
480
+ * Sweep every in-flight run, driving each forward one `driveRunOnce` pass.
481
+ * Per-run `try`/`catch` isolation — one run's thrown error (including
482
+ * `PlannerNotRegisteredError`) never blocks the others — mirrors
483
+ * `codexCronotonTickOnce`'s own per-row isolation posture.
484
+ *
485
+ * `excludeRunIds`, when given, skips those runs entirely this sweep. This
486
+ * exists so `loop.ts` can pass the run ids `codexCronotonTickOnce` already
487
+ * drove inline this SAME pass (design.md §B's "plan-and-submit step 0
488
+ * within one tick pass," built by calling `driveRunOnce` directly from the
489
+ * tick) — without it, a freshly created run would be driven twice in one
490
+ * loop callback: once inline by the tick, once again by this sweep moments
491
+ * later, doubling `pollRequestKeys` traffic for no benefit (the inline call
492
+ * already drove it as far as it would synchronously go).
493
+ */
494
+ export async function advanceInFlightRunsOnce(now, ctx, excludeRunIds) {
495
+ const drivenIds = [];
496
+ const erroredIds = [];
497
+ const runs = listInFlightRuns({ db: ctx.db });
498
+ for (const run of runs) {
499
+ if (excludeRunIds?.has(run.id))
500
+ continue;
501
+ try {
502
+ await driveRunOnce(run.id, now, ctx);
503
+ drivenIds.push(run.id);
504
+ }
505
+ catch (err) {
506
+ console.error(`khronoton runs-driver: run ${run.id} errored while advancing`, err);
507
+ erroredIds.push(run.id);
508
+ }
509
+ }
510
+ return { drivenIds, erroredIds };
511
+ }
@@ -1,5 +1,8 @@
1
1
  /**
2
- * The consolidated DDL installer for the three codex-cronoton tables.
2
+ * The consolidated DDL installer for the six codex-cronoton tables: the
3
+ * original three (`codex_cronotons`, `codex_cronoton_fires`,
4
+ * `codex_cronoton_manual_batches`) plus the chained-runs tables
5
+ * (`codex_cronoton_steps`, `codex_cronoton_runs`, `codex_cronoton_run_legs`).
3
6
  *
4
7
  * This folds the Hub's incremental migration chain (base tables + a dozen later
5
8
  * ALTERs) into the FINAL column shapes, emitted in a single idempotent call.
@@ -21,7 +24,7 @@
21
24
  */
22
25
  import type { Database } from "./seams.js";
23
26
  /**
24
- * Installs the three codex-cronoton tables and their indexes on the injected DB
27
+ * Installs the six codex-cronoton tables and their indexes on the injected DB
25
28
  * handle in a single idempotent call. Safe to re-run: every statement is guarded
26
29
  * with `IF NOT EXISTS`.
27
30
  *
@@ -79,6 +79,71 @@ CREATE INDEX IF NOT EXISTS idx_codex_manual_batches_due
79
79
 
80
80
  CREATE INDEX IF NOT EXISTS idx_codex_manual_batches_cronoton
81
81
  ON codex_cronoton_manual_batches(codex_cronoton_id);
82
+
83
+ CREATE TABLE IF NOT EXISTS codex_cronoton_steps (
84
+ id TEXT PRIMARY KEY,
85
+ codex_cronoton_id TEXT NOT NULL REFERENCES codex_cronotons(id) ON DELETE CASCADE,
86
+ step_index INTEGER NOT NULL,
87
+ mode TEXT NOT NULL CHECK (mode IN ('single', 'planner')),
88
+ pact_code TEXT,
89
+ config_json TEXT,
90
+ payload_json TEXT,
91
+ gas_payer_json TEXT,
92
+ signers_json TEXT,
93
+ planner_name TEXT,
94
+ planner_params_json TEXT,
95
+ idempotent INTEGER NOT NULL DEFAULT 0,
96
+ created_at TEXT NOT NULL,
97
+ modified_at TEXT NOT NULL,
98
+ CHECK (
99
+ (mode = 'single' AND pact_code IS NOT NULL) OR
100
+ (mode = 'planner' AND planner_name IS NOT NULL)
101
+ )
102
+ );
103
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_codex_cronoton_steps_order
104
+ ON codex_cronoton_steps(codex_cronoton_id, step_index);
105
+
106
+ CREATE TABLE IF NOT EXISTS codex_cronoton_runs (
107
+ id TEXT PRIMARY KEY,
108
+ codex_cronoton_id TEXT NOT NULL REFERENCES codex_cronotons(id) ON DELETE CASCADE,
109
+ fire_id TEXT REFERENCES codex_cronoton_fires(id),
110
+ status TEXT NOT NULL DEFAULT 'planning' CHECK (status IN (
111
+ 'planning', 'awaiting', 'advancing', 'succeeded', 'failed', 'abandoned'
112
+ )),
113
+ step_index INTEGER NOT NULL DEFAULT 0,
114
+ step_count INTEGER NOT NULL,
115
+ started_at TEXT NOT NULL,
116
+ modified_at TEXT NOT NULL,
117
+ deadline_at TEXT,
118
+ attempt INTEGER NOT NULL DEFAULT 0,
119
+ context_json TEXT
120
+ );
121
+ CREATE INDEX IF NOT EXISTS idx_codex_cronoton_runs_due
122
+ ON codex_cronoton_runs(status, modified_at);
123
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_codex_cronoton_runs_self_block
124
+ ON codex_cronoton_runs(codex_cronoton_id)
125
+ WHERE status IN ('planning', 'awaiting', 'advancing');
126
+
127
+ CREATE TABLE IF NOT EXISTS codex_cronoton_run_legs (
128
+ id TEXT PRIMARY KEY,
129
+ run_id TEXT NOT NULL REFERENCES codex_cronoton_runs(id) ON DELETE CASCADE,
130
+ step_index INTEGER NOT NULL,
131
+ leg_index INTEGER NOT NULL,
132
+ status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN (
133
+ 'pending', 'submitted', 'confirmed', 'failed'
134
+ )),
135
+ request_key TEXT,
136
+ chain_id TEXT,
137
+ definition_json TEXT NOT NULL,
138
+ error_message TEXT,
139
+ submitted_at TEXT,
140
+ confirmed_at TEXT,
141
+ attempt INTEGER NOT NULL DEFAULT 0
142
+ );
143
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_codex_cronoton_run_legs_slot
144
+ ON codex_cronoton_run_legs(run_id, step_index, leg_index);
145
+ CREATE INDEX IF NOT EXISTS idx_codex_cronoton_run_legs_barrier
146
+ ON codex_cronoton_run_legs(run_id, step_index, status);
82
147
  `;
83
148
  /**
84
149
  * Idempotently ensures `column` exists on `table`, adding it via `ALTER TABLE
@@ -103,7 +168,7 @@ function ensureColumn(db, table, column, columnDefSql) {
103
168
  }
104
169
  }
105
170
  /**
106
- * Installs the three codex-cronoton tables and their indexes on the injected DB
171
+ * Installs the six codex-cronoton tables and their indexes on the injected DB
107
172
  * handle in a single idempotent call. Safe to re-run: every statement is guarded
108
173
  * with `IF NOT EXISTS`.
109
174
  *
@@ -122,4 +187,11 @@ export function installSchema(db) {
122
187
  ensureColumn(db, "codex_cronotons", "public_token", "TEXT");
123
188
  ensureColumn(db, "codex_cronotons", "public_observable", "INTEGER NOT NULL DEFAULT 0");
124
189
  db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_codex_cronotons_public_token ON codex_cronotons(public_token)`);
190
+ // Phase 5 fix: a real claim-before-submit marker for `codex_cronoton_run_legs`,
191
+ // closing the concurrency gap `runs-driver.ts`'s `handleSubmitLeg` doc
192
+ // previously deferred ("adding one changes the leg status enum, which is
193
+ // schema scope beyond this phase") — additive via `ensureColumn`, NOT a new
194
+ // status enum value, so the existing CHECK constraint is untouched. See
195
+ // `store/runs.ts`'s `claimLegForSubmit`/`releaseLegClaim`.
196
+ ensureColumn(db, "codex_cronoton_run_legs", "claimed_at", "TEXT");
125
197
  }
@@ -117,6 +117,28 @@ export interface ChainClient {
117
117
  * on any failure or absence, matching OuronetUI's own documented choice here.
118
118
  */
119
119
  getBlockGasLimit?(): Promise<number>;
120
+ /**
121
+ * Poll many request keys on ONE chain in a single call. Returns a map keyed
122
+ * by requestKey; a key ABSENT from the map means "still pending" (Kadena's
123
+ * /poll endpoint only returns landed transactions), a key PRESENT means a
124
+ * definite {@link ListenResult}. Declared optional for Phase 0's own
125
+ * single-fire executor, which never calls it. What matters there is that a
126
+ * multi-leg step costs one request per chain per tick, not one request per
127
+ * leg — see `blockchain/stoachain.ts` for the implementation, which always
128
+ * provides it.
129
+ *
130
+ * CAVEAT (Phase 3): `runs-driver.ts`'s `handlePoll` is this method's only
131
+ * consumer, and it does NOT fall back to polling one key at a time via
132
+ * `listen` when this is absent — a `ChainClient` that omits it will never
133
+ * observe a chained run's submitted legs land, stalling every step until
134
+ * its deadline fails the run outright (even one that actually succeeded
135
+ * on-chain). In practice this is a theoretical gap, not a live bug: the
136
+ * only shipped adapter (`blockchain/stoachain.ts`) always implements this
137
+ * method. Treat it as effectively REQUIRED for a `ChainClient` meant to
138
+ * back chained runs, not merely a performance optimization, despite the
139
+ * `?` in its signature.
140
+ */
141
+ pollRequestKeys?(keys: string[], chainId: string): Promise<Record<string, ListenResult>>;
120
142
  submit(tx: unknown): Promise<{
121
143
  requestKey: string;
122
144
  }>;
@@ -275,6 +297,8 @@ export interface Config {
275
297
  max: number;
276
298
  intervalSeconds: number;
277
299
  };
300
+ runStepDeadlineMs?: number;
301
+ maxReplanAttempts?: number;
278
302
  }
279
303
  /** Default tick cadence (30s) — {@link Config.tickIntervalMs}. */
280
304
  export declare const TICK_INTERVAL_MS = 30000;
@@ -292,3 +316,7 @@ export declare const MANUAL_BATCH_MIN = 2;
292
316
  export declare const MANUAL_BATCH_MAX = 60;
293
317
  /** Default manual-batch inter-fire interval (seconds) — {@link Config.manualBatch}.intervalSeconds. */
294
318
  export declare const MANUAL_BATCH_INTERVAL_SECONDS = 60;
319
+ /** Default per-step barrier deadline (10 minutes) — {@link Config.runStepDeadlineMs}. */
320
+ export declare const RUN_STEP_DEADLINE_MS = 600000;
321
+ /** Default max replan attempts — {@link Config.maxReplanAttempts}. */
322
+ export declare const DEFAULT_MAX_REPLAN_ATTEMPTS = 3;