@toist/aja 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -1
- package/README.md +62 -0
- package/package.json +6 -4
- package/src/cache-db.ts +1 -9
- package/src/cli.ts +17 -11
- package/src/client.ts +76 -0
- package/src/data-db.ts +1 -13
- package/src/index.ts +35 -2
- package/src/instance-metadata.ts +48 -0
- package/src/kinds/index.ts +23 -61
- package/src/lock.ts +27 -53
- package/src/migrate.ts +3 -3
- package/src/pipeline-store.ts +31 -0
- package/src/resources-fs.ts +43 -0
- package/src/resources.ts +27 -190
- package/src/run-events.ts +42 -0
- package/src/runtime-db.ts +11 -30
- package/src/server.ts +506 -496
- package/src/sqlite-runtime.ts +135 -0
- package/src/startRunner.ts +56 -70
- package/src/stores/sqlite.ts +243 -0
- package/src/stores/types.ts +18 -0
- package/src/config.ts +0 -129
- package/src/db-handles.ts +0 -70
- package/src/hitl.ts +0 -257
- package/src/instance.ts +0 -64
- package/src/kinds/control.ts +0 -26
- package/src/kinds/data.ts +0 -30
- package/src/kinds/db.ts +0 -92
- package/src/kinds/hitl.ts +0 -56
- package/src/kinds/http.ts +0 -134
- package/src/kinds/runs.ts +0 -130
- package/src/kinds/transform.ts +0 -123
- package/src/kinds/types.ts +0 -16
- package/src/pipeline.ts +0 -605
- package/src/runs.ts +0 -53
package/src/pipeline.ts
DELETED
|
@@ -1,605 +0,0 @@
|
|
|
1
|
-
// 2121
|
|
2
|
-
// Pipeline loader, in-memory store, and (legacy) sequential executor.
|
|
3
|
-
//
|
|
4
|
-
// Spec types and validation live in `@toist/spec`; this file is the
|
|
5
|
-
// runner side: file watch, registry-aware validation, the executor, and the
|
|
6
|
-
// public-spec serialization helper. Phase B replaces the sequential executor
|
|
7
|
-
// with a discriminator-aware parallel dispatcher.
|
|
8
|
-
|
|
9
|
-
import { readFileSync, readdirSync, watch } from "node:fs"
|
|
10
|
-
import { basename, extname, join, resolve as resolvePath } from "node:path"
|
|
11
|
-
import {
|
|
12
|
-
parseYaml,
|
|
13
|
-
validateSpec as specValidate,
|
|
14
|
-
resolveDisc,
|
|
15
|
-
YamlError,
|
|
16
|
-
type PipelineSpec,
|
|
17
|
-
type PipelineNode,
|
|
18
|
-
type ParsedNode,
|
|
19
|
-
type ValidateResult,
|
|
20
|
-
type SubRunOutcome,
|
|
21
|
-
type HitlSpec,
|
|
22
|
-
type ErrorReviewSpec,
|
|
23
|
-
} from "@toist/spec"
|
|
24
|
-
import { getKind, type ExecContext, type PlatformCtx } from "./kinds/index.ts"
|
|
25
|
-
import { runtimeDb } from "./db-handles.ts"
|
|
26
|
-
import { HitlSuspend, ErrorReviewSuspend, makeSuspend, createErrorReviewTask, persistNodeOutputs } from "./hitl.ts"
|
|
27
|
-
|
|
28
|
-
export type { PipelineSpec, PipelineNode } from "@toist/spec"
|
|
29
|
-
|
|
30
|
-
const SLUG_RE = /^[A-Za-z][\w-]*$/
|
|
31
|
-
|
|
32
|
-
export interface StepResult {
|
|
33
|
-
id: string
|
|
34
|
-
kind: string
|
|
35
|
-
label: string
|
|
36
|
-
status: "complete" | "failed"
|
|
37
|
-
params?: Record<string, unknown>
|
|
38
|
-
input?: Record<string, unknown>
|
|
39
|
-
output?: unknown
|
|
40
|
-
error?: string
|
|
41
|
-
duration_ms: number
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
// ─── Validation (runner-aware: knows the kind registry) ──────────────────────
|
|
45
|
-
|
|
46
|
-
/** Runner-side validation. Injects the live kind registry as the resolver
|
|
47
|
-
* so spec validation reports unknown kinds precisely AND enforces per-kind
|
|
48
|
-
* manifest shape (required params / inputs, no unknown keys). CLI tools
|
|
49
|
-
* that don't have the registry call `specValidate` directly with no resolver. */
|
|
50
|
-
export function validateSpec(input: unknown): ValidateResult {
|
|
51
|
-
return specValidate(input, {
|
|
52
|
-
hasKind: (id) => !!getKind(id),
|
|
53
|
-
getKindManifest: (id) => {
|
|
54
|
-
const k = getKind(id)
|
|
55
|
-
if (!k) return undefined
|
|
56
|
-
const { run: _r, ...manifest } = k
|
|
57
|
-
return manifest
|
|
58
|
-
},
|
|
59
|
-
})
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// ─── In-memory store ─────────────────────────────────────────────────────────
|
|
63
|
-
|
|
64
|
-
const _pipelines: Map<string, PipelineSpec> = new Map()
|
|
65
|
-
let _dir = "../pipelines"
|
|
66
|
-
|
|
67
|
-
export function getPipelines(): PipelineSpec[] {
|
|
68
|
-
return [..._pipelines.values()].sort((a, b) => a.id.localeCompare(b.id))
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
export function getPipeline(id: string): PipelineSpec | undefined {
|
|
72
|
-
return _pipelines.get(id)
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
function loadFromFile(path: string): PipelineSpec | null {
|
|
76
|
-
let raw: string
|
|
77
|
-
try {
|
|
78
|
-
raw = readFileSync(path, "utf8")
|
|
79
|
-
} catch (err) {
|
|
80
|
-
console.warn(`[runner] failed to read ${path}: ${(err as Error).message}`)
|
|
81
|
-
return null
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
let parsed: unknown
|
|
85
|
-
try {
|
|
86
|
-
parsed = parseYaml(raw)
|
|
87
|
-
} catch (err) {
|
|
88
|
-
if (err instanceof YamlError) {
|
|
89
|
-
console.warn(`[runner] ${path}: ${err.message}`)
|
|
90
|
-
} else {
|
|
91
|
-
console.warn(`[runner] ${path}: YAML parse error: ${(err as Error).message}`)
|
|
92
|
-
}
|
|
93
|
-
return null
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
// Default id from filename stem when stem matches the slug pattern (§2).
|
|
97
|
-
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
98
|
-
const stem = basename(path, extname(path))
|
|
99
|
-
if ((parsed as { id?: unknown }).id === undefined && SLUG_RE.test(stem)) {
|
|
100
|
-
;(parsed as Record<string, unknown>).id = stem
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
const result = validateSpec(parsed)
|
|
105
|
-
if (!result.ok) {
|
|
106
|
-
console.warn(`[runner] ${path}: validation failed`)
|
|
107
|
-
for (const e of result.errors) console.warn(` - ${e}`)
|
|
108
|
-
return null
|
|
109
|
-
}
|
|
110
|
-
for (const w of result.warnings ?? []) {
|
|
111
|
-
console.warn(`[runner] ${path}: warning: ${w}`)
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
result.parsed!._source = path
|
|
115
|
-
return result.parsed!
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
export function loadAll(dir = _dir): Map<string, PipelineSpec> {
|
|
119
|
-
_dir = dir
|
|
120
|
-
_pipelines.clear()
|
|
121
|
-
let entries: string[] = []
|
|
122
|
-
try {
|
|
123
|
-
entries = readdirSync(dir).filter((f) => /\.ya?ml$/.test(f))
|
|
124
|
-
} catch {
|
|
125
|
-
console.warn(`[runner] pipelines dir not found: ${dir}`)
|
|
126
|
-
return _pipelines
|
|
127
|
-
}
|
|
128
|
-
for (const f of entries) {
|
|
129
|
-
const spec = loadFromFile(join(dir, f))
|
|
130
|
-
if (spec) _pipelines.set(spec.id, spec)
|
|
131
|
-
}
|
|
132
|
-
console.log(`[runner] loaded ${_pipelines.size} pipeline(s) from ${resolvePath(dir)}`)
|
|
133
|
-
return _pipelines
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
export function watchAll(dir = _dir, onChange?: (id: string, action: "load" | "remove") => void) {
|
|
137
|
-
watch(dir, { persistent: true }, (_event, filename) => {
|
|
138
|
-
if (!filename || !/\.ya?ml$/.test(filename)) return
|
|
139
|
-
const path = join(dir, filename)
|
|
140
|
-
const spec = loadFromFile(path)
|
|
141
|
-
if (spec) {
|
|
142
|
-
_pipelines.set(spec.id, spec)
|
|
143
|
-
console.log(`[runner] reloaded pipeline "${spec.id}" (${spec.nodes.length} nodes)`)
|
|
144
|
-
onChange?.(spec.id, "load")
|
|
145
|
-
} else {
|
|
146
|
-
// file may have been deleted or invalid; try to find by source path.
|
|
147
|
-
for (const [id, p] of _pipelines) {
|
|
148
|
-
if (p._source === path) {
|
|
149
|
-
_pipelines.delete(id)
|
|
150
|
-
console.log(`[runner] removed pipeline "${id}"`)
|
|
151
|
-
onChange?.(id, "remove")
|
|
152
|
-
break
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
})
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
/** Strip non-serializable fields (parsed trees, ajv validator) from a spec
|
|
160
|
-
* before sending to HTTP clients. `edges` are returned as a flat
|
|
161
|
-
* { from, to } list since clients expect that shape. */
|
|
162
|
-
export function publicSpec(spec: PipelineSpec): Record<string, unknown> {
|
|
163
|
-
return {
|
|
164
|
-
apiVersion: spec.apiVersion,
|
|
165
|
-
id: spec.id,
|
|
166
|
-
label: spec.label,
|
|
167
|
-
description: spec.description,
|
|
168
|
-
schedule: spec.schedule,
|
|
169
|
-
payloadSchema: spec.payloadSchema,
|
|
170
|
-
nodes: spec.nodes,
|
|
171
|
-
edges: spec.edges.map(({ from, to }) => ({ from, to })),
|
|
172
|
-
warnings: spec.warnings,
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
/** Backwards-compatible deps map keyed by node id. Computed from
|
|
177
|
-
* parsedNodes.refs ∪ dependsOn. */
|
|
178
|
-
export function deriveDeps(spec: PipelineSpec): Map<string, Set<string>> {
|
|
179
|
-
const out = new Map<string, Set<string>>()
|
|
180
|
-
for (const node of spec.parsedNodes) {
|
|
181
|
-
const deps = new Set<string>([...node.refs, ...node.dependsOn])
|
|
182
|
-
out.set(node.id, deps)
|
|
183
|
-
}
|
|
184
|
-
return out
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
// ─── Execution — parallel dispatcher (pipeline-spec v1 §11) ──────────────────
|
|
188
|
-
|
|
189
|
-
/** Suspension provenance — distinguishes HITL pauses from error_review
|
|
190
|
-
* suspends. The shape carries the same fields either way (prompt + assignee
|
|
191
|
-
* appear on both spec types); kind drives UI presentation and resume payload
|
|
192
|
-
* semantics. */
|
|
193
|
-
export type SuspendCause =
|
|
194
|
-
| { kind: "human.input"; spec: HitlSpec }
|
|
195
|
-
| { kind: "error_review"; spec: ErrorReviewSpec }
|
|
196
|
-
|
|
197
|
-
export type RunOutcome =
|
|
198
|
-
| { status: "done"; output: unknown; steps: StepResult[]; results: Record<string, unknown> }
|
|
199
|
-
| { status: "suspended"; suspendedAt: { taskId: number; nodeId: string; cause: SuspendCause }
|
|
200
|
-
steps: StepResult[]; results: Record<string, unknown> }
|
|
201
|
-
|
|
202
|
-
export interface RunOptions {
|
|
203
|
-
/** Pre-computed node outputs from a previous (suspended) run. Nodes whose
|
|
204
|
-
* id appears here are skipped — their output is treated as already-known.
|
|
205
|
-
* Used for resume after HITL response. */
|
|
206
|
-
resumeOutputs?: Record<string, unknown>
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
export type RunResult = Extract<RunOutcome, { status: "done" }>
|
|
210
|
-
|
|
211
|
-
interface NodeOutcome {
|
|
212
|
-
params: Record<string, unknown>
|
|
213
|
-
input: Record<string, unknown>
|
|
214
|
-
durationMs: number
|
|
215
|
-
result:
|
|
216
|
-
| { kind: "complete"; output: unknown }
|
|
217
|
-
| { kind: "suspend"; taskId: number; nodeId: string; cause: SuspendCause }
|
|
218
|
-
| { kind: "error"; error: Error }
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
export async function runPipeline(
|
|
222
|
-
spec: PipelineSpec,
|
|
223
|
-
payload: Record<string, unknown>,
|
|
224
|
-
baseCtx: PlatformCtx,
|
|
225
|
-
options: RunOptions = {},
|
|
226
|
-
): Promise<RunOutcome> {
|
|
227
|
-
// Run-time payload validation (pipeline-spec §10).
|
|
228
|
-
if (spec.payloadValidator && !spec.payloadValidator(payload)) {
|
|
229
|
-
const errs = (spec.payloadValidator.errors ?? [])
|
|
230
|
-
.map((e) => `${e.instancePath || "/"} ${e.message}`)
|
|
231
|
-
.join("; ")
|
|
232
|
-
throw new Error(`payload schema violation: ${errs}`)
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
const nodes = spec.parsedNodes
|
|
236
|
-
const byId = new Map(nodes.map((n) => [n.id, n]))
|
|
237
|
-
|
|
238
|
-
// Build dep maps from spec.edges.
|
|
239
|
-
const incoming = new Map<string, Set<string>>()
|
|
240
|
-
const outgoing = new Map<string, Set<string>>()
|
|
241
|
-
for (const n of nodes) { incoming.set(n.id, new Set()); outgoing.set(n.id, new Set()) }
|
|
242
|
-
for (const e of spec.edges) {
|
|
243
|
-
incoming.get(e.to)!.add(e.from)
|
|
244
|
-
outgoing.get(e.from)!.add(e.to)
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
const results: Record<string, unknown> = { ...(options.resumeOutputs ?? {}) }
|
|
248
|
-
const steps: StepResult[] = []
|
|
249
|
-
const done = new Set<string>(Object.keys(options.resumeOutputs ?? {}))
|
|
250
|
-
|
|
251
|
-
// Synthesise step entries for resumed nodes so the timeline is complete.
|
|
252
|
-
for (const id of done) {
|
|
253
|
-
const node = byId.get(id)
|
|
254
|
-
if (!node) continue
|
|
255
|
-
const kind = getKind(node.kind)
|
|
256
|
-
steps.push({
|
|
257
|
-
id, kind: node.kind, label: node.label ?? kind?.label ?? id,
|
|
258
|
-
status: "complete", output: results[id], duration_ms: 0,
|
|
259
|
-
})
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
// Branch-level suspend tracking. A node is blocked if any of its transitive
|
|
263
|
-
// ancestors suspended; sibling branches whose ancestors are all done can
|
|
264
|
-
// still run.
|
|
265
|
-
const blocked = new Set<string>()
|
|
266
|
-
const propagateBlock = (id: string) => {
|
|
267
|
-
if (blocked.has(id)) return
|
|
268
|
-
blocked.add(id)
|
|
269
|
-
for (const next of outgoing.get(id) ?? []) propagateBlock(next)
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
// Initial ready set: any unblocked, undone node whose deps are all done.
|
|
273
|
-
const ready: string[] = []
|
|
274
|
-
const enqueueIfReady = (id: string) => {
|
|
275
|
-
if (done.has(id) || blocked.has(id) || ready.includes(id)) return
|
|
276
|
-
if (inFlight.has(id)) return
|
|
277
|
-
const deps = incoming.get(id)!
|
|
278
|
-
for (const d of deps) if (!done.has(d)) return
|
|
279
|
-
ready.push(id)
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
const inFlight = new Map<string, Promise<{ id: string; outcome: NodeOutcome }>>()
|
|
283
|
-
let firstSuspend: { taskId: number; nodeId: string; cause: SuspendCause } | null = null
|
|
284
|
-
let failure: Error | null = null
|
|
285
|
-
let failedNodeId: string | null = null
|
|
286
|
-
|
|
287
|
-
for (const n of nodes) enqueueIfReady(n.id)
|
|
288
|
-
|
|
289
|
-
// Main dispatch loop: drain ready, race in-flight, propagate completion.
|
|
290
|
-
while (true) {
|
|
291
|
-
while (ready.length > 0 && !failure) {
|
|
292
|
-
const id = ready.shift()!
|
|
293
|
-
startOne(id)
|
|
294
|
-
}
|
|
295
|
-
if (inFlight.size === 0) break
|
|
296
|
-
|
|
297
|
-
const winner = await Promise.race(inFlight.values())
|
|
298
|
-
inFlight.delete(winner.id)
|
|
299
|
-
|
|
300
|
-
const node = byId.get(winner.id)!
|
|
301
|
-
const kind = getKind(node.kind)!
|
|
302
|
-
const label = node.label ?? kind.label
|
|
303
|
-
const o = winner.outcome
|
|
304
|
-
|
|
305
|
-
if (o.result.kind === "complete") {
|
|
306
|
-
results[winner.id] = o.result.output
|
|
307
|
-
done.add(winner.id)
|
|
308
|
-
// Persist every completed node output, not only pre-suspend checkpoints.
|
|
309
|
-
// This keeps /runs/:id/nodes and runs.nodeOutput useful for normal done
|
|
310
|
-
// runs, while preserving the same resume-skip storage semantics.
|
|
311
|
-
persistNodeOutputs(runtimeDb(), baseCtx.runId, { [winner.id]: o.result.output })
|
|
312
|
-
steps.push({
|
|
313
|
-
id: winner.id, kind: node.kind, label, status: "complete",
|
|
314
|
-
params: o.params, input: o.input, output: o.result.output,
|
|
315
|
-
duration_ms: o.durationMs,
|
|
316
|
-
})
|
|
317
|
-
for (const next of outgoing.get(winner.id) ?? []) enqueueIfReady(next)
|
|
318
|
-
} else if (o.result.kind === "suspend") {
|
|
319
|
-
// Branch-level suspend. Mark this node and all transitive descendants
|
|
320
|
-
// blocked; sibling branches keep running. Capture the first suspend as
|
|
321
|
-
// the surfaced one (multiple HITL nodes in a single dispatch are rare;
|
|
322
|
-
// resume handles them serially).
|
|
323
|
-
if (!firstSuspend) {
|
|
324
|
-
firstSuspend = { taskId: o.result.taskId, nodeId: winner.id, cause: o.result.cause }
|
|
325
|
-
}
|
|
326
|
-
propagateBlock(winner.id)
|
|
327
|
-
steps.push({
|
|
328
|
-
id: winner.id, kind: node.kind, label, status: "complete",
|
|
329
|
-
params: o.params, input: o.input,
|
|
330
|
-
output: {
|
|
331
|
-
__suspended: true,
|
|
332
|
-
taskId: o.result.taskId,
|
|
333
|
-
taskKind: o.result.cause.kind,
|
|
334
|
-
prompt: o.result.cause.spec.prompt,
|
|
335
|
-
},
|
|
336
|
-
duration_ms: o.durationMs,
|
|
337
|
-
})
|
|
338
|
-
} else {
|
|
339
|
-
// Strict abort. v1 default per pipeline-spec §11. Capture the failure;
|
|
340
|
-
// wait for in-flight to settle (best-effort), then throw.
|
|
341
|
-
failure = o.result.error
|
|
342
|
-
failedNodeId = winner.id
|
|
343
|
-
steps.push({
|
|
344
|
-
id: winner.id, kind: node.kind, label, status: "failed",
|
|
345
|
-
params: o.params, input: o.input,
|
|
346
|
-
error: o.result.error.message,
|
|
347
|
-
duration_ms: o.durationMs,
|
|
348
|
-
})
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
if (failure) {
|
|
353
|
-
// Drain any remaining in-flight promises that started before the abort.
|
|
354
|
-
await Promise.allSettled(inFlight.values())
|
|
355
|
-
throw new Error(`Step "${failedNodeId}" (${byId.get(failedNodeId!)?.kind}) failed: ${failure.message}`)
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
if (firstSuspend) {
|
|
359
|
-
return { status: "suspended", suspendedAt: firstSuspend, steps, results }
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
// Final output: topologically last node's output. Per pipeline-spec §11.7,
|
|
363
|
-
// a terminal `sink` is the output when present; since `sink.run` returns
|
|
364
|
-
// its `input.value`, the topo-last result coincides.
|
|
365
|
-
const topo = topoSort(spec)
|
|
366
|
-
const lastNode = topo[topo.length - 1]
|
|
367
|
-
const output = lastNode ? results[lastNode.id] : null
|
|
368
|
-
return { status: "done", output, steps, results }
|
|
369
|
-
|
|
370
|
-
// ─── helpers (closures over loop state) ────────────────────────────────────
|
|
371
|
-
|
|
372
|
-
function startOne(id: string) {
|
|
373
|
-
const node = byId.get(id)!
|
|
374
|
-
const promise = runOneNode(node, payload, results, baseCtx).then((outcome) => ({ id, outcome }))
|
|
375
|
-
inFlight.set(id, promise)
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
async function runOneNode(
|
|
380
|
-
node: ParsedNode,
|
|
381
|
-
payload: Record<string, unknown>,
|
|
382
|
-
results: Record<string, unknown>,
|
|
383
|
-
baseCtx: PlatformCtx,
|
|
384
|
-
): Promise<NodeOutcome> {
|
|
385
|
-
const t0 = performance.now()
|
|
386
|
-
const ctx = { params: payload, results, resource: baseCtx.resource }
|
|
387
|
-
|
|
388
|
-
let params: Record<string, unknown> = {}
|
|
389
|
-
let input: Record<string, unknown> = {}
|
|
390
|
-
try {
|
|
391
|
-
params = (resolveDisc(node.paramsTree, ctx) ?? {}) as Record<string, unknown>
|
|
392
|
-
input = (resolveDisc(node.inputTree, ctx) ?? {}) as Record<string, unknown>
|
|
393
|
-
} catch (err) {
|
|
394
|
-
return {
|
|
395
|
-
params, input,
|
|
396
|
-
durationMs: Math.round(performance.now() - t0),
|
|
397
|
-
result: { kind: "error", error: err instanceof Error ? err : new Error(String(err)) },
|
|
398
|
-
}
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
const kind = getKind(node.kind)!
|
|
402
|
-
const stepCtx: ExecContext = {
|
|
403
|
-
...baseCtx,
|
|
404
|
-
step: { nodeId: node.id },
|
|
405
|
-
suspend: makeSuspend(runtimeDb(), baseCtx.runId, node.id),
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
// §11 / §16.1 — failure-handling policy (default abort).
|
|
409
|
-
const policy = node.onError
|
|
410
|
-
const maxAttempts = policy.kind === "retry" ? policy.times + 1 : 1
|
|
411
|
-
|
|
412
|
-
let lastError: Error | null = null
|
|
413
|
-
|
|
414
|
-
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
415
|
-
if (attempt > 0 && policy.kind === "retry") {
|
|
416
|
-
// Backoff before each retry. Linear: 1s, 2s, 3s ...
|
|
417
|
-
// Exponential: 1s, 2s, 4s, 8s ... capped at 30s.
|
|
418
|
-
const delayMs = policy.backoff === "exponential"
|
|
419
|
-
? Math.min(1000 * 2 ** (attempt - 1), 30000)
|
|
420
|
-
: 1000 * attempt
|
|
421
|
-
await new Promise((r) => setTimeout(r, delayMs))
|
|
422
|
-
baseCtx.log("info", `retrying node "${node.id}" (attempt ${attempt + 1}/${maxAttempts}) after ${delayMs}ms`)
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
try {
|
|
426
|
-
const output = await kind.run(stepCtx, params, input)
|
|
427
|
-
return {
|
|
428
|
-
params, input,
|
|
429
|
-
durationMs: Math.round(performance.now() - t0),
|
|
430
|
-
result: { kind: "complete", output },
|
|
431
|
-
}
|
|
432
|
-
} catch (err) {
|
|
433
|
-
// HITL short-circuits the retry loop — it's a pause, not a failure.
|
|
434
|
-
if (err instanceof HitlSuspend) {
|
|
435
|
-
return {
|
|
436
|
-
params, input,
|
|
437
|
-
durationMs: Math.round(performance.now() - t0),
|
|
438
|
-
result: { kind: "suspend", taskId: err.taskId, nodeId: err.nodeId, cause: { kind: "human.input", spec: err.spec } },
|
|
439
|
-
}
|
|
440
|
-
}
|
|
441
|
-
lastError = err instanceof Error ? err : new Error(String(err))
|
|
442
|
-
// Continue to next retry attempt if any. Otherwise fall through below.
|
|
443
|
-
}
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
// All attempts exhausted (or maxAttempts === 1, i.e. no retry policy).
|
|
447
|
-
// Apply the post-retry policy. retry-exhausted falls through to abort.
|
|
448
|
-
const errFinal = lastError ?? new Error(`node "${node.id}" failed without an error object`)
|
|
449
|
-
|
|
450
|
-
if (policy.kind === "skip-with") {
|
|
451
|
-
try {
|
|
452
|
-
const subst = resolveDisc(policy.valueTree, ctx) ?? null
|
|
453
|
-
baseCtx.log("warn", `node "${node.id}" failed (${errFinal.message}); skip-with substitution applied`)
|
|
454
|
-
return {
|
|
455
|
-
params, input,
|
|
456
|
-
durationMs: Math.round(performance.now() - t0),
|
|
457
|
-
result: { kind: "complete", output: subst },
|
|
458
|
-
}
|
|
459
|
-
} catch (substErr) {
|
|
460
|
-
// skip-with substitution itself blew up — escalate as an error.
|
|
461
|
-
return {
|
|
462
|
-
params, input,
|
|
463
|
-
durationMs: Math.round(performance.now() - t0),
|
|
464
|
-
result: { kind: "error", error: substErr instanceof Error ? substErr : new Error(String(substErr)) },
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
if (policy.kind === "suspend") {
|
|
470
|
-
const errSpec: ErrorReviewSpec = {
|
|
471
|
-
prompt: errFinal.message,
|
|
472
|
-
error: errFinal.message,
|
|
473
|
-
details: { kind: node.kind, errorName: errFinal.name },
|
|
474
|
-
}
|
|
475
|
-
try {
|
|
476
|
-
const { id: taskId } = createErrorReviewTask(runtimeDb(), baseCtx.runId, node.id, errSpec)
|
|
477
|
-
baseCtx.log("warn", `node "${node.id}" failed (${errFinal.message}); suspended for error_review task ${taskId}`)
|
|
478
|
-
return {
|
|
479
|
-
params, input,
|
|
480
|
-
durationMs: Math.round(performance.now() - t0),
|
|
481
|
-
result: { kind: "suspend", taskId, nodeId: node.id, cause: { kind: "error_review", spec: errSpec } },
|
|
482
|
-
}
|
|
483
|
-
} catch (taskErr) {
|
|
484
|
-
// Failed to even create the suspend task — escalate the original error.
|
|
485
|
-
baseCtx.log("error", `node "${node.id}" failed AND error_review task creation failed: ${(taskErr as Error).message}`)
|
|
486
|
-
return {
|
|
487
|
-
params, input,
|
|
488
|
-
durationMs: Math.round(performance.now() - t0),
|
|
489
|
-
result: { kind: "error", error: errFinal },
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
// abort (default) and retry-exhausted both end here.
|
|
495
|
-
return {
|
|
496
|
-
params, input,
|
|
497
|
-
durationMs: Math.round(performance.now() - t0),
|
|
498
|
-
result: { kind: "error", error: errFinal },
|
|
499
|
-
}
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
/** Deterministic topological order. Used to pick the final output node. */
|
|
503
|
-
function topoSort(spec: PipelineSpec): PipelineNode[] {
|
|
504
|
-
const indeg = new Map<string, number>()
|
|
505
|
-
const reverse = new Map<string, string[]>()
|
|
506
|
-
|
|
507
|
-
for (const n of spec.nodes) indeg.set(n.id, 0)
|
|
508
|
-
for (const e of spec.edges) {
|
|
509
|
-
indeg.set(e.to, (indeg.get(e.to) ?? 0) + 1)
|
|
510
|
-
if (!reverse.has(e.from)) reverse.set(e.from, [])
|
|
511
|
-
reverse.get(e.from)!.push(e.to)
|
|
512
|
-
}
|
|
513
|
-
|
|
514
|
-
const byId = new Map(spec.nodes.map((n) => [n.id, n]))
|
|
515
|
-
const queue: string[] = []
|
|
516
|
-
for (const [id, d] of indeg) if (d === 0) queue.push(id)
|
|
517
|
-
|
|
518
|
-
const order: PipelineNode[] = []
|
|
519
|
-
while (queue.length) {
|
|
520
|
-
const id = queue.shift()!
|
|
521
|
-
order.push(byId.get(id)!)
|
|
522
|
-
for (const next of reverse.get(id) ?? []) {
|
|
523
|
-
const v = (indeg.get(next) ?? 0) - 1
|
|
524
|
-
indeg.set(next, v)
|
|
525
|
-
if (v === 0) queue.push(next)
|
|
526
|
-
}
|
|
527
|
-
}
|
|
528
|
-
return order
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
// ─── Sub-run trigger (pipeline-spec.md §12, M2) ──────────────────────────────
|
|
532
|
-
|
|
533
|
-
/** Builds a run-scoped logger that writes to the logs table. Public so the
|
|
534
|
-
* HTTP layer can use the same shape as sub-run launches. */
|
|
535
|
-
export function makeLogger(runId: number): (level: "info" | "warn" | "error", msg: string) => void {
|
|
536
|
-
const stmt = runtimeDb().prepare("INSERT INTO logs (run_id, level, msg) VALUES (?, ?, ?)")
|
|
537
|
-
return (level, msg) => { stmt.run(runId, level, msg) }
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
/** Invoke another pipeline as a sub-run from inside a kind's run() (the
|
|
541
|
-
* pipeline.run kind's path). Each sub-run gets its own row in `runs`,
|
|
542
|
-
* its own logger, and persists into `node_outputs` exactly like a
|
|
543
|
-
* top-level run. The returned SubRunOutcome surfaces the result shape
|
|
544
|
-
* the kind needs to honour its `onFailure` policy.
|
|
545
|
-
*
|
|
546
|
-
* baseCtx is the parent run's context. The sub-run inherits db / cache /
|
|
547
|
-
* resource (shared platform-instance singletons) but gets a fresh runId
|
|
548
|
-
* and logger. The sub-run's own subRun closure is rebound so its own
|
|
549
|
-
* recursive calls thread correctly. */
|
|
550
|
-
export async function triggerSubRun(
|
|
551
|
-
pipelineId: string,
|
|
552
|
-
payload: Record<string, unknown>,
|
|
553
|
-
baseCtx: PlatformCtx,
|
|
554
|
-
): Promise<SubRunOutcome> {
|
|
555
|
-
const spec = getPipeline(pipelineId)
|
|
556
|
-
if (!spec) {
|
|
557
|
-
return { status: "failed", runId: -1, error: `pipeline "${pipelineId}" not found`, partialResults: {} }
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
const inserted = runtimeDb().prepare(
|
|
561
|
-
"INSERT INTO runs (pipeline, status, payload, trigger, updated_at) VALUES (?, 'running', ?, 'sub-run', datetime('now')) RETURNING id",
|
|
562
|
-
).get(pipelineId, JSON.stringify(payload)) as { id: number }
|
|
563
|
-
const subRunId = inserted.id
|
|
564
|
-
|
|
565
|
-
// Build a sub-run baseCtx with its own runId, logger, and a self-bound
|
|
566
|
-
// subRun closure so recursive sub-runs thread correctly.
|
|
567
|
-
const subBaseCtx: PlatformCtx = {
|
|
568
|
-
...baseCtx,
|
|
569
|
-
runId: subRunId,
|
|
570
|
-
log: makeLogger(subRunId),
|
|
571
|
-
subRun: undefined as unknown as PlatformCtx["subRun"],
|
|
572
|
-
}
|
|
573
|
-
subBaseCtx.subRun = (pid, p) => triggerSubRun(pid, p, subBaseCtx)
|
|
574
|
-
|
|
575
|
-
try {
|
|
576
|
-
const outcome = await runPipeline(spec, payload, subBaseCtx)
|
|
577
|
-
|
|
578
|
-
if (outcome.status === "done") {
|
|
579
|
-
runtimeDb().prepare(
|
|
580
|
-
`UPDATE runs SET status='done', result=?, steps=?, finished_at=datetime('now'),
|
|
581
|
-
updated_at=datetime('now'), current_node=NULL WHERE id=?`,
|
|
582
|
-
).run(JSON.stringify(outcome.output), JSON.stringify(outcome.steps), subRunId)
|
|
583
|
-
return { status: "done", runId: subRunId, output: outcome.output }
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
// suspended (HITL or error_review). Sub-run is parked; the parent's
|
|
587
|
-
// pipeline.run kind will surface the suspension in its output.
|
|
588
|
-
persistNodeOutputs(runtimeDb(), subRunId, outcome.results)
|
|
589
|
-
runtimeDb().prepare(
|
|
590
|
-
`UPDATE runs SET status='suspended', steps=?, current_node=?, updated_at=datetime('now') WHERE id=?`,
|
|
591
|
-
).run(JSON.stringify(outcome.steps), outcome.suspendedAt.nodeId, subRunId)
|
|
592
|
-
return {
|
|
593
|
-
status: "suspended",
|
|
594
|
-
runId: subRunId,
|
|
595
|
-
suspendedAt: outcome.suspendedAt.nodeId,
|
|
596
|
-
partialResults: outcome.results,
|
|
597
|
-
}
|
|
598
|
-
} catch (err) {
|
|
599
|
-
const message = err instanceof Error ? err.message : String(err)
|
|
600
|
-
runtimeDb().prepare(
|
|
601
|
-
"UPDATE runs SET status='error', error=?, finished_at=datetime('now'), updated_at=datetime('now') WHERE id=?",
|
|
602
|
-
).run(message, subRunId)
|
|
603
|
-
return { status: "failed", runId: subRunId, error: message, partialResults: {} }
|
|
604
|
-
}
|
|
605
|
-
}
|
package/src/runs.ts
DELETED
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
// 2121
|
|
2
|
-
// Read-only capability over the runtime ledger, exposed to kinds as
|
|
3
|
-
// ctx.runs. Powers the runs.lastOutput and runs.nodeOutput kinds (M3, M4
|
|
4
|
-
// in pipeline-spec.md §12). Kept here rather than in runtime-db.ts because
|
|
5
|
-
// the latter is intentionally not exposed to kinds — kinds see ctx.db
|
|
6
|
-
// (data store) and ctx.cache (capability), not the runtime tables.
|
|
7
|
-
|
|
8
|
-
import type { Database } from "bun:sqlite"
|
|
9
|
-
import type { RunStore } from "@toist/spec"
|
|
10
|
-
|
|
11
|
-
interface RunRow {
|
|
12
|
-
id: number
|
|
13
|
-
result: string | null
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
interface NodeOutputRow {
|
|
17
|
-
output_json: string | null
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
export function makeRunStore(runtimeDb: Database): RunStore {
|
|
21
|
-
return {
|
|
22
|
-
lastOutput(pipeline, opts) {
|
|
23
|
-
const status = opts?.status ?? "done"
|
|
24
|
-
const row = runtimeDb.prepare(
|
|
25
|
-
"SELECT id, result FROM runs WHERE pipeline = ? AND status = ? ORDER BY id DESC LIMIT 1",
|
|
26
|
-
).get(pipeline, status) as RunRow | undefined
|
|
27
|
-
if (!row || row.result === null) return null
|
|
28
|
-
try {
|
|
29
|
-
return JSON.parse(row.result) as unknown
|
|
30
|
-
} catch {
|
|
31
|
-
return null
|
|
32
|
-
}
|
|
33
|
-
},
|
|
34
|
-
|
|
35
|
-
nodeOutput(pipeline, node, opts) {
|
|
36
|
-
const status = opts?.status ?? "done"
|
|
37
|
-
const row = runtimeDb.prepare(
|
|
38
|
-
`SELECT no.output_json
|
|
39
|
-
FROM node_outputs no
|
|
40
|
-
JOIN runs r ON no.run_id = r.id
|
|
41
|
-
WHERE r.pipeline = ? AND r.status = ? AND no.node_id = ?
|
|
42
|
-
ORDER BY r.id DESC
|
|
43
|
-
LIMIT 1`,
|
|
44
|
-
).get(pipeline, status, node) as NodeOutputRow | undefined
|
|
45
|
-
if (!row || row.output_json === null) return null
|
|
46
|
-
try {
|
|
47
|
-
return JSON.parse(row.output_json) as unknown
|
|
48
|
-
} catch {
|
|
49
|
-
return null
|
|
50
|
-
}
|
|
51
|
-
},
|
|
52
|
-
}
|
|
53
|
-
}
|