@toist/aja 0.6.1 → 0.8.0

package/src/server.ts

@@ -1,551 +1,561 @@
 // 2121
+import type { Database } from "bun:sqlite"
 import { Hono } from "hono"
 import { cors } from "hono/cors"
 import { existsSync, readFileSync, writeFileSync } from "node:fs"
 import { join } from "node:path"
-import { runtimeDb, dataDb, cacheDb, initDbs } from "./db-handles.ts"
-import { makeCache } from "./cache.ts"
-import { loadAll, watchAll, getPipelines, getPipeline, runPipeline, validateSpec, publicSpec, triggerSubRun, makeLogger, type RunOutcome } from "./pipeline.ts"
-import { parseYaml, type PlatformCtx } from "@toist/spec"
-import { manifest, getKind, type ExecContext } from "./kinds/index.ts"
-import { answerTask, getTask, loadNodeOutputs, persistNodeOutputs } from "./hitl.ts"
-import { loadInstance } from "./instance.ts"
 import {
-  buildResourceCtx, listResourceTypes, listResources, getResource,
-  upsertResource, patchResource, deleteResource,
+  getPipeline,
+  getPipelines,
+  getKind,
+  loadAll,
+  makeLogger,
+  manifest,
+  publicSpec,
+  runPipeline,
+  validateSpec,
+  type RunCtx,
+  type RunOutcome,
+  type ToistEvent,
+  type ToistRuntime,
+} from "@toist/core"
+import { parseYaml, type ExecContext, type PipelineSpec } from "@toist/spec"
+import { loadInstance } from "./instance-metadata.ts"
+import {
+  deleteResource,
+  getResource,
+  listResources,
+  listResourceTypes,
+  patchResource,
+  upsertResource,
 } from "./resources.ts"
-import { makeRunStore } from "./runs.ts"
-import { pipelinesDir, isUiDisabled, isWatchDisabled, getCorsOrigins } from "./config.ts"
+import { createRunEventBroker, type RunEventBroker } from "./run-events.ts"
 
 const VALID_PIPELINE_ID = /^[a-z0-9][a-z0-9-]*$/
 
-// Legacy entry path: when bun runs this module directly (pm2 dev mode), do
-// the lifecycle init here. When startRunner imports this module, init has
-// already happened — initDbs() is idempotent so the guard below is belt-and-
-// suspenders. The conditional avoids unnecessary work when import.meta.main
-// is false (the startRunner-imported case).
-if (import.meta.main) {
-  await initDbs()
+export interface MountApiOptions {
+  rootDir: string
+  pipelinesDir?: string
+  adminDb?: Database
+  disableUi?: boolean
+  corsOrigins?: string | string[]
 }
 
-// apiApp holds every HTTP route. The outer `app` (declared at the bottom)
-// mounts apiApp at /api and owns CORS + future static UI serving. Per
-// instance-spec.md §9 + §12 Roadmap: routes live under /api/* so the UI
-// can mount cleanly at / when packaging lands.
-const apiApp = new Hono()
-
-const cache = makeCache(cacheDb())
-
-const PIPELINES_DIR = pipelinesDir()
-loadAll(PIPELINES_DIR)
-if (!isWatchDisabled()) {
-  watchAll(PIPELINES_DIR)
+function resolveRuntimeKind(runtime: ToistRuntime, id: string) {
+  if (!runtime.registry) return getKind(id)
+  const kind = runtime.registry.get(id)
+  if (kind) return kind
+  return runtime.registry.exclusive ? undefined : getKind(id)
 }
 
-// ─── instance metadata ───────────────────────────────────────────────────────
-apiApp.get("/", (c) => {
-  const inst = loadInstance()
-  return c.json({
-    name: inst.instanceName ?? process.env.PLATFORM_INSTANCE_NAME ?? "platform",
-    version: "0.1.0",
-    platformVersion: inst.platformVersion,
-    tier: inst.tier,
-    pipelines: getPipelines().length,
-    kinds: manifest().length,
-  })
-})
+function runtimeManifest(runtime: ToistRuntime) {
+  return runtime.registry ? runtime.registry.manifest() : manifest()
+}
 
-// Per-instance metadata: tier, teasers, branding. Distinct from runtime
-// state in /runs and pipeline state in /pipelines.
-apiApp.get("/instance", (c) => c.json(loadInstance()))
+export async function mountApi(runtime: ToistRuntime, options: MountApiOptions): Promise<Hono> {
+  const apiApp = new Hono()
+  const pipelinesDir = options.pipelinesDir ?? join(options.rootDir, "pipelines")
+  const runEvents = new Map<number, RunEventBroker>()
+  const adHocRuns = new Map<number, { spec: PipelineSpec; payload: Record<string, unknown> }>()
 
-// ─── kinds (capability) ──────────────────────────────────────────────────────
-apiApp.get("/manifest", (c) => c.json({ kinds: manifest() }))
+  apiApp.get("/", (c) => {
+    const inst = loadInstance(options.rootDir)
+    return c.json({
+      name: inst.instanceName ?? process.env.PLATFORM_INSTANCE_NAME ?? "platform",
+      version: "0.1.0",
+      platformVersion: inst.platformVersion,
+      tier: inst.tier,
+      pipelines: getPipelines().length,
+      kinds: runtimeManifest(runtime).length,
+    })
+  })
 
-apiApp.post("/kinds/:id/invoke", async (c) => {
-  const kind = getKind(c.req.param("id"))
-  if (!kind) return c.json({ error: "kind not found" }, 404)
+  apiApp.get("/instance", (c) => c.json(loadInstance(options.rootDir)))
 
-  const body = await c.req.json().catch(() => ({}))
-  const { params = {}, input = {}, confirm = false } = body as {
-    params?: Record<string, unknown>
-    input?:  Record<string, unknown>
-    confirm?: boolean
-  }
+  apiApp.get("/manifest", (c) => c.json({
+    version: 1,
+    schema: "https://toist.in/schemas/manifest-v1.json",
+    kinds: runtimeManifest(runtime),
+  }))
 
-  if (kind.sideEffect && !confirm) {
-    return c.json({
-      error: "side_effect_requires_confirm",
-      message: `Kind "${kind.id}" mutates external state. Pass { confirm: true } to invoke ad-hoc.`,
-    }, 400)
-  }
+  apiApp.post("/kinds/:id/invoke", async (c) => {
+    const kind = resolveRuntimeKind(runtime, c.req.param("id"))
+    if (!kind) return c.json({ error: "kind not found" }, 404)
 
-  const invokeCtx: ExecContext = {
-    runId: -1,                        // synthetic — invoke is not a tracked run
-    db: dataDb(),
-    cache,
-    log: (level, msg) => console.log(`[invoke ${kind.id}] ${level}: ${msg}`),
-    resource: buildResourceCtx(runtimeDb()),
-    runs: makeRunStore(runtimeDb()),
-    subRun: async () => {
-      throw new Error("ctx.subRun is not available in ad-hoc kinds.invoke — call from inside a pipeline run instead.")
-    },
-    step: { nodeId: "<invoke>" },
-    suspend: async () => {
-      throw new Error("ctx.suspend is not available in ad-hoc kinds.invoke — call from inside a pipeline run instead.")
-    },
-  }
+    const body = await c.req.json().catch(() => ({}))
+    const { params = {}, input = {}, confirm = false } = body as {
+      params?: Record<string, unknown>
+      input?: Record<string, unknown>
+      confirm?: boolean
+    }
 
-  const t0 = performance.now()
-  try {
-    const output = await kind.run(invokeCtx, params, input)
-    return c.json({
-      kind: kind.id,
-      output,
-      duration_ms: Math.round(performance.now() - t0),
-    })
-  } catch (err: unknown) {
-    return c.json({
-      kind: kind.id,
-      error: err instanceof Error ? err.message : String(err),
-      duration_ms: Math.round(performance.now() - t0),
-    }, 500)
-  }
-})
+    if (kind.sideEffect && !confirm) {
+      return c.json({
+        error: "side_effect_requires_confirm",
+        message: `Kind "${kind.id}" mutates external state. Pass { confirm: true } to invoke ad-hoc.`,
+      }, 400)
+    }
 
-// ─── resources ───────────────────────────────────────────────────────────────
+    const invokeCtx: ExecContext = {
+      runId: -1,
+      db: runtime.data,
+      cache: runtime.cache,
+      log: (level, msg) => console.log(`[invoke ${kind.id}] ${level}: ${msg}`),
+      resource: await runtime.resources.resolve(),
+      runs: runtime.runs,
+      subRun: async () => {
+        throw new Error("ctx.subRun is not available in ad-hoc kinds.invoke — call from inside a pipeline run instead.")
+      },
+      step: { nodeId: "<invoke>" },
+      suspend: async () => {
+        throw new Error("ctx.suspend is not available in ad-hoc kinds.invoke — call from inside a pipeline run instead.")
+      },
+    }
 
-apiApp.get("/resource-types", (c) => c.json(listResourceTypes()))
+    const t0 = performance.now()
+    try {
+      const output = await kind.run(invokeCtx, params, input)
+      return c.json({
+        kind: kind.id,
+        output,
+        duration_ms: Math.round(performance.now() - t0),
+      })
+    } catch (err: unknown) {
+      return c.json({
+        kind: kind.id,
+        error: err instanceof Error ? err.message : String(err),
+        duration_ms: Math.round(performance.now() - t0),
+      }, 500)
+    }
+  })
 
-apiApp.get("/resources", (c) => c.json(listResources(runtimeDb())))
+  apiApp.get("/resource-types", (c) => c.json(listResourceTypes()))
 
-apiApp.get("/resources/:name", (c) => {
-  const r = getResource(runtimeDb(), c.req.param("name"))
-  if (!r) return c.json({ error: "not found" }, 404)
-  return c.json(r)
-})
+  apiApp.get("/resources", (c) => {
+    if (!options.adminDb) return c.json({ error: "resource admin unavailable for this runtime" }, 501)
+    return c.json(listResources(options.adminDb))
+  })
 
-apiApp.post("/resources", async (c) => {
-  const body = await c.req.json().catch(() => ({})) as {
-    name?: string; type?: string; fields?: Record<string, unknown>
-  }
-  if (!body.name || typeof body.name !== "string") return c.json({ error: "name required" }, 400)
-  if (!body.type || typeof body.type !== "string") return c.json({ error: "type required" }, 400)
-  const fields = body.fields ?? {}
-  const r = upsertResource(runtimeDb(), body.name, body.type, fields)
-  return c.json(r, 201)
-})
-
-apiApp.put("/resources/:name", async (c) => {
-  const name = c.req.param("name")
-  const body = await c.req.json().catch(() => ({})) as {
-    type?: string; fields?: Record<string, unknown>
-  }
-  if (body.type !== undefined && body.fields !== undefined) {
-    // Full replacement
-    const r = upsertResource(runtimeDb(), name, body.type, body.fields)
-    return c.json(r)
-  }
-  if (body.fields !== undefined) {
-    // Patch fields only
-    const r = patchResource(runtimeDb(), name, body.fields)
+  apiApp.get("/resources/:name", (c) => {
+    if (!options.adminDb) return c.json({ error: "resource admin unavailable for this runtime" }, 501)
+    const r = getResource(options.adminDb, c.req.param("name"))
     if (!r) return c.json({ error: "not found" }, 404)
     return c.json(r)
-  }
-  return c.json({ error: "provide type+fields for replacement or fields for patch" }, 400)
-})
-
-apiApp.delete("/resources/:name", (c) => {
-  const ok = deleteResource(runtimeDb(), c.req.param("name"))
-  if (!ok) return c.json({ error: "not found" }, 404)
-  return new Response(null, { status: 204 })
-})
-
-// ─── pipelines ───────────────────────────────────────────────────────────────
-apiApp.get("/pipelines", (c) => {
-  const list = getPipelines().map((p) => ({
-    id: p.id,
-    label: p.label ?? p.id,
-    description: p.description,
-    nodeCount: p.nodes.length,
-  }))
-  return c.json(list)
-})
-
-apiApp.get("/pipelines/:id", (c) => {
-  const spec = getPipeline(c.req.param("id"))
-  if (!spec) return c.json({ error: "not found" }, 404)
-  return c.json(publicSpec(spec))
-})
-
-// Raw YAML source for editing. Returns the file as written to disk (preserves
-// comments and formatting), not the parsed spec.
-apiApp.get("/pipelines/:id/source", (c) => {
-  const id = c.req.param("id")
-  if (!VALID_PIPELINE_ID.test(id)) return c.json({ error: "invalid id" }, 400)
-  const path = join(PIPELINES_DIR, `${id}.yaml`)
-  if (!existsSync(path)) return c.json({ error: "not found" }, 404)
-  return c.json({ id, yaml: readFileSync(path, "utf8") })
-})
-
-// Create a new pipeline file. The id comes from the YAML's `id:` field — no
-// separate body.id, since two sources of truth caused silent overwrites.
-// Refuses to overwrite an existing id.
-apiApp.post("/pipelines", async (c) => {
-  const body = await c.req.json().catch(() => ({})) as { yaml?: string }
-  if (!body.yaml) return c.json({ error: "yaml required" }, 400)
-
-  let parsed: unknown
-  try { parsed = parseYaml(body.yaml) }
-  catch (err) { return c.json({ ok: false, errors: [`YAML parse: ${(err as Error).message}`] }, 400) }
-
-  const id = (parsed as { id?: string })?.id
-  if (!id) return c.json({ error: "yaml must contain a top-level `id:` field" }, 400)
-  if (!VALID_PIPELINE_ID.test(id)) return c.json({ error: `invalid id "${id}" — lowercase alphanum + hyphens, must start with alphanum` }, 400)
-
-  const path = join(PIPELINES_DIR, `${id}.yaml`)
-  if (existsSync(path)) return c.json({ error: `pipeline "${id}" already exists` }, 409)
-
-  const result = validateSpec(parsed)
-  if (!result.ok) return c.json({ ok: false, errors: result.errors }, 400)
-
-  writeFileSync(path, body.yaml)
-  // Sync reload — don't rely on the fs watcher, since the client may navigate
-  // to the new pipeline immediately and would otherwise see a 404.
-  loadAll(PIPELINES_DIR)
-  return c.json({ id, ok: true }, 201)
-})
-
-// Update an existing pipeline file. Refuses to change the id (renames are a
-// separate operation we don't yet support).
-apiApp.put("/pipelines/:id", async (c) => {
-  const id = c.req.param("id")
-  if (!VALID_PIPELINE_ID.test(id)) return c.json({ error: "invalid id" }, 400)
-
-  const path = join(PIPELINES_DIR, `${id}.yaml`)
-  if (!existsSync(path)) return c.json({ error: "not found" }, 404)
-
-  const body = await c.req.json().catch(() => ({})) as { yaml?: string }
-  if (!body.yaml) return c.json({ error: "yaml required" }, 400)
-
-  let parsed: unknown
-  try { parsed = parseYaml(body.yaml) }
-  catch (err) { return c.json({ ok: false, errors: [`YAML parse: ${(err as Error).message}`] }, 400) }
-
-  const yamlId = (parsed as { id?: string })?.id
-  if (yamlId && yamlId !== id) {
-    return c.json({ error: `id mismatch: url says "${id}", yaml says "${yamlId}". Renames are not supported via edit.` }, 400)
-  }
+  })
 
-  const result = validateSpec(parsed)
-  if (!result.ok) return c.json({ ok: false, errors: result.errors }, 400)
-
-  writeFileSync(path, body.yaml)
-  loadAll(PIPELINES_DIR)
-  return c.json({ id, ok: true })
-})
-
-apiApp.post("/pipelines/validate", async (c) => {
-  const body = await c.req.json().catch(() => ({})) as { yaml?: string; spec?: unknown }
-  let spec: unknown
-  if (typeof body.yaml === "string") {
-    try { spec = parseYaml(body.yaml) }
-    catch (err) {
-      return c.json({ ok: false, errors: [`YAML parse error: ${(err as Error).message}`] }, 400)
+  apiApp.post("/resources", async (c) => {
+    if (!options.adminDb) return c.json({ error: "resource admin unavailable for this runtime" }, 501)
+    const body = await c.req.json().catch(() => ({})) as {
+      name?: string
+      type?: string
+      fields?: Record<string, unknown>
     }
-  } else if (body.spec) {
-    spec = body.spec
-  } else {
-    return c.json({ ok: false, errors: ["Provide either `yaml` (string) or `spec` (object)"] }, 400)
-  }
+    if (!body.name || typeof body.name !== "string") return c.json({ error: "name required" }, 400)
+    if (!body.type || typeof body.type !== "string") return c.json({ error: "type required" }, 400)
+    const fields = body.fields ?? {}
+    const r = upsertResource(options.adminDb, body.name, body.type, fields)
+    return c.json(r, 201)
+  })
 
-  const result = validateSpec(spec)
-  return c.json({
-    ok: result.ok,
-    errors: result.errors,
-    warnings: result.warnings ?? [],
-    edges: result.edges ?? [],
-  }, result.ok ? 200 : 400)
-})
-
-/** Build the per-run baseCtx for runPipeline. Self-references via subRun
- *  so sub-runs threaded from this top-level run carry the right context;
- *  triggerSubRun rebinds the same closure to the sub-run's own runId. */
-function buildBaseCtx(runId: number): PlatformCtx {
-  const baseCtx: PlatformCtx = {
-    runId,
-    db: dataDb(),
-    cache,
-    log: makeLogger(runId),
-    resource: buildResourceCtx(runtimeDb()),
-    runs: makeRunStore(runtimeDb()),
-    subRun: undefined as unknown as PlatformCtx["subRun"],
-  }
-  baseCtx.subRun = (pid, payload) => triggerSubRun(pid, payload, baseCtx)
-  return baseCtx
-}
+  apiApp.put("/resources/:name", async (c) => {
+    if (!options.adminDb) return c.json({ error: "resource admin unavailable for this runtime" }, 501)
+    const name = c.req.param("name")
+    const body = await c.req.json().catch(() => ({})) as {
+      type?: string
+      fields?: Record<string, unknown>
+    }
+    if (body.type !== undefined && body.fields !== undefined) {
+      const r = upsertResource(options.adminDb, name, body.type, body.fields)
+      return c.json(r)
+    }
+    if (body.fields !== undefined) {
+      const r = patchResource(options.adminDb, name, body.fields)
+      if (!r) return c.json({ error: "not found" }, 404)
+      return c.json(r)
+    }
+    return c.json({ error: "provide type+fields for replacement or fields for patch" }, 400)
+  })
 
-function persistOutcome(runId: number, outcome: RunOutcome): void {
-  if (outcome.status === "done") {
-    runtimeDb().prepare(
-      `UPDATE runs SET status='done', result=?, steps=?, finished_at=datetime('now'),
-                       updated_at=datetime('now'), current_node=NULL WHERE id=?`,
-    ).run(JSON.stringify(outcome.output), JSON.stringify(outcome.steps), runId)
-  } else {
-    persistNodeOutputs(runtimeDb(), runId, outcome.results)
-    runtimeDb().prepare(
-      `UPDATE runs SET status='suspended', steps=?, current_node=?, updated_at=datetime('now') WHERE id=?`,
-    ).run(JSON.stringify(outcome.steps), outcome.suspendedAt.nodeId, runId)
-  }
-}
+  apiApp.delete("/resources/:name", (c) => {
+    if (!options.adminDb) return c.json({ error: "resource admin unavailable for this runtime" }, 501)
+    const ok = deleteResource(options.adminDb, c.req.param("name"))
+    if (!ok) return c.json({ error: "not found" }, 404)
+    return new Response(null, { status: 204 })
+  })
+
+  apiApp.get("/pipelines", (c) => {
+    const list = getPipelines().map((p) => ({
+      id: p.id,
+      label: p.label ?? p.id,
+      description: p.description,
+      nodeCount: p.nodes.length,
+    }))
+    return c.json(list)
+  })
+
+  apiApp.get("/pipelines/:id", (c) => {
+    const spec = getPipeline(c.req.param("id"))
+    if (!spec) return c.json({ error: "not found" }, 404)
+    return c.json(publicSpec(spec))
+  })
+
+  apiApp.get("/pipelines/:id/source", (c) => {
+    const id = c.req.param("id")
+    if (!VALID_PIPELINE_ID.test(id)) return c.json({ error: "invalid id" }, 400)
+    const path = join(pipelinesDir, `${id}.yaml`)
+    if (!existsSync(path)) return c.json({ error: "not found" }, 404)
+    return c.json({ id, yaml: readFileSync(path, "utf8") })
+  })
+
+  apiApp.post("/pipelines", async (c) => {
+    const body = await c.req.json().catch(() => ({})) as { yaml?: string }
+    if (!body.yaml) return c.json({ error: "yaml required" }, 400)
+
+    let parsed: unknown
+    try { parsed = parseYaml(body.yaml) }
+    catch (err) { return c.json({ ok: false, errors: [`YAML parse: ${(err as Error).message}`] }, 400) }
+
+    const id = (parsed as { id?: string })?.id
+    if (!id) return c.json({ error: "yaml must contain a top-level `id:` field" }, 400)
+    if (!VALID_PIPELINE_ID.test(id)) return c.json({ error: `invalid id "${id}" — lowercase alphanum + hyphens, must start with alphanum` }, 400)
+
+    const path = join(pipelinesDir, `${id}.yaml`)
+    if (existsSync(path)) return c.json({ error: `pipeline "${id}" already exists` }, 409)
+
+    const result = validateSpec(parsed, { registry: runtime.registry })
+    if (!result.ok) return c.json({ ok: false, errors: result.errors }, 400)
+
+    writeFileSync(path, body.yaml)
+    loadAll(pipelinesDir, { registry: runtime.registry })
+    return c.json({ id, ok: true }, 201)
+  })
+
+  apiApp.put("/pipelines/:id", async (c) => {
+    const id = c.req.param("id")
+    if (!VALID_PIPELINE_ID.test(id)) return c.json({ error: "invalid id" }, 400)
+
+    const path = join(pipelinesDir, `${id}.yaml`)
+    if (!existsSync(path)) return c.json({ error: "not found" }, 404)
+
+    const body = await c.req.json().catch(() => ({})) as { yaml?: string }
+    if (!body.yaml) return c.json({ error: "yaml required" }, 400)
+
+    let parsed: unknown
+    try { parsed = parseYaml(body.yaml) }
+    catch (err) { return c.json({ ok: false, errors: [`YAML parse: ${(err as Error).message}`] }, 400) }
+
+    const yamlId = (parsed as { id?: string })?.id
+    if (yamlId && yamlId !== id) {
+      return c.json({ error: `id mismatch: url says "${id}", yaml says "${yamlId}". Renames are not supported via edit.` }, 400)
+    }
+
+    const result = validateSpec(parsed, { registry: runtime.registry })
+    if (!result.ok) return c.json({ ok: false, errors: result.errors }, 400)
 
-function outcomeResponse(runId: number, pipelineId: string, outcome: RunOutcome) {
-  if (outcome.status === "done") {
-    return { id: runId, pipeline: pipelineId, status: "done", result: outcome.output, steps: outcome.steps }
+    writeFileSync(path, body.yaml)
+    loadAll(pipelinesDir, { registry: runtime.registry })
+    return c.json({ id, ok: true })
+  })
+
+  apiApp.post("/pipelines/validate", async (c) => {
+    const body = await c.req.json().catch(() => ({})) as { yaml?: string; spec?: unknown }
+    let spec: unknown
+    if (typeof body.yaml === "string") {
+      try { spec = parseYaml(body.yaml) }
+      catch (err) {
+        return c.json({ ok: false, errors: [`YAML parse error: ${(err as Error).message}`] }, 400)
+      }
+    } else if (body.spec) {
+      spec = body.spec
+    } else {
+      return c.json({ ok: false, errors: ["Provide either `yaml` (string) or `spec` (object)"] }, 400)
+    }
+
+    const result = validateSpec(spec, { registry: runtime.registry })
+    return c.json({
+      ok: result.ok,
+      errors: result.errors,
+      warnings: result.warnings ?? [],
+      edges: result.edges ?? [],
+    }, result.ok ? 200 : 400)
+  })
+
+  function getRunBroker(runId: number): RunEventBroker {
+    let broker = runEvents.get(runId)
+    if (!broker) {
+      broker = createRunEventBroker()
+      runEvents.set(runId, broker)
+    }
+    return broker
   }
-  const task = getTask(runtimeDb(), outcome.suspendedAt.taskId)!
-  return {
-    id: runId, pipeline: pipelineId, status: "suspended",
-    suspendedAt: outcome.suspendedAt.nodeId,
-    task: {
-      id: task.id,
-      kind: task.kind,
-      prompt: task.prompt,
-      schema: task.schema,
-      assignee: task.assignee,
-      responseToken: task.responseToken,
-    },
-    steps: outcome.steps,
+
+  async function buildBaseCtx(runId: number, emit?: (event: ToistEvent) => void | Promise<void>): Promise<RunCtx> {
+    return {
+      runId,
+      runtime,
+      log: makeLogger(runtime, runId),
+      resource: await runtime.resources.resolve(),
+      emit,
+    }
   }
-}
 
-apiApp.post("/pipelines/:id/run", async (c) => {
-  const id = c.req.param("id")
-  const spec = getPipeline(id)
-  if (!spec) return c.json({ error: "not found" }, 404)
-
-  const payload = await c.req.json().catch(() => ({}))
-  const run = runtimeDb().prepare(
-    "INSERT INTO runs (pipeline, status, payload, trigger, updated_at) VALUES (?, 'running', ?, 'api', datetime('now')) RETURNING id",
-  ).get(id, JSON.stringify(payload)) as { id: number }
-
-  try {
-    const outcome = await runPipeline(spec, payload as Record<string, unknown>, buildBaseCtx(run.id))
-    persistOutcome(run.id, outcome)
-    return c.json(outcomeResponse(run.id, id, outcome))
-  } catch (err: unknown) {
-    const message = err instanceof Error ? err.message : String(err)
-    runtimeDb().prepare(
-      "UPDATE runs SET status='error', error=?, finished_at=datetime('now'), updated_at=datetime('now') WHERE id=?",
-    ).run(message, run.id)
-    return c.json({ id: run.id, pipeline: id, status: "error", error: message }, 500)
+  async function persistOutcome(runId: number, outcome: RunOutcome): Promise<void> {
+    if (outcome.status === "done") {
+      await runtime.runs.markDone(runId, outcome.output, outcome.steps)
+    } else {
+      await runtime.outputs.putMany(runId, outcome.results)
+      await runtime.runs.markSuspended(runId, outcome.suspendedAt.nodeId, outcome.steps)
+    }
   }
-})
-
-// ─── HITL — tasks queue + resume ─────────────────────────────────────────────
-
-apiApp.get("/tasks", (c) => {
-  const status   = c.req.query("status") ?? "open"
-  const assignee = c.req.query("assignee")
-  const runId    = c.req.query("run_id")
-  const limit    = Number(c.req.query("limit") ?? 200)
-
-  const where: string[] = []
-  const args: Array<string | number> = []
-  if (status !== "all") { where.push("t.status = ?");   args.push(status) }
-  if (assignee)         { where.push("t.assignee = ?"); args.push(assignee) }
-  if (runId)            { where.push("t.run_id = ?");   args.push(Number(runId)) }
-
-  const whereSql = where.length ? `WHERE ${where.join(" AND ")}` : ""
-  args.push(limit)
-  const sql =
-    `SELECT t.id, t.run_id, t.node_id, t.kind, t.prompt, t.assignee, t.status,
-            t.created_at, t.responded_at, r.pipeline AS pipeline
-       FROM tasks t LEFT JOIN runs r ON t.run_id = r.id
-       ${whereSql}
-       ORDER BY t.id DESC LIMIT ?`
-  return c.json(runtimeDb().prepare(sql).all(...args))
-})
-
-apiApp.get("/tasks/:id", (c) => {
-  const task = getTask(runtimeDb(), Number(c.req.param("id")))
-  if (!task) return c.json({ error: "not found" }, 404)
-  return c.json(task)
-})
-
-apiApp.post("/runs/:runId/tasks/:taskId/respond", async (c) => {
-  const runId  = Number(c.req.param("runId"))
-  const taskId = Number(c.req.param("taskId"))
-  const body = await c.req.json().catch(() => ({})) as {
-    token?: string
-    response?: unknown
-    respondedBy?: string
+
+  async function outcomeResponse(runId: number, pipelineId: string, outcome: RunOutcome) {
+    if (outcome.status === "done") {
+      return { id: runId, pipeline: pipelineId, status: "done", result: outcome.output, steps: outcome.steps }
+    }
+    const task = await runtime.tasks.get(outcome.suspendedAt.taskId)
+    if (!task) throw new Error(`task ${outcome.suspendedAt.taskId} not found`)
+    return {
+      id: runId,
+      pipeline: pipelineId,
+      status: "suspended",
+      suspendedAt: outcome.suspendedAt.nodeId,
+      task: {
+        id: task.id,
+        kind: task.kind,
+        prompt: task.prompt,
+        schema: task.schema,
+        assignee: task.assignee,
+        responseToken: task.responseToken,
+      },
+      steps: outcome.steps,
+    }
   }
-  if (!body.token) return c.json({ error: "missing token" }, 400)
 
-  // Read the task BEFORE answering so we know its kind (drives resume semantics).
-  const taskBefore = getTask(runtimeDb(), taskId)
-  if (!taskBefore) return c.json({ error: "task not found" }, 404)
+  async function executeRun(
+    spec: PipelineSpec,
+    pipelineId: string,
+    payload: Record<string, unknown>,
+    runId: number,
+    options: { rememberAdHoc?: boolean; resumeOutputs?: Record<string, unknown> } = {},
+  ) {
+    if (options.rememberAdHoc || adHocRuns.has(runId)) adHocRuns.set(runId, { spec, payload })
+
+    const broker = getRunBroker(runId)
+    const emit = async (event: ToistEvent) => { await broker.emit(event) }
+    await emit({ type: "run.started", runId, pipelineId })
+
+    try {
+      const outcome = await runPipeline(spec, payload, await buildBaseCtx(runId, emit), {
+        resumeOutputs: options.resumeOutputs,
+      })
+      await persistOutcome(runId, outcome)
+
+      if (outcome.status === "done") {
+        adHocRuns.delete(runId)
+        await emit({ type: "run.completed", runId, output: outcome.output })
+      } else {
+        await emit({
+          type: "run.suspended",
+          runId,
+          nodeId: outcome.suspendedAt.nodeId,
+          taskId: outcome.suspendedAt.taskId,
+          taskKind: outcome.suspendedAt.cause.kind,
+        })
+      }
 
-  try {
-    answerTask(runtimeDb(), taskId, body.token, body.response, body.respondedBy ?? null)
-  } catch (err) {
-    return c.json({ error: err instanceof Error ? err.message : String(err) }, 400)
+      return await outcomeResponse(runId, pipelineId, outcome)
+    } catch (err: unknown) {
+      adHocRuns.delete(runId)
+      const message = err instanceof Error ? err.message : String(err)
+      await runtime.runs.markError(runId, message)
+      await emit({ type: "run.error", runId, error: message })
+      return { id: runId, pipeline: pipelineId, status: "error", error: message }
+    } finally {
+      broker.finish()
+    }
   }
 
-  const runRow = runtimeDb().prepare("SELECT id, pipeline, payload FROM runs WHERE id = ?").get(runId) as
-    { id: number; pipeline: string; payload: string | null } | undefined
-  if (!runRow) return c.json({ error: "run not found" }, 404)
+  apiApp.post("/pipelines/:id/run", async (c) => {
+    const id = c.req.param("id")
+    const spec = getPipeline(id)
+    if (!spec) return c.json({ error: "not found" }, 404)
+
+    const payload = await c.req.json().catch(() => ({})) as Record<string, unknown>
+    const runId = await runtime.runs.create({ pipeline: id, payload, trigger: "api" })
+    const response = await executeRun(spec, id, payload, runId)
+    return c.json(response, response.status === "error" ? 500 : 200)
+  })
+
+  apiApp.post("/runs", async (c) => {
+    const body = await c.req.json().catch(() => ({})) as {
+      spec?: unknown
+      payload?: Record<string, unknown>
+    }
+    if (!body.spec) return c.json({ error: "spec required" }, 400)
+
+    const result = validateSpec(body.spec, { registry: runtime.registry })
+    if (!result.ok || !result.parsed) {
+      return c.json({ ok: false, errors: result.errors, warnings: result.warnings ?? [] }, 400)
+    }
 
-  const spec = getPipeline(runRow.pipeline)
-  if (!spec) return c.json({ error: `pipeline "${runRow.pipeline}" no longer loaded` }, 500)
+    const payload = body.payload ?? {}
+    const runId = await runtime.runs.create({ pipeline: result.parsed.id, payload, trigger: "api" })
+    void executeRun(result.parsed, result.parsed.id, payload, runId, { rememberAdHoc: true })
+    return c.json({ id: runId, pipeline: result.parsed.id, status: "running" }, 202)
+  })
 
-  const payload = runRow.payload ? JSON.parse(runRow.payload) : {}
+  apiApp.get("/tasks", async (c) => {
+    const status = c.req.query("status") ?? "open"
+    const assignee = c.req.query("assignee")
+    const runId = c.req.query("run_id")
+    const limit = Number(c.req.query("limit") ?? 200)
+    return c.json(await runtime.tasks.list({
+      status,
+      assignee,
+      runId: runId ? Number(runId) : undefined,
+      limit,
+    }))
+  })
 
-  // error_review tasks have a different resume contract: the response carries
-  // an `action` (retry | skip | abort), not free-form user input. Translate
-  // before re-entering the dispatcher.
-  if (taskBefore.kind === "error_review") {
-    const action = (body.response as { action?: string } | null)?.action
-    const value  = (body.response as { value?: unknown } | null)?.value
+  apiApp.get("/tasks/:id", async (c) => {
+    const task = await runtime.tasks.get(Number(c.req.param("id")))
+    if (!task) return c.json({ error: "not found" }, 404)
+    return c.json(task)
+  })
 
-    if (action === "abort") {
-      runtimeDb().prepare(
-        "UPDATE runs SET status='error', error=?, finished_at=datetime('now'), updated_at=datetime('now') WHERE id=?",
-      ).run(`aborted via error_review task ${taskId}`, runId)
-      return c.json({ id: runId, pipeline: runRow.pipeline, status: "error", error: "aborted by reviewer" })
+  apiApp.post("/runs/:runId/tasks/:taskId/respond", async (c) => {
+    const runId = Number(c.req.param("runId"))
+    const taskId = Number(c.req.param("taskId"))
+    const body = await c.req.json().catch(() => ({})) as {
+      token?: string
+      response?: unknown
+      respondedBy?: string
     }
+    if (!body.token) return c.json({ error: "missing token" }, 400)
 
-    if (action === "skip") {
-      // Insert the supplied value as the failed node's output. The dispatcher
-      // resume path will treat the node as already-done.
-      persistNodeOutputs(runtimeDb(), runId, { [taskBefore.nodeId]: value ?? null })
-    } else if (action !== "retry") {
-      return c.json({
-        error: `error_review response must include action: "retry" | "skip" | "abort" (got ${JSON.stringify(action)})`,
-      }, 400)
+    const taskBefore = await runtime.tasks.get(taskId)
+    if (!taskBefore) return c.json({ error: "task not found" }, 404)
+
+    try {
+      await runtime.tasks.answer(taskId, body.token, body.response, body.respondedBy ?? null)
+    } catch (err) {
+      return c.json({ error: err instanceof Error ? err.message : String(err) }, 400)
     }
-    // retry falls through — the failed node has no node_outputs entry, so
-    // re-entering the dispatcher will execute it again.
-  }
 
-  const resumeOutputs = loadNodeOutputs(runtimeDb(), runId)
-
-  runtimeDb().prepare(
-    "UPDATE runs SET status='running', trigger='resumed', current_node=NULL, updated_at=datetime('now') WHERE id=?",
-  ).run(runId)
-
-  try {
-    const outcome = await runPipeline(spec, payload, buildBaseCtx(runId), { resumeOutputs })
-    persistOutcome(runId, outcome)
-    return c.json(outcomeResponse(runId, runRow.pipeline, outcome))
-  } catch (err: unknown) {
-    const message = err instanceof Error ? err.message : String(err)
-    runtimeDb().prepare(
-      "UPDATE runs SET status='error', error=?, finished_at=datetime('now'), updated_at=datetime('now') WHERE id=?",
-    ).run(message, runId)
-    return c.json({ id: runId, pipeline: runRow.pipeline, status: "error", error: message }, 500)
-  }
-})
-
-// ─── runs (runtime ledger) ───────────────────────────────────────────────────
-apiApp.get("/runs", (c) => {
-  const limit = Number(c.req.query("limit") ?? 50)
-  const pipelineFilter = c.req.query("pipeline")
-  const rows = pipelineFilter
-    ? runtimeDb().prepare("SELECT * FROM runs WHERE pipeline = ? ORDER BY id DESC LIMIT ?").all(pipelineFilter, limit)
-    : runtimeDb().prepare("SELECT * FROM runs ORDER BY id DESC LIMIT ?").all(limit)
-  return c.json(rows)
-})
-
-apiApp.get("/runs/:id", (c) => {
-  const row = runtimeDb().prepare("SELECT * FROM runs WHERE id=?").get(Number(c.req.param("id")))
-  if (!row) return c.json({ error: "not found" }, 404)
-  return c.json(row)
-})
-
-apiApp.get("/runs/:id/nodes", (c) => {
-  const runId = Number(c.req.param("id"))
-  const rows = runtimeDb().prepare(
-    "SELECT node_id, output_json, started_at, finished_at FROM node_outputs WHERE run_id = ? ORDER BY started_at ASC",
-  ).all(runId) as Array<{ node_id: string; output_json: string | null; started_at: string; finished_at: string }>
-  return c.json(rows.map((r) => ({
-    nodeId: r.node_id,
-    output: r.output_json != null ? JSON.parse(r.output_json) as unknown : null,
-    startedAt: r.started_at,
-    finishedAt: r.finished_at,
-  })))
-})
-
-apiApp.get("/runs/:id/logs", (c) => {
-  const runId = Number(c.req.param("id"))
-  const rows = runtimeDb().prepare(
-    "SELECT id, ts, level, msg FROM logs WHERE run_id = ? ORDER BY id ASC",
-  ).all(runId)
-  return c.json(rows)
-})
-
-// ─── lifecycle ───────────────────────────────────────────────────────────────
-const cachePruneTimer = setInterval(() => cache.prune(), 5 * 60 * 1000)
-cachePruneTimer.unref?.()
-
-const PORT = Number(process.env.PORT ?? 2132)
-
-// Bun's auto-launch path: only emit the port-log when index.ts is the entry
-// (i.e. legacy dev mode `bun src/index.ts`). When startRunner.ts imports this
-// module to extract the Hono app, it manages its own Bun.serve() and logs
-// against options.port, so this log would be wrong.
-if (import.meta.main) {
-  console.log(`[runner] http://localhost:${PORT}`)
-}
+    const runRow = await runtime.runs.get(runId)
+    if (!runRow) return c.json({ error: "run not found" }, 404)
 
-// Outer app: owns CORS, the /api mount, and the bundled UI static serving
-// (per instance-spec.md §7 lifecycle step 9). Hosts can opt out via
-// startRunner({ disableUi: true }) — useful when fronting with a separate UI
-// or running headless.
-const app = new Hono()
-const corsOrigins = getCorsOrigins()
-app.use("*", corsOrigins !== null ? cors({ origin: corsOrigins }) : cors())
-app.route("/api", apiApp)
-
-if (!isUiDisabled()) {
-  const { distDir } = await import("@toist/ui")
-  const indexPath = join(distDir, "index.html")
-  if (existsSync(indexPath)) {
-    // SPA serving: any GET maps to a file under dist/; non-asset paths fall
-    // back to index.html so the client-side router takes over.
-    //
-    // Content-Type is set explicitly from BunFile.type. `new Response(file)`
-    // alone does not always carry the MIME header through to the wire — at
-    // least in some host environments (observed: published @toist/ui in
-    // node_modules served from a host's runner) the header arrives empty,
-    // which trips the browser's strict MIME check on JS module scripts.
-    // Setting headers explicitly is correct and safe regardless of host.
-    const respond = (file: ReturnType<typeof Bun.file>) =>
-      new Response(file, {
-        headers: { "Content-Type": file.type || "application/octet-stream" },
-      })
-    app.get("/*", async (c) => {
-      const url = new URL(c.req.url).pathname
-      const tryPath = url === "/" ? "/index.html" : url
-      const file = Bun.file(join(distDir, tryPath))
-      if (await file.exists()) return respond(file)
-      // SPA fallback for unknown non-asset routes (no extension)
-      if (!tryPath.includes(".")) {
-        return respond(Bun.file(indexPath))
+    const adHoc = adHocRuns.get(runId)
+    const spec = adHoc?.spec ?? getPipeline(runRow.pipeline)
+    if (!spec) return c.json({ error: `pipeline "${runRow.pipeline}" no longer loaded` }, 500)
+
+    const payload = adHoc?.payload ?? (runRow.payload ? JSON.parse(runRow.payload) : {})
+
+    if (taskBefore.kind === "error_review") {
+      const action = (body.response as { action?: string } | null)?.action
+      const value = (body.response as { value?: unknown } | null)?.value
+
+      if (action === "abort") {
+        await runtime.runs.markError(runId, `aborted via error_review task ${taskId}`)
+        return c.json({ id: runId, pipeline: runRow.pipeline, status: "error", error: "aborted by reviewer" })
+      }
+
+      if (action === "skip") {
+        await runtime.outputs.putMany(runId, { [taskBefore.nodeId]: value ?? null })
+      } else if (action !== "retry") {
+        return c.json({
+          error: `error_review response must include action: "retry" | "skip" | "abort" (got ${JSON.stringify(action)})`,
+        }, 400)
       }
-      return c.notFound()
+    }
+
+    const resumeOutputs = await runtime.outputs.list(runId)
+    await runtime.runs.markRunning(runId, { trigger: "resumed", clearCurrentNode: true })
+
+    const response = await executeRun(spec, runRow.pipeline, payload, runId, {
+      rememberAdHoc: !!adHoc,
+      resumeOutputs,
     })
-  } else {
-    console.warn(
-      `[runner] UI dist not found at ${distDir}; skipping static mount. ` +
-      `Run \`bun run build\` in @toist/ui to populate it.`,
-    )
-  }
-}
+    return c.json(response, response.status === "error" ? 500 : 200)
+  })
+
+  apiApp.get("/runs", async (c) => {
+    const limit = Number(c.req.query("limit") ?? 50)
+    const pipeline = c.req.query("pipeline")
+    return c.json(await runtime.runs.list({ pipeline, limit }))
+  })
+
+  apiApp.get("/runs/:id", async (c) => {
+    const row = await runtime.runs.get(Number(c.req.param("id")))
+    if (!row) return c.json({ error: "not found" }, 404)
+    return c.json(row)
+  })
+
+  apiApp.get("/runs/:id/events", async (c) => {
+    const runId = Number(c.req.param("id"))
+    const broker = runEvents.get(runId)
+    if (!broker) return c.json({ error: "event stream not found" }, 404)
+
+    const encoder = new TextEncoder()
+    const stream = new ReadableStream({
+      async start(controller) {
+        try {
+          for await (const event of broker.subscribe()) {
+            controller.enqueue(encoder.encode(`data: ${JSON.stringify(event)}\n\n`))
+          }
+        } finally {
+          controller.close()
+        }
+      },
+    })
+
+    return new Response(stream, {
+      headers: {
+        "Content-Type": "text/event-stream",
+        "Cache-Control": "no-cache",
+        Connection: "keep-alive",
+      },
+    })
+  })
 
-// Exported `fetch` lets startRunner.ts (and any future caller) bind the Hono
-// app to its own Bun.serve options without going through `default`.
-export const fetch = app.fetch
+  apiApp.get("/runs/:id/nodes", async (c) => {
+    const runId = Number(c.req.param("id"))
+    return c.json(await runtime.outputs.listOrdered(runId))
+  })
+
+  apiApp.get("/runs/:id/logs", async (c) => {
+    const runId = Number(c.req.param("id"))
+    return c.json(await runtime.logs.list(runId))
+  })
+
+  const app = new Hono()
+  app.use("*", options.corsOrigins !== undefined ? cors({ origin: options.corsOrigins }) : cors())
+  app.route("/api", apiApp)
+
+  if (!options.disableUi) {
+    const { distDir } = await import("@toist/ui")
+    const indexPath = join(distDir, "index.html")
+    if (!existsSync(indexPath)) {
+      console.warn(
+        `[runner] UI dist not found at ${distDir}; skipping static mount. ` +
+        "Run `bun run build` in @toist/ui to populate it.",
+      )
+    } else {
+      const respond = (file: ReturnType<typeof Bun.file>) =>
+        new Response(file, {
+          headers: { "Content-Type": file.type || "application/octet-stream" },
+        })
+
+      app.get("/*", async (c) => {
+        const url = new URL(c.req.url).pathname
+        const tryPath = url === "/" ? "/index.html" : url
+        const file = Bun.file(join(distDir, tryPath))
+        if (await file.exists()) return respond(file)
+        if (!tryPath.includes(".")) return respond(Bun.file(indexPath))
+        return c.notFound()
+      })
+    }
+  }
 
-export default { port: PORT, fetch: app.fetch }
+  return app
+}