@toist/aja 0.5.0

package/src/hitl.ts

@@ -0,0 +1,257 @@
+// 2121
+// Platform-side HITL (human-in-the-loop) primitives.
+//
+// `human.input` kinds call ctx.suspend(spec). The suspend implementation
+// here decides between three cases:
+//
+//   1. An answered task already exists for this node in this run
+//      → return its response, kind continues normally.
+//   2. An open task exists (re-suspension before resume completed)
+//      → throw HitlSuspend referencing that task.
+//   3. No task yet
+//      → create a new task with a single-use response_token, throw HitlSuspend.
+//
+// runPipeline catches HitlSuspend and converts it into a SUSPENDED outcome.
+// The HTTP layer persists node_outputs, marks runs.status='suspended', and
+// returns the task descriptor to the caller.
+//
+// On resume, the HTTP layer hydrates node_outputs back into the executor's
+// results map. Re-execution from the suspended node calls ctx.suspend again;
+// case (1) above kicks in and the human.input node returns the response.
+// Side effects in already-completed nodes do not re-run because the executor
+// skips any node whose output is in the resume map.
+
+import { randomBytes } from "node:crypto"
+import type { Database } from "bun:sqlite"
+import type { HitlSpec, ErrorReviewSpec } from "@toist/spec"
+
+export type { HitlSpec, ErrorReviewSpec }
+
+export interface TaskDescriptor {
+  id: number
+  runId: number
+  pipeline: string | null
+  nodeId: string
+  /** "human.input" for HITL pauses, "error_review" for failed-node suspends. */
+  kind: string
+  prompt: string
+  /** For human.input: the structured schema for the response form.
+   *  For error_review: the structured error details (kind id, name) to display. */
+  schema: unknown
+  assignee: string | null
+  responseToken: string
+  status: "open" | "answered" | "expired" | "cancelled"
+}
+
+/**
+ * Thrown by ctx.suspend() to signal "this run cannot proceed past this node
+ * without external input". runPipeline catches this and converts it to a
+ * SUSPENDED RunOutcome — it does not propagate to the HTTP layer as an error.
+ */
+export class HitlSuspend extends Error {
+  constructor(
+    public readonly taskId: number,
+    public readonly nodeId: string,
+    public readonly spec: HitlSpec,
+  ) {
+    super(`HITL suspend at node "${nodeId}" (task ${taskId})`)
+    this.name = "HitlSuspend"
+  }
+}
+
+/**
+ * Thrown by the dispatcher when a node fails and its onError policy is
+ * "suspend". Mirrors HitlSuspend's shape so the dispatcher can treat both
+ * via a single suspend path. The persisted task carries kind='error_review';
+ * resume payload is `{ action: "retry" | "skip" | "abort", value? }`.
+ */
+export class ErrorReviewSuspend extends Error {
+  constructor(
+    public readonly taskId: number,
+    public readonly nodeId: string,
+    public readonly spec: ErrorReviewSpec,
+  ) {
+    super(`error_review suspend at node "${nodeId}" (task ${taskId})`)
+    this.name = "ErrorReviewSuspend"
+  }
+}
+
+/**
+ * Build a suspend(spec) function bound to a specific run + node. Called
+ * fresh for each step in the executor; the closure captures runId / nodeId
+ * so the kind itself never has to think about identifiers.
+ */
+export function makeSuspend(
+  runtimeDb: Database,
+  runId: number,
+  nodeId: string,
+): (spec: HitlSpec) => Promise<unknown> {
+  return async (spec: HitlSpec): Promise<unknown> => {
+    // Case 1: already answered — return the response, kind continues.
+    const answered = runtimeDb.prepare(
+      "SELECT response_json FROM tasks WHERE run_id = ? AND node_id = ? AND status = 'answered' ORDER BY id DESC LIMIT 1",
+    ).get(runId, nodeId) as { response_json: string | null } | undefined
+
+    if (answered) {
+      return answered.response_json ? JSON.parse(answered.response_json) : null
+    }
+
+    // Case 2: an open task already exists — re-throw without creating a duplicate.
+    const open = runtimeDb.prepare(
+      "SELECT id FROM tasks WHERE run_id = ? AND node_id = ? AND status = 'open' ORDER BY id DESC LIMIT 1",
+    ).get(runId, nodeId) as { id: number } | undefined
+
+    if (open) {
+      throw new HitlSuspend(open.id, nodeId, spec)
+    }
+
+    // Case 3: create new task with a single-use response token.
+    const token = randomBytes(16).toString("hex")
+    const inserted = runtimeDb.prepare(
+      `INSERT INTO tasks (run_id, node_id, kind, prompt, schema_json, assignee, response_token)
+       VALUES (?, ?, 'human.input', ?, ?, ?, ?)
+       RETURNING id`,
+    ).get(
+      runId, nodeId, spec.prompt,
+      spec.schema !== undefined ? JSON.stringify(spec.schema) : null,
+      spec.assignee ?? null,
+      token,
+    ) as { id: number }
+
+    throw new HitlSuspend(inserted.id, nodeId, spec)
+  }
+}
+
+/**
+ * Create a fresh `error_review` task for a failing node and return its
+ * id + token. Called by the dispatcher when a node throws and its onError
+ * policy is "suspend". Same pattern as makeSuspend's case-3 path: insert
+ * with a single-use response_token, then throw ErrorReviewSuspend so the
+ * dispatcher's suspend path takes over.
+ *
+ * The task's `prompt` carries the error message; structured details (kind,
+ * stack) live in `schema_json` so the UI can surface them without an
+ * additional column. Resume payload distinguishes by tasks.kind, not by
+ * column shape — keeps the existing tasks table unchanged.
+ */
+export function createErrorReviewTask(
+  runtimeDb: Database,
+  runId: number,
+  nodeId: string,
+  spec: ErrorReviewSpec,
+): { id: number; token: string } {
+  const token = randomBytes(16).toString("hex")
+  const inserted = runtimeDb.prepare(
+    `INSERT INTO tasks (run_id, node_id, kind, prompt, schema_json, assignee, response_token)
+     VALUES (?, ?, 'error_review', ?, ?, ?, ?)
+     RETURNING id`,
+  ).get(
+    runId, nodeId, spec.prompt,
+    spec.details !== undefined ? JSON.stringify(spec.details) : null,
+    spec.assignee ?? null,
+    token,
+  ) as { id: number }
+  return { id: inserted.id, token }
+}
+
+/**
+ * Persist node outputs gathered before a suspension. Idempotent: re-running
+ * the same insertion is a no-op via INSERT OR REPLACE — useful when the
+ * executor re-runs upstream nodes during a resume.
+ */
+export function persistNodeOutputs(
+  runtimeDb: Database,
+  runId: number,
+  outputs: Record<string, unknown>,
+): void {
+  const stmt = runtimeDb.prepare(
+    `INSERT OR REPLACE INTO node_outputs (run_id, node_id, output_json, finished_at)
+     VALUES (?, ?, ?, datetime('now'))`,
+  )
+  const tx = runtimeDb.transaction(() => {
+    for (const [nodeId, output] of Object.entries(outputs)) {
+      stmt.run(runId, nodeId, JSON.stringify(output ?? null))
+    }
+  })
+  tx()
+}
+
+/**
+ * Load all previously-persisted node outputs for a run. Returned as a
+ * { nodeId → output } map ready to seed the executor's results scope.
+ */
+export function loadNodeOutputs(
+  runtimeDb: Database,
+  runId: number,
+): Record<string, unknown> {
+  const rows = runtimeDb.prepare(
+    "SELECT node_id, output_json FROM node_outputs WHERE run_id = ?",
+  ).all(runId) as { node_id: string; output_json: string | null }[]
+
+  const out: Record<string, unknown> = {}
+  for (const row of rows) {
+    out[row.node_id] = row.output_json ? JSON.parse(row.output_json) : null
+  }
+  return out
+}
+
+/**
+ * Fetch a single task by id. Joins runs to surface the pipeline id alongside
+ * the task — useful for cross-section navigation in the UI. Returns null if
+ * not found.
+ */
+export function getTask(runtimeDb: Database, taskId: number): TaskDescriptor | null {
+  const row = runtimeDb.prepare(
+    `SELECT t.id, t.run_id, t.node_id, t.kind, t.prompt, t.schema_json, t.assignee,
+            t.response_token, t.status, r.pipeline AS pipeline
+       FROM tasks t LEFT JOIN runs r ON t.run_id = r.id
+      WHERE t.id = ?`,
+  ).get(taskId) as {
+    id: number; run_id: number; node_id: string; kind: string; prompt: string | null
+    schema_json: string | null; assignee: string | null
+    response_token: string; status: string; pipeline: string | null
+  } | undefined
+
+  if (!row) return null
+  return {
+    id:            row.id,
+    runId:         row.run_id,
+    pipeline:      row.pipeline,
+    nodeId:        row.node_id,
+    kind:          row.kind,
+    prompt:        row.prompt ?? "",
+    schema:        row.schema_json ? JSON.parse(row.schema_json) : null,
+    assignee:      row.assignee,
+    responseToken: row.response_token,
+    status:        row.status as TaskDescriptor["status"],
+  }
+}
+
+/**
+ * Record a response against an open task. Validates the token matches and
+ * the task is still open; refuses replay (token is single-use). Returns the
+ * updated task descriptor on success, throws on validation failure.
+ */
+export function answerTask(
+  runtimeDb: Database,
+  taskId: number,
+  token: string,
+  response: unknown,
+  respondedBy: string | null,
+): TaskDescriptor {
+  const task = getTask(runtimeDb, taskId)
+  if (!task) throw new Error(`task ${taskId} not found`)
+  if (task.status !== "open") throw new Error(`task ${taskId} is ${task.status}, cannot answer`)
+  if (task.responseToken !== token) throw new Error(`task ${taskId}: invalid response token`)
+
+  runtimeDb.prepare(
+    `UPDATE tasks
+       SET status = 'answered',
+           response_json = ?,
+           responded_by  = ?,
+           responded_at  = datetime('now')
+     WHERE id = ?`,
+  ).run(JSON.stringify(response ?? null), respondedBy, taskId)
+
+  return { ...task, status: "answered" }
+}

package/src/index.ts

@@ -0,0 +1,34 @@
+// 2121
+// @toist/aja — public API entry per context/instance-spec.md §4.
+//
+// Hosts that consume this package get exactly the surface declared here.
+// Anything not re-exported is internal and may change without notice.
+//
+// Bundled MCP, UI mounting, per-path overrides, custom logger injection,
+// and the built-in kind set are all reachable through `startRunner`.
+
+// Lifecycle entry — what hosts call to start an instance.
+export { startRunner, type StartRunnerOptions, type RunnerHandle } from "./startRunner.ts"
+
+// Kind registration — must be called before startRunner.
+export { register, getKind, manifest } from "./kinds/index.ts"
+
+// Type re-exports from @toist/spec, so a host that imports only
+// @toist/aja doesn't need a second package on its dependency list when
+// it only writes kinds (rather than authoring pipeline-format tooling).
+export type {
+  NodeKind,
+  NodeKindManifest,
+  ParamDef,
+  PortDef,
+  ExecContext,
+  PlatformCtx,
+  Cache,
+  HitlSpec,
+  OnErrorPolicy,
+  ErrorReviewSpec,
+  RunStore,
+  SubRun,
+  SubRunOutcome,
+  ResourceTypeDef,
+} from "@toist/spec"

package/src/instance.ts

@@ -0,0 +1,64 @@
+// 2121
+// Per-instance metadata: what kind of platform-instance this is (which
+// customer, what tier, what teasers to surface), distinct from the runtime
+// state in runtime.db.
+//
+// Lives at <rootDir>/instance.json (per context/instance-spec.md §6).
+// Optional — when missing, sensible defaults are returned (useful for the
+// in-monorepo template-runner where no scaffold has happened).
+//
+// Schema (current, may grow):
+//
+//   {
+//     "platformVersion": "0.1.0",      // recorded at scaffold time
+//     "instanceName":    "Enio Runner", // human-readable label, optional
+//     "tier":            "pilot",      // free-form string, optional
+//     "teasers": [                     // available-but-not-shipped pipelines
+//       {
+//         "id":          "sales-kpi",
+//         "title":       "Salesman KPI tracking",
+//         "description": "Weekly KPI rollup per salesman with conversion tracking.",
+//         "category":    "analytics"
+//       }
+//     ]
+//   }
+
+import { existsSync, readFileSync } from "node:fs"
+import { instanceFilePath } from "./config.ts"
+
+export interface Teaser {
+  id:           string
+  title:        string
+  description?: string
+  category?:    string
+}
+
+export interface Instance {
+  platformVersion?: string
+  instanceName?:    string
+  tier?:            string
+  teasers:          Teaser[]
+}
+
+const DEFAULTS: Instance = { teasers: [] }
+
+// Re-read on every call. instance.json is tiny and changes rarely; the cost
+// is negligible and avoids stale-cache surprises when the file is edited.
+export function loadInstance(): Instance {
+  const path = instanceFilePath()
+  if (!existsSync(path)) return DEFAULTS
+
+  try {
+    const raw = readFileSync(path, "utf8")
+    const parsed = JSON.parse(raw) as Partial<Instance>
+    return {
+      platformVersion: parsed.platformVersion,
+      instanceName:    parsed.instanceName,
+      tier:            parsed.tier,
+      teasers:         Array.isArray(parsed.teasers) ? parsed.teasers : [],
+    }
+  } catch (err) {
+    console.warn(`[instance] failed to read ${path}:`, (err as Error).message)
+    return DEFAULTS
+  }
+}

package/src/kinds/control.ts

@@ -0,0 +1,26 @@
+// 2121
+import type { NodeKind } from "./types.ts"
+
+export const trigger: NodeKind<Record<string, never>, Record<string, never>> = {
+  id: "trigger",
+  category: "control",
+  label: "Trigger",
+  description: "Pipeline entrypoint. Outputs the run payload — reference the run params with $params.X anywhere in the graph.",
+  icon: "Zap",
+  params: {},
+  inputs: {},
+  outputs: { payload: { type: "any" } },
+  run: (_ctx, _params, _input) => null,
+}
+
+export const sink: NodeKind<Record<string, never>, { value: unknown }> = {
+  id: "sink",
+  category: "control",
+  label: "Output",
+  description: "Pipeline output. The value passed in becomes the run result.",
+  icon: "ArrowUpFromLine",
+  params: {},
+  inputs:  { value: { type: "any", required: true } },
+  outputs: {},
+  run: (_ctx, _params, input) => input.value,
+}

package/src/kinds/custom.ts

@@ -0,0 +1,19 @@
+// 2121
+// Domain kinds — register your project-specific kinds here. This file is
+// imported automatically when the kind registry initialises. It ships empty
+// in a fresh `platform init` project; add your kinds and call `register(...)`.
+//
+// Example (eniopro):
+//
+//   import { register } from "./index.ts"
+//   import { listBuildingsKind } from "./enio.list-buildings.ts"
+//   import { signalKind }        from "./enio.signal.ts"
+//   import { rankedBuildingsKind } from "./score.ranked-buildings.ts"
+//
+//   register(listBuildingsKind, signalKind, rankedBuildingsKind)
+//
+// Builtin kinds (trigger, sink, data.json, transform.*, db.*) are registered
+// in ./index.ts. Custom kinds can shadow builtins by re-using their id, but
+// you'll see a warning at startup.
+
+export {}

package/src/kinds/data.ts

@@ -0,0 +1,30 @@
+// 2121
+import type { NodeKind } from "./types.ts"
+
+export const dataJson: NodeKind<{ value: unknown }, Record<string, never>> = {
+  id: "data.json",
+  category: "data",
+  label: "JSON",
+  description: "Emit a constant JSON value as the output. Useful as a source.",
+  icon: "Braces",
+  params: {
+    value: { type: "json", label: "Value", required: true, description: "Any JSON value (object, array, scalar)" },
+  },
+  inputs: {},
+  outputs: { value: { type: "any" } },
+  run: (_ctx, params) => params.value,
+}
+
+export const dataMerge: NodeKind<{ overlay?: object }, { input: object }> = {
+  id: "data.merge",
+  category: "data",
+  label: "Merge",
+  description: "Shallow-merge runtime input with a constant overlay.",
+  icon: "Combine",
+  params: {
+    overlay: { type: "json", label: "Overlay", default: {} },
+  },
+  inputs:  { input: { type: "object", required: true } },
+  outputs: { output: { type: "object" } },
+  run: (_ctx, params, input) => ({ ...input.input, ...(params.overlay ?? {}) }),
+}

package/src/kinds/db.ts

@@ -0,0 +1,92 @@
+// 2121
+import type { NodeKind } from "./types.ts"
+
+export const dbInsert: NodeKind<
+  { table: string; mode?: "insert" | "upsert"; key?: string | string[] },
+  { rows: unknown[] }
+> = {
+  id: "db.insert",
+  category: "db",
+  label: "DB Insert",
+  description:
+    "Insert (or upsert) array items into a SQLite table. Columns are inferred from the first row. " +
+    "For mode=upsert you must specify `key` (column or column list) — a UNIQUE index is created on " +
+    "first run so subsequent runs replace matching rows instead of duplicating them.",
+  icon: "Database",
+  sideEffect: true,
+  params: {
+    table: { type: "string", label: "Table", required: true, placeholder: "salesman_kpi" },
+    mode:  { type: "select", label: "Mode", default: "insert",
+             options: [
+               { value: "insert", label: "INSERT (append)" },
+               { value: "upsert", label: "UPSERT (replace by key)" },
+             ] },
+    key:   { type: "string", label: "Unique key", description: "Column name (or comma-separated list) for UPSERT mode. The kind creates a UNIQUE index on first run.", placeholder: "id" },
+  },
+  inputs:  { rows: { type: "array", required: true } },
+  outputs: { result: { type: "object" } },
+  run: (ctx, params, input) => {
+    const rows = Array.isArray(input.rows) ? input.rows : []
+    if (rows.length === 0) return { inserted: 0, table: params.table }
+
+    const cols = Object.keys(rows[0] as Record<string, unknown>)
+    if (cols.length === 0) return { inserted: 0, table: params.table }
+
+    const mode = params.mode ?? "insert"
+
+    if (mode === "upsert" && !params.key) {
+      throw new Error(
+        `db.insert: mode "upsert" requires "key" param (column or column list). ` +
+        `Without a UNIQUE index every run would duplicate every row.`,
+      )
+    }
+
+    const colDefs = cols.map((c) => `${c} TEXT`).join(", ")
+    ctx.db.exec(`CREATE TABLE IF NOT EXISTS ${params.table} (${colDefs})`)
+
+    if (mode === "upsert") {
+      const keyCols = Array.isArray(params.key) ? params.key : String(params.key).split(",").map(s => s.trim())
+      const idxName = `idx_${params.table}_${keyCols.join("_")}_unique`
+      try {
+        ctx.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS ${idxName} ON ${params.table} (${keyCols.join(", ")})`)
+      } catch (err) {
+        throw new Error(
+          `db.insert: cannot create UNIQUE index on ${params.table}(${keyCols.join(", ")}) — ` +
+          `existing rows likely violate uniqueness. Drop the table or remove duplicates first. ` +
+          `Underlying error: ${(err as Error).message}`,
+        )
+      }
+    }
+
+    const placeholders = cols.map(() => "?").join(", ")
+    const verb = mode === "upsert" ? "INSERT OR REPLACE" : "INSERT"
+    const stmt = ctx.db.prepare(`${verb} INTO ${params.table} (${cols.join(", ")}) VALUES (${placeholders})`)
+
+    const txn = ctx.db.transaction((items: unknown[]) => {
+      for (const it of items) {
+        const r = it as Record<string, unknown>
+        stmt.run(...cols.map((c) => {
+          const v = r[c]
+          return typeof v === "object" && v !== null ? JSON.stringify(v) : (v as never)
+        }))
+      }
+    })
+    txn(rows)
+
+    return { inserted: rows.length, table: params.table, mode }
+  },
+}
+
+export const dbQuery: NodeKind<{ sql: string }, Record<string, never>> = {
+  id: "db.query",
+  category: "db",
+  label: "DB Query",
+  description: "Run a SELECT query against the runner's SQLite DB.",
+  icon: "Search",
+  params: {
+    sql: { type: "string", label: "SQL", required: true, placeholder: "SELECT * FROM salesman_kpi ORDER BY score DESC LIMIT 10" },
+  },
+  inputs:  {},
+  outputs: { rows: { type: "array" } },
+  run: (ctx, params) => ctx.db.prepare(params.sql).all(),
+}

package/src/kinds/hitl.ts

@@ -0,0 +1,56 @@
+// 2121
+// HITL kinds. The actual suspend/resume bookkeeping lives in ../hitl.ts on
+// the platform side; this file only declares the kind contract.
+
+import type { NodeKind } from "./types.ts"
+
+interface HumanInputParams {
+  prompt:    string
+  assignee?: string
+  schema?:   unknown
+  timeout?:  string
+}
+
+export const humanInput: NodeKind<HumanInputParams, Record<string, never>> = {
+  id: "human.input",
+  category: "control",
+  label: "Human input",
+  description: "Suspends the run until the assignee responds with input. The response becomes this node's output and downstream nodes can $results.<id>.<field> it.",
+  icon: "MessageSquareReply",
+  params: {
+    prompt: {
+      type: "string", required: true,
+      description: "Question shown to the assignee in the task queue.",
+      placeholder: "Approve campaign launch?",
+    },
+    assignee: {
+      type: "string",
+      description: "Who should respond. Format: user:<email> or agent:<id>. Empty = open to anyone.",
+      placeholder: "user:teemu@2121.fi",
+    },
+    schema: {
+      type: "json",
+      description: "JSON Schema describing the expected response shape. v1: descriptive only, not enforced.",
+    },
+    timeout: {
+      type: "string",
+      description: "ISO 8601 duration before the task expires (e.g. \"P7D\"). v1: stored, not enforced.",
+      placeholder: "P7D",
+    },
+  },
+  inputs: {},
+  outputs: {
+    response: {
+      type: "any",
+      description: "Whatever the assignee submitted as their response.",
+    },
+  },
+  run: async (ctx, params) => {
+    return await ctx.suspend({
+      prompt:   params.prompt,
+      assignee: params.assignee,
+      schema:   params.schema,
+      timeout:  params.timeout,
+    })
+  },
+}