@toist/aja 0.6.1 → 0.8.0

package/src/db-handles.ts

@@ -1,70 +0,0 @@
-// 2121
-// Singleton db handle holder + lifecycle orchestration. Decouples the db
-// modules (which are now pure factories per `openX()`) from the consumers
-// (index.ts, pipeline.ts) that need stable handle references throughout the
-// HTTP request lifetime.
-//
-// Lifecycle (matches context/instance-spec.md §7):
-//
-//   initDbs()     — acquire runner lock, migrate runtime.db, open all three
-//                   handles. Idempotent: second call is a no-op when handles
-//                   are already populated.
-//   runtimeDb()   — getter; throws when init has not run
-//   dataDb()      — getter; throws when init has not run
-//   cacheDb()     — getter; throws when init has not run
-//   closeDbs()    — close all three handles, release the lock. Used by
-//                   `startRunner`'s graceful stop path.
-//
-// Why getters and not exported instances: the db modules used to do their
-// init at module load via top-level await, which forced lifecycle to happen
-// during the import graph. That works for source-mod single-instance runs
-// but breaks the moment startRunner() needs to set rootDir before init runs.
-// Getters defer the resolution to call time; init runs once, before any
-// HTTP request fires.
-
-import type { Database } from "bun:sqlite"
-import { openRuntimeDb } from "./runtime-db.ts"
-import { openDataDb } from "./data-db.ts"
-import { openCacheDb } from "./cache-db.ts"
-import { acquireRunnerLock, releaseRunnerLock } from "./lock.ts"
-
-let _runtimeDb: Database | null = null
-let _dataDb: Database | null = null
-let _cacheDb: Database | null = null
-
-export async function initDbs(): Promise<void> {
-  if (_runtimeDb !== null) return  // idempotent
-
-  await acquireRunnerLock()
-  _runtimeDb = openRuntimeDb()
-  _dataDb = openDataDb()
-  _cacheDb = openCacheDb()
-}
-
-export async function closeDbs(): Promise<void> {
-  if (_runtimeDb) { _runtimeDb.close(); _runtimeDb = null }
-  if (_dataDb)    { _dataDb.close();    _dataDb = null }
-  if (_cacheDb)   { _cacheDb.close();   _cacheDb = null }
-  await releaseRunnerLock()
-}
-
-export function runtimeDb(): Database {
-  if (_runtimeDb === null) {
-    throw new Error("[db-handles] runtimeDb() called before initDbs() — lifecycle violation")
-  }
-  return _runtimeDb
-}
-
-export function dataDb(): Database {
-  if (_dataDb === null) {
-    throw new Error("[db-handles] dataDb() called before initDbs() — lifecycle violation")
-  }
-  return _dataDb
-}
-
-export function cacheDb(): Database {
-  if (_cacheDb === null) {
-    throw new Error("[db-handles] cacheDb() called before initDbs() — lifecycle violation")
-  }
-  return _cacheDb
-}

package/src/hitl.ts

@@ -1,257 +0,0 @@
-// 2121
-// Platform-side HITL (human-in-the-loop) primitives.
-//
-// `human.input` kinds call ctx.suspend(spec). The suspend implementation
-// here decides between three cases:
-//
-//   1. An answered task already exists for this node in this run
-//      → return its response, kind continues normally.
-//   2. An open task exists (re-suspension before resume completed)
-//      → throw HitlSuspend referencing that task.
-//   3. No task yet
-//      → create a new task with a single-use response_token, throw HitlSuspend.
-//
-// runPipeline catches HitlSuspend and converts it into a SUSPENDED outcome.
-// The HTTP layer persists node_outputs, marks runs.status='suspended', and
-// returns the task descriptor to the caller.
-//
-// On resume, the HTTP layer hydrates node_outputs back into the executor's
-// results map. Re-execution from the suspended node calls ctx.suspend again;
-// case (1) above kicks in and the human.input node returns the response.
-// Side effects in already-completed nodes do not re-run because the executor
-// skips any node whose output is in the resume map.
-
-import { randomBytes } from "node:crypto"
-import type { Database } from "bun:sqlite"
-import type { HitlSpec, ErrorReviewSpec } from "@toist/spec"
-
-export type { HitlSpec, ErrorReviewSpec }
-
-export interface TaskDescriptor {
-  id: number
-  runId: number
-  pipeline: string | null
-  nodeId: string
-  /** "human.input" for HITL pauses, "error_review" for failed-node suspends. */
-  kind: string
-  prompt: string
-  /** For human.input: the structured schema for the response form.
-   *  For error_review: the structured error details (kind id, name) to display. */
-  schema: unknown
-  assignee: string | null
-  responseToken: string
-  status: "open" | "answered" | "expired" | "cancelled"
-}
-
-/**
- * Thrown by ctx.suspend() to signal "this run cannot proceed past this node
- * without external input". runPipeline catches this and converts it to a
- * SUSPENDED RunOutcome — it does not propagate to the HTTP layer as an error.
- */
-export class HitlSuspend extends Error {
-  constructor(
-    public readonly taskId: number,
-    public readonly nodeId: string,
-    public readonly spec: HitlSpec,
-  ) {
-    super(`HITL suspend at node "${nodeId}" (task ${taskId})`)
-    this.name = "HitlSuspend"
-  }
-}
-
-/**
- * Thrown by the dispatcher when a node fails and its onError policy is
- * "suspend". Mirrors HitlSuspend's shape so the dispatcher can treat both
- * via a single suspend path. The persisted task carries kind='error_review';
- * resume payload is `{ action: "retry" | "skip" | "abort", value? }`.
- */
-export class ErrorReviewSuspend extends Error {
-  constructor(
-    public readonly taskId: number,
-    public readonly nodeId: string,
-    public readonly spec: ErrorReviewSpec,
-  ) {
-    super(`error_review suspend at node "${nodeId}" (task ${taskId})`)
-    this.name = "ErrorReviewSuspend"
-  }
-}
-
-/**
- * Build a suspend(spec) function bound to a specific run + node. Called
- * fresh for each step in the executor; the closure captures runId / nodeId
- * so the kind itself never has to think about identifiers.
- */
-export function makeSuspend(
-  runtimeDb: Database,
-  runId: number,
-  nodeId: string,
-): (spec: HitlSpec) => Promise<unknown> {
-  return async (spec: HitlSpec): Promise<unknown> => {
-    // Case 1: already answered — return the response, kind continues.
-    const answered = runtimeDb.prepare(
-      "SELECT response_json FROM tasks WHERE run_id = ? AND node_id = ? AND status = 'answered' ORDER BY id DESC LIMIT 1",
-    ).get(runId, nodeId) as { response_json: string | null } | undefined
-
-    if (answered) {
-      return answered.response_json ? JSON.parse(answered.response_json) : null
-    }
-
-    // Case 2: an open task already exists — re-throw without creating a duplicate.
-    const open = runtimeDb.prepare(
-      "SELECT id FROM tasks WHERE run_id = ? AND node_id = ? AND status = 'open' ORDER BY id DESC LIMIT 1",
-    ).get(runId, nodeId) as { id: number } | undefined
-
-    if (open) {
-      throw new HitlSuspend(open.id, nodeId, spec)
-    }
-
-    // Case 3: create new task with a single-use response token.
-    const token = randomBytes(16).toString("hex")
-    const inserted = runtimeDb.prepare(
-      `INSERT INTO tasks (run_id, node_id, kind, prompt, schema_json, assignee, response_token)
-       VALUES (?, ?, 'human.input', ?, ?, ?, ?)
-       RETURNING id`,
-    ).get(
-      runId, nodeId, spec.prompt,
-      spec.schema !== undefined ? JSON.stringify(spec.schema) : null,
-      spec.assignee ?? null,
-      token,
-    ) as { id: number }
-
-    throw new HitlSuspend(inserted.id, nodeId, spec)
-  }
-}
-
-/**
- * Create a fresh `error_review` task for a failing node and return its
- * id + token. Called by the dispatcher when a node throws and its onError
- * policy is "suspend". Same pattern as makeSuspend's case-3 path: insert
- * with a single-use response_token, then throw ErrorReviewSuspend so the
- * dispatcher's suspend path takes over.
- *
- * The task's `prompt` carries the error message; structured details (kind,
- * stack) live in `schema_json` so the UI can surface them without an
- * additional column. Resume payload distinguishes by tasks.kind, not by
- * column shape — keeps the existing tasks table unchanged.
- */
-export function createErrorReviewTask(
-  runtimeDb: Database,
-  runId: number,
-  nodeId: string,
-  spec: ErrorReviewSpec,
-): { id: number; token: string } {
-  const token = randomBytes(16).toString("hex")
-  const inserted = runtimeDb.prepare(
-    `INSERT INTO tasks (run_id, node_id, kind, prompt, schema_json, assignee, response_token)
-     VALUES (?, ?, 'error_review', ?, ?, ?, ?)
-     RETURNING id`,
-  ).get(
-    runId, nodeId, spec.prompt,
-    spec.details !== undefined ? JSON.stringify(spec.details) : null,
-    spec.assignee ?? null,
-    token,
-  ) as { id: number }
-  return { id: inserted.id, token }
-}
-
-/**
- * Persist node outputs gathered before a suspension. Idempotent: re-running
- * the same insertion is a no-op via INSERT OR REPLACE — useful when the
- * executor re-runs upstream nodes during a resume.
- */
-export function persistNodeOutputs(
-  runtimeDb: Database,
-  runId: number,
-  outputs: Record<string, unknown>,
-): void {
-  const stmt = runtimeDb.prepare(
-    `INSERT OR REPLACE INTO node_outputs (run_id, node_id, output_json, finished_at)
-     VALUES (?, ?, ?, datetime('now'))`,
-  )
-  const tx = runtimeDb.transaction(() => {
-    for (const [nodeId, output] of Object.entries(outputs)) {
-      stmt.run(runId, nodeId, JSON.stringify(output ?? null))
-    }
-  })
-  tx()
-}
-
-/**
- * Load all previously-persisted node outputs for a run. Returned as a
- * { nodeId → output } map ready to seed the executor's results scope.
- */
-export function loadNodeOutputs(
-  runtimeDb: Database,
-  runId: number,
-): Record<string, unknown> {
-  const rows = runtimeDb.prepare(
-    "SELECT node_id, output_json FROM node_outputs WHERE run_id = ?",
-  ).all(runId) as { node_id: string; output_json: string | null }[]
-
-  const out: Record<string, unknown> = {}
-  for (const row of rows) {
-    out[row.node_id] = row.output_json ? JSON.parse(row.output_json) : null
-  }
-  return out
-}
-
-/**
- * Fetch a single task by id. Joins runs to surface the pipeline id alongside
- * the task — useful for cross-section navigation in the UI. Returns null if
- * not found.
- */
-export function getTask(runtimeDb: Database, taskId: number): TaskDescriptor | null {
-  const row = runtimeDb.prepare(
-    `SELECT t.id, t.run_id, t.node_id, t.kind, t.prompt, t.schema_json, t.assignee,
-            t.response_token, t.status, r.pipeline AS pipeline
-       FROM tasks t LEFT JOIN runs r ON t.run_id = r.id
-      WHERE t.id = ?`,
-  ).get(taskId) as {
-    id: number; run_id: number; node_id: string; kind: string; prompt: string | null
-    schema_json: string | null; assignee: string | null
-    response_token: string; status: string; pipeline: string | null
-  } | undefined
-
-  if (!row) return null
-  return {
-    id:            row.id,
-    runId:         row.run_id,
-    pipeline:      row.pipeline,
-    nodeId:        row.node_id,
-    kind:          row.kind,
-    prompt:        row.prompt ?? "",
-    schema:        row.schema_json ? JSON.parse(row.schema_json) : null,
-    assignee:      row.assignee,
-    responseToken: row.response_token,
-    status:        row.status as TaskDescriptor["status"],
-  }
-}
-
-/**
- * Record a response against an open task. Validates the token matches and
- * the task is still open; refuses replay (token is single-use). Returns the
- * updated task descriptor on success, throws on validation failure.
- */
-export function answerTask(
-  runtimeDb: Database,
-  taskId: number,
-  token: string,
-  response: unknown,
-  respondedBy: string | null,
-): TaskDescriptor {
-  const task = getTask(runtimeDb, taskId)
-  if (!task) throw new Error(`task ${taskId} not found`)
-  if (task.status !== "open") throw new Error(`task ${taskId} is ${task.status}, cannot answer`)
-  if (task.responseToken !== token) throw new Error(`task ${taskId}: invalid response token`)
-
-  runtimeDb.prepare(
-    `UPDATE tasks
-       SET status = 'answered',
-           response_json = ?,
-           responded_by  = ?,
-           responded_at  = datetime('now')
-     WHERE id = ?`,
-  ).run(JSON.stringify(response ?? null), respondedBy, taskId)
-
-  return { ...task, status: "answered" }
-}

package/src/instance.ts

@@ -1,64 +0,0 @@
-// 2121
-// Per-instance metadata: what kind of platform-instance this is (which
-// customer, what tier, what teasers to surface), distinct from the runtime
-// state in runtime.db.
-//
-// Lives at <rootDir>/instance.json (per context/instance-spec.md §6).
-// Optional — when missing, sensible defaults are returned (useful for the
-// in-monorepo template-runner where no scaffold has happened).
-//
-// Schema (current, may grow):
-//
-//   {
-//     "platformVersion": "0.1.0",      // recorded at scaffold time
-//     "instanceName":    "Enio Runner", // human-readable label, optional
-//     "tier":            "pilot",      // free-form string, optional
-//     "teasers": [                     // available-but-not-shipped pipelines
-//       {
-//         "id":          "sales-kpi",
-//         "title":       "Salesman KPI tracking",
-//         "description": "Weekly KPI rollup per salesman with conversion tracking.",
-//         "category":    "analytics"
-//       }
-//     ]
-//   }
-
-import { existsSync, readFileSync } from "node:fs"
-import { instanceFilePath } from "./config.ts"
-
-export interface Teaser {
-  id:           string
-  title:        string
-  description?: string
-  category?:    string
-}
-
-export interface Instance {
-  platformVersion?: string
-  instanceName?:    string
-  tier?:            string
-  teasers:          Teaser[]
-}
-
-const DEFAULTS: Instance = { teasers: [] }
-
-// Re-read on every call. instance.json is tiny and changes rarely; the cost
-// is negligible and avoids stale-cache surprises when the file is edited.
-export function loadInstance(): Instance {
-  const path = instanceFilePath()
-  if (!existsSync(path)) return DEFAULTS
-
-  try {
-    const raw = readFileSync(path, "utf8")
-    const parsed = JSON.parse(raw) as Partial<Instance>
-    return {
-      platformVersion: parsed.platformVersion,
-      instanceName:    parsed.instanceName,
-      tier:            parsed.tier,
-      teasers:         Array.isArray(parsed.teasers) ? parsed.teasers : [],
-    }
-  } catch (err) {
-    console.warn(`[instance] failed to read ${path}:`, (err as Error).message)
-    return DEFAULTS
-  }
-}

package/src/kinds/control.ts

@@ -1,26 +0,0 @@
-// 2121
-import type { NodeKind } from "./types.ts"
-
-export const trigger: NodeKind<Record<string, never>, Record<string, never>> = {
-  id: "trigger",
-  category: "control",
-  label: "Trigger",
-  description: "Pipeline entrypoint. Outputs the run payload — reference the run params with $params.X anywhere in the graph.",
-  icon: "Zap",
-  params: {},
-  inputs: {},
-  outputs: { payload: { type: "any" } },
-  run: (_ctx, _params, _input) => null,
-}
-
-export const sink: NodeKind<Record<string, never>, { value: unknown }> = {
-  id: "sink",
-  category: "control",
-  label: "Output",
-  description: "Pipeline output. The value passed in becomes the run result.",
-  icon: "ArrowUpFromLine",
-  params: {},
-  inputs:  { value: { type: "any", required: true } },
-  outputs: {},
-  run: (_ctx, _params, input) => input.value,
-}

package/src/kinds/data.ts

@@ -1,30 +0,0 @@
-// 2121
-import type { NodeKind } from "./types.ts"
-
-export const dataJson: NodeKind<{ value: unknown }, Record<string, never>> = {
-  id: "data.json",
-  category: "data",
-  label: "JSON",
-  description: "Emit a constant JSON value as the output. Useful as a source.",
-  icon: "Braces",
-  params: {
-    value: { type: "json", label: "Value", required: true, description: "Any JSON value (object, array, scalar)" },
-  },
-  inputs: {},
-  outputs: { value: { type: "any" } },
-  run: (_ctx, params) => params.value,
-}
-
-export const dataMerge: NodeKind<{ overlay?: object }, { input: object }> = {
-  id: "data.merge",
-  category: "data",
-  label: "Merge",
-  description: "Shallow-merge runtime input with a constant overlay.",
-  icon: "Combine",
-  params: {
-    overlay: { type: "json", label: "Overlay", default: {} },
-  },
-  inputs:  { input: { type: "object", required: true } },
-  outputs: { output: { type: "object" } },
-  run: (_ctx, params, input) => ({ ...input.input, ...(params.overlay ?? {}) }),
-}

package/src/kinds/db.ts

@@ -1,92 +0,0 @@
-// 2121
-import type { NodeKind } from "./types.ts"
-
-export const dbInsert: NodeKind<
-  { table: string; mode?: "insert" | "upsert"; key?: string | string[] },
-  { rows: unknown[] }
-> = {
-  id: "db.insert",
-  category: "db",
-  label: "DB Insert",
-  description:
-    "Insert (or upsert) array items into a SQLite table. Columns are inferred from the first row. " +
-    "For mode=upsert you must specify `key` (column or column list) — a UNIQUE index is created on " +
-    "first run so subsequent runs replace matching rows instead of duplicating them.",
-  icon: "Database",
-  sideEffect: true,
-  params: {
-    table: { type: "string", label: "Table", required: true, placeholder: "salesman_kpi" },
-    mode:  { type: "select", label: "Mode", default: "insert",
-             options: [
-               { value: "insert", label: "INSERT (append)" },
-               { value: "upsert", label: "UPSERT (replace by key)" },
-             ] },
-    key:   { type: "string", label: "Unique key", description: "Column name (or comma-separated list) for UPSERT mode. The kind creates a UNIQUE index on first run.", placeholder: "id" },
-  },
-  inputs:  { rows: { type: "array", required: true } },
-  outputs: { result: { type: "object" } },
-  run: (ctx, params, input) => {
-    const rows = Array.isArray(input.rows) ? input.rows : []
-    if (rows.length === 0) return { inserted: 0, table: params.table }
-
-    const cols = Object.keys(rows[0] as Record<string, unknown>)
-    if (cols.length === 0) return { inserted: 0, table: params.table }
-
-    const mode = params.mode ?? "insert"
-
-    if (mode === "upsert" && !params.key) {
-      throw new Error(
-        `db.insert: mode "upsert" requires "key" param (column or column list). ` +
-        `Without a UNIQUE index every run would duplicate every row.`,
-      )
-    }
-
-    const colDefs = cols.map((c) => `${c} TEXT`).join(", ")
-    ctx.db.exec(`CREATE TABLE IF NOT EXISTS ${params.table} (${colDefs})`)
-
-    if (mode === "upsert") {
-      const keyCols = Array.isArray(params.key) ? params.key : String(params.key).split(",").map(s => s.trim())
-      const idxName = `idx_${params.table}_${keyCols.join("_")}_unique`
-      try {
-        ctx.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS ${idxName} ON ${params.table} (${keyCols.join(", ")})`)
-      } catch (err) {
-        throw new Error(
-          `db.insert: cannot create UNIQUE index on ${params.table}(${keyCols.join(", ")}) — ` +
-          `existing rows likely violate uniqueness. Drop the table or remove duplicates first. ` +
-          `Underlying error: ${(err as Error).message}`,
-        )
-      }
-    }
-
-    const placeholders = cols.map(() => "?").join(", ")
-    const verb = mode === "upsert" ? "INSERT OR REPLACE" : "INSERT"
-    const stmt = ctx.db.prepare(`${verb} INTO ${params.table} (${cols.join(", ")}) VALUES (${placeholders})`)
-
-    const txn = ctx.db.transaction((items: unknown[]) => {
-      for (const it of items) {
-        const r = it as Record<string, unknown>
-        stmt.run(...cols.map((c) => {
-          const v = r[c]
-          return typeof v === "object" && v !== null ? JSON.stringify(v) : (v as never)
-        }))
-      }
-    })
-    txn(rows)
-
-    return { inserted: rows.length, table: params.table, mode }
-  },
-}
-
-export const dbQuery: NodeKind<{ sql: string }, Record<string, never>> = {
-  id: "db.query",
-  category: "db",
-  label: "DB Query",
-  description: "Run a SELECT query against the runner's SQLite DB.",
-  icon: "Search",
-  params: {
-    sql: { type: "string", label: "SQL", required: true, placeholder: "SELECT * FROM salesman_kpi ORDER BY score DESC LIMIT 10" },
-  },
-  inputs:  {},
-  outputs: { rows: { type: "array" } },
-  run: (ctx, params) => ctx.db.prepare(params.sql).all(),
-}

package/src/kinds/hitl.ts

@@ -1,56 +0,0 @@
-// 2121
-// HITL kinds. The actual suspend/resume bookkeeping lives in ../hitl.ts on
-// the platform side; this file only declares the kind contract.
-
-import type { NodeKind } from "./types.ts"
-
-interface HumanInputParams {
-  prompt:    string
-  assignee?: string
-  schema?:   unknown
-  timeout?:  string
-}
-
-export const humanInput: NodeKind<HumanInputParams, Record<string, never>> = {
-  id: "human.input",
-  category: "control",
-  label: "Human input",
-  description: "Suspends the run until the assignee responds with input. The response becomes this node's output and downstream nodes can $results.<id>.<field> it.",
-  icon: "MessageSquareReply",
-  params: {
-    prompt: {
-      type: "string", required: true,
-      description: "Question shown to the assignee in the task queue.",
-      placeholder: "Approve campaign launch?",
-    },
-    assignee: {
-      type: "string",
-      description: "Who should respond. Format: user:<email> or agent:<id>. Empty = open to anyone.",
-      placeholder: "user:teemu@2121.fi",
-    },
-    schema: {
-      type: "json",
-      description: "JSON Schema describing the expected response shape. v1: descriptive only, not enforced.",
-    },
-    timeout: {
-      type: "string",
-      description: "ISO 8601 duration before the task expires (e.g. \"P7D\"). v1: stored, not enforced.",
-      placeholder: "P7D",
-    },
-  },
-  inputs: {},
-  outputs: {
-    response: {
-      type: "any",
-      description: "Whatever the assignee submitted as their response.",
-    },
-  },
-  run: async (ctx, params) => {
-    return await ctx.suspend({
-      prompt:   params.prompt,
-      assignee: params.assignee,
-      schema:   params.schema,
-      timeout:  params.timeout,
-    })
-  },
-}