@toist/aja 0.5.0

package/src/kinds/http.ts

@@ -0,0 +1,134 @@
+// 2121
+// Generic HTTP kinds. Together with transform.* and db.*, these let pipelines
+// talk to any external API without writing domain-specific kinds first —
+// critical for `pipelines.create` from natural-language conversations.
+
+import type { NodeKind } from "./types.ts"
+
+interface FetchParams {
+  url: string
+  headers?: Record<string, string>
+  cache?: string                 // TTL string, e.g. "5m" — enables transparent caching
+  parse?: "json" | "text" | "auto"  // default "auto": parse JSON if Content-Type matches
+  timeoutMs?: number
+}
+
+async function doFetch(
+  url: string,
+  init: RequestInit,
+  parse: FetchParams["parse"],
+  timeoutMs?: number,
+): Promise<{ status: number; headers: Record<string, string>; body: unknown }> {
+  const controller = timeoutMs != null ? new AbortController() : undefined
+  const timer = controller && timeoutMs != null
+    ? setTimeout(() => controller.abort(), timeoutMs)
+    : undefined
+  try {
+    const r = await fetch(url, { ...init, signal: controller?.signal })
+    const headers: Record<string, string> = {}
+    r.headers.forEach((v, k) => { headers[k] = v })
+
+    const ct = r.headers.get("content-type") ?? ""
+    const wantJson = parse === "json" || (parse === "auto" && ct.includes("application/json"))
+    const body = wantJson ? await r.json() : await r.text()
+    return { status: r.status, headers, body }
+  } finally {
+    if (timer) clearTimeout(timer)
+  }
+}
+
+export const httpFetch: NodeKind<FetchParams, Record<string, never>> = {
+  id: "http.fetch",
+  category: "io",
+  label: "HTTP GET",
+  description: "Fetch a URL and return the parsed body. Optionally cached transparently with TTL.",
+  icon: "Globe",
+  params: {
+    url:       { type: "string", label: "URL", required: true, placeholder: "https://api.example.com/v1/items" },
+    headers:   { type: "json", label: "Headers", description: "Object of header name → value" },
+    cache:     { type: "string", label: "Cache TTL", description: "e.g. '5m', '1h'. Omit to disable caching.", placeholder: "5m" },
+    parse:     { type: "select", label: "Parse mode", default: "auto",
+                 options: [
+                   { value: "auto", label: "Auto (by Content-Type)" },
+                   { value: "json", label: "Always JSON" },
+                   { value: "text", label: "Plain text" },
+                 ] },
+    timeoutMs: { type: "number", label: "Timeout (ms)", default: 10000 },
+  },
+  inputs: {},
+  outputs: {
+    status:  { type: "number" },
+    headers: { type: "object" },
+    body:    { type: "any" },
+  },
+  run: async (ctx, params) => {
+    const parseMode = params.parse ?? "auto"
+
+    if (params.cache) {
+      const key = ctx.cache.key("http.fetch", params.url, params.headers ?? {}, parseMode)
+      const hit = ctx.cache.get<unknown>(key)
+      if (hit) {
+        ctx.log("info", `cache hit ${params.url}`)
+        return hit
+      }
+      const result = await doFetch(params.url, {
+        method: "GET",
+        headers: params.headers,
+      }, parseMode, params.timeoutMs)
+      ctx.cache.set(key, result, { ttl: params.cache })
+      return result
+    }
+
+    return doFetch(params.url, {
+      method: "GET",
+      headers: params.headers,
+    }, parseMode, params.timeoutMs)
+  },
+}
+
+interface PostParams {
+  url: string
+  headers?: Record<string, string>
+  parse?: "json" | "text" | "auto"
+  timeoutMs?: number
+}
+
+interface PostInput {
+  body: unknown
+}
+
+export const httpPost: NodeKind<PostParams, PostInput> = {
+  id: "http.post",
+  category: "io",
+  label: "HTTP POST",
+  description: "POST a JSON body to a URL and return the parsed response. Side-effecting — requires confirm in ad-hoc invokes.",
+  icon: "Send",
+  sideEffect: true,
+  params: {
+    url:       { type: "string", label: "URL", required: true, placeholder: "https://api.example.com/v1/items" },
+    headers:   { type: "json", label: "Headers" },
+    parse:     { type: "select", label: "Parse mode", default: "auto",
+                 options: [
+                   { value: "auto", label: "Auto (by Content-Type)" },
+                   { value: "json", label: "Always JSON" },
+                   { value: "text", label: "Plain text" },
+                 ] },
+    timeoutMs: { type: "number", label: "Timeout (ms)", default: 10000 },
+  },
+  inputs: {
+    body: { type: "any", required: true, label: "Request body (JSON-serialised)" },
+  },
+  outputs: {
+    status:  { type: "number" },
+    headers: { type: "object" },
+    body:    { type: "any" },
+  },
+  run: async (ctx, params, input) => {
+    ctx.log("info", `POST ${params.url}`)
+    return doFetch(params.url, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", ...(params.headers ?? {}) },
+      body: JSON.stringify(input.body),
+    }, params.parse ?? "auto", params.timeoutMs)
+  },
+}

package/src/kinds/index.ts

@@ -0,0 +1,66 @@
+// 2121
+import type { NodeKind, NodeKindManifest } from "./types.ts"
+import { dataJson, dataMerge } from "./data.ts"
+import { transformFilter, transformMap, transformSort, transformAggregate } from "./transform.ts"
+import { dbInsert, dbQuery } from "./db.ts"
+import { httpFetch, httpPost } from "./http.ts"
+import { trigger, sink } from "./control.ts"
+import { humanInput } from "./hitl.ts"
+import { runsLastOutput, runsNodeOutput, pipelineRun } from "./runs.ts"
+
+const builtins: NodeKind<any, any>[] = [
+  trigger,
+  sink,
+  dataJson,
+  dataMerge,
+  transformFilter,
+  transformMap,
+  transformSort,
+  transformAggregate,
+  dbInsert,
+  dbQuery,
+  httpFetch,
+  httpPost,
+  humanInput,
+  runsLastOutput,
+  runsNodeOutput,
+  pipelineRun,
+]
+
+export const registry: Map<string, NodeKind<any, any>> = new Map(builtins.map((k) => [k.id, k]))
+
+/**
+ * Register one or more custom domain kinds. Domain projects edit
+ * `runner/src/kinds/custom.ts` (created by `platform init`) and call
+ * `register(...)` there with their own kinds. Builtin kinds and domain kinds
+ * live in the same registry — `getKind(id)` resolves both transparently.
+ *
+ * Re-registering an existing id replaces the previous entry; useful for hot
+ * iteration but warn loudly so accidental shadows are visible.
+ */
+export function register(...kinds: NodeKind<any, any>[]): void {
+  for (const k of kinds) {
+    if (registry.has(k.id) && !builtins.find((b) => b.id === k.id)) {
+      console.warn(`[kinds] re-registering "${k.id}" — previous registration replaced`)
+    }
+    registry.set(k.id, k)
+  }
+}
+
+export function getKind(id: string): NodeKind<any, any> | undefined {
+  return registry.get(id)
+}
+
+export function manifest(): NodeKindManifest[] {
+  return [...registry.values()].map(({ run: _e, ...rest }) => rest)
+}
+
+export type { NodeKind, NodeKindManifest, ParamDef, PortDef, ExecContext, PlatformCtx } from "./types.ts"
+
+// Side-effect import: domain kinds register themselves when this module loads.
+// Dynamic import + top-level await defers custom.ts evaluation until AFTER
+// `registry` and `register` are initialised — a static `import "./custom.ts"`
+// gets hoisted ahead of the const init, putting `registry` in the temporal
+// dead zone when custom.ts calls register(). The dynamic form runs at
+// statement position, after init, breaking the cycle.
+await import("./custom.ts")

package/src/kinds/runs.ts

@@ -0,0 +1,130 @@
+// 2121
+// Cross-pipeline composition kinds. Per pipeline-spec.md §12 (M2, M3, M4):
+//
+//   pipeline.run    — invoke another pipeline as a sub-run (M2)
+//   runs.lastOutput — final output of the most recent run of a pipeline (M3)
+//   runs.nodeOutput — output of a specific node from the most recent run (M4)
+//
+// Read kinds go through ctx.runs (RunStore capability); the sub-run kind
+// goes through ctx.subRun (SubRun capability). Neither sees the runtime
+// DB directly — the surface stays narrow and immutable.
+
+import type { NodeKind } from "./types.ts"
+
+export const runsLastOutput: NodeKind<
+  { pipeline: string; status?: string },
+  Record<string, never>
+> = {
+  id: "runs.lastOutput",
+  category: "io",
+  label: "Last run output",
+  description:
+    "Read the final output of the most recent run of another pipeline. " +
+    "Defaults to runs with status \"done\"; pass status to read failed/suspended runs " +
+    "(useful for monitoring or replay flows). Returns null when no matching run exists.",
+  icon: "History",
+  params: {
+    pipeline: { type: "string", label: "Pipeline", required: true, placeholder: "callpool" },
+    status:   { type: "select", label: "Status", default: "done",
+                options: [
+                  { value: "done",      label: "done" },
+                  { value: "failed",    label: "failed" },
+                  { value: "suspended", label: "suspended" },
+                ] },
+  },
+  inputs:  {},
+  outputs: { value: { type: "any" } },
+  run: (ctx, params) => ctx.runs.lastOutput(params.pipeline, { status: params.status ?? "done" }),
+}
+
+export const runsNodeOutput: NodeKind<
+  { pipeline: string; node: string; status?: string },
+  Record<string, never>
+> = {
+  id: "runs.nodeOutput",
+  category: "io",
+  label: "Last run node output",
+  description:
+    "Read the output of a specific node from the most recent run of another pipeline. " +
+    "Backed by the same node_outputs table that powers HITL resume — useful for " +
+    "consuming intermediate results without rerunning upstream work.",
+  icon: "GitBranch",
+  params: {
+    pipeline: { type: "string", label: "Pipeline", required: true, placeholder: "callpool" },
+    node:     { type: "string", label: "Node id",  required: true, placeholder: "buildings" },
+    status:   { type: "select", label: "Status", default: "done",
+                options: [
+                  { value: "done",      label: "done" },
+                  { value: "failed",    label: "failed" },
+                  { value: "suspended", label: "suspended" },
+                ] },
+  },
+  inputs:  {},
+  outputs: { value: { type: "any" } },
+  run: (ctx, params) => ctx.runs.nodeOutput(params.pipeline, params.node, { status: params.status ?? "done" }),
+}
+
+// ─── pipeline.run — sub-pipeline invocation (M2) ─────────────────────────────
+
+export const pipelineRun: NodeKind<
+  { pipeline: string; onFailure?: "propagate" | "capture" },
+  { payload: Record<string, unknown> }
+> = {
+  id: "pipeline.run",
+  category: "control",
+  label: "Sub-pipeline",
+  description:
+    "Invoke another pipeline as a sub-run. Each sub-run gets its own run-id, " +
+    "its own node_outputs, and its own lifecycle. The output flows back as the " +
+    "kind's value on success. v1 ships two onFailure modes: `propagate` (default — " +
+    "sub-run failure raises in the parent) and `capture` (failure becomes the " +
+    "node's output as { status, error, partialResults, runId } so downstream can " +
+    "branch on shape). `suspend` lands with the recovery model implementation.",
+  icon: "Workflow",
+  sideEffect: true,
+  params: {
+    pipeline:  { type: "string", label: "Pipeline id", required: true, placeholder: "callpool" },
+    onFailure: { type: "select", label: "On failure", default: "propagate",
+                 options: [
+                   { value: "propagate", label: "propagate (parent's onError takes over)" },
+                   { value: "capture",   label: "capture (failure becomes node output)" },
+                 ] },
+  },
+  inputs:  { payload: { type: "object", label: "Payload", required: true } },
+  outputs: { value:   { type: "any" } },
+  run: async (ctx, params, input) => {
+    const onFailure = params.onFailure ?? "propagate"
+    const outcome = await ctx.subRun(params.pipeline, input.payload)
+
+    if (outcome.status === "done") return outcome.output
+
+    if (outcome.status === "suspended") {
+      // Sub-run is parked on a HITL task. Surface the suspension as
+      // structured output regardless of onFailure — the sub-run isn't
+      // failed (there's no error), it's waiting. Cross-nesting resume is
+      // §16.1 work; for v1 this signals "nothing to consume yet."
+      return {
+        __subRun: {
+          status: "suspended",
+          runId: outcome.runId,
+          suspendedAt: outcome.suspendedAt,
+          partialResults: outcome.partialResults,
+        },
+      }
+    }
+
+    // status === "failed"
+    if (onFailure === "capture") {
+      return {
+        __subRun: {
+          status: "failed",
+          runId: outcome.runId,
+          error: outcome.error,
+          partialResults: outcome.partialResults,
+        },
+      }
+    }
+    // propagate (default): re-throw so the parent's onError abort policy applies
+    throw new Error(`sub-run "${params.pipeline}" (run ${outcome.runId}) failed: ${outcome.error}`)
+  },
+}

package/src/kinds/transform.ts

@@ -0,0 +1,123 @@
+// 2121
+import type { NodeKind } from "./types.ts"
+
+// Per pipeline-spec v1: `{ expr: "..." }` is evaluated by the runner before
+// the kind sees it, so a kind that needs a callable receives the function
+// directly. The pipeline author writes:
+//
+//   condition: { expr: "(item) => item.score >= 0.5" }
+//   expression: { expr: "(item) => ({ ...item, doubled: item.x * 2 })" }
+//
+// The discriminator resolves these to actual function values; the kind just
+// invokes them per array item.
+
+type ItemFn<R> = (item: unknown, input: unknown) => R
+
+function asFn<R>(v: unknown, kindId: string, paramName: string): ItemFn<R> {
+  if (typeof v !== "function") {
+    throw new Error(
+      `${kindId}: params.${paramName} must be a function ` +
+      `(authored as { expr: "(item) => ..." }); got ${v === null ? "null" : typeof v}`,
+    )
+  }
+  return v as ItemFn<R>
+}
+
+export const transformFilter: NodeKind<{ condition: unknown }, { items: unknown[] }> = {
+  id: "transform.filter",
+  category: "transform",
+  label: "Filter",
+  description: "Keep array items matching a boolean predicate.",
+  icon: "Filter",
+  params: {
+    condition: { type: "expression", label: "Condition", required: true, placeholder: "(item) => item.score >= 0.5" },
+  },
+  inputs:  { items: { type: "array", required: true } },
+  outputs: { items: { type: "array" } },
+  run: (_ctx, params, input) => {
+    const items = Array.isArray(input.items) ? input.items : []
+    const fn = asFn<boolean>(params.condition, "transform.filter", "condition")
+    return items.filter((item) => !!fn(item, input))
+  },
+}
+
+export const transformMap: NodeKind<{ expression: unknown }, { items: unknown[] }> = {
+  id: "transform.map",
+  category: "transform",
+  label: "Map",
+  description: "Apply a function to each array item, producing a new item.",
+  icon: "MoveRight",
+  params: {
+    expression: { type: "expression", label: "Expression", required: true, placeholder: "(item) => ({ ...item, score: item.signals / 20 })" },
+  },
+  inputs:  { items: { type: "array", required: true } },
+  outputs: { items: { type: "array" } },
+  run: (_ctx, params, input) => {
+    const items = Array.isArray(input.items) ? input.items : []
+    const fn = asFn<unknown>(params.expression, "transform.map", "expression")
+    return items.map((item) => fn(item, input))
+  },
+}
+
+export const transformSort: NodeKind<{ by: string; direction?: "asc" | "desc" }, { items: unknown[] }> = {
+  id: "transform.sort",
+  category: "transform",
+  label: "Sort",
+  description: "Sort an array by a field, ascending or descending.",
+  icon: "ArrowDownAZ",
+  params: {
+    by:        { type: "string", label: "Sort by", required: true, placeholder: "score" },
+    direction: { type: "select", label: "Direction", default: "desc",
+                 options: [{ value: "asc", label: "Ascending" }, { value: "desc", label: "Descending" }] },
+  },
+  inputs:  { items: { type: "array", required: true } },
+  outputs: { items: { type: "array" } },
+  run: (_ctx, params, input) => {
+    const items = Array.isArray(input.items) ? [...input.items] : []
+    const dir = params.direction === "asc" ? 1 : -1
+    items.sort((a, b) => {
+      const av = (a as Record<string, unknown>)[params.by]
+      const bv = (b as Record<string, unknown>)[params.by]
+      if ((av as number) < (bv as number)) return -1 * dir
+      if ((av as number) > (bv as number)) return  1 * dir
+      return 0
+    })
+    return items
+  },
+}
+
+export const transformAggregate: NodeKind<
+  { op: "count" | "sum" | "avg" | "min" | "max"; key?: string },
+  { items: unknown[] }
+> = {
+  id: "transform.aggregate",
+  category: "transform",
+  label: "Aggregate",
+  description: "Reduce an array to a single value (count, sum, avg, min, max).",
+  icon: "Sigma",
+  params: {
+    op:    { type: "select", label: "Operation", required: true, default: "count",
+             options: [
+               { value: "count", label: "Count" },
+               { value: "sum",   label: "Sum"   },
+               { value: "avg",   label: "Average" },
+               { value: "min",   label: "Min" },
+               { value: "max",   label: "Max" },
+             ] },
+    key:   { type: "string", label: "Numeric key", description: "Field to aggregate (required for sum/avg/min/max)", placeholder: "score" },
+  },
+  inputs:  { items: { type: "array", required: true } },
+  outputs: { value: { type: "number" } },
+  run: (_ctx, params, input) => {
+    const items = Array.isArray(input.items) ? input.items : []
+    if (params.op === "count") return items.length
+    if (!params.key) throw new Error(`aggregate.${params.op} requires "key"`)
+    const nums = items.map((it) => Number((it as Record<string, unknown>)[params.key!])).filter(Number.isFinite)
+    if (nums.length === 0) return 0
+    if (params.op === "sum") return nums.reduce((a, b) => a + b, 0)
+    if (params.op === "avg") return nums.reduce((a, b) => a + b, 0) / nums.length
+    if (params.op === "min") return Math.min(...nums)
+    if (params.op === "max") return Math.max(...nums)
+    return 0
+  },
+}

package/src/kinds/types.ts

@@ -0,0 +1,16 @@
+// 2121
+// Backwards-compat re-export shim. Kind contract types now live in
+// `@toist/spec`. Domain projects editing `kinds/custom.ts` can keep
+// importing from `./types.ts`; new runner-internal code should import
+// from `@toist/spec` directly.
+
+export type {
+  ParamDef,
+  PortDef,
+  Cache,
+  HitlSpec,
+  PlatformCtx,
+  ExecContext,
+  NodeKind,
+  NodeKindManifest,
+} from "@toist/spec"

package/src/lock.ts

@@ -0,0 +1,64 @@
+// 2121
+// Single-leader runner lock. Prevents two runner processes from racing on
+// migrations, scheduled cron-wheel firings, or HITL resume dispatch.
+//
+// The lock guards the data/ directory using a sentinel lockfile at
+// data/.lock. proper-lockfile creates a holder directory that includes mtime;
+// stale locks (from a crashed previous process) are detected after `stale` ms
+// and overtaken automatically.
+//
+// Watch-mode reload tolerance: bun --watch SIGTERMs the old process before
+// spawning the new one. The cleanup handlers below release the lock on
+// SIGTERM/SIGINT/exit. Async lock acquisition with retries handles the brief
+// window where the old process has not yet released.
+
+import lockfile from "proper-lockfile"
+import { mkdirSync } from "node:fs"
+import { dataDir, lockfilePath } from "./config.ts"
+
+let release: (() => Promise<void>) | null = null
+
+export async function acquireRunnerLock(): Promise<void> {
+  const dir = dataDir()
+  const lockPath = lockfilePath()
+  mkdirSync(dir, { recursive: true })
+
+  try {
+    release = await lockfile.lock(dir, {
+      lockfilePath: lockPath,
+      realpath: false,
+      stale: 10_000,
+      retries: { retries: 5, factor: 2, minTimeout: 100, maxTimeout: 1000 },
+    })
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err)
+    console.error(`[runner] failed to acquire ${lockPath}: ${msg}`)
+    console.error(`[runner] another runner appears live. Stop it first; only one runner per project is allowed.`)
+    process.exit(1)
+  }
+
+  const cleanup = (): void => {
+    if (!release) return
+    const r = release
+    release = null
+    r().catch(() => { /* best-effort */ })
+  }
+
+  process.on("exit",    cleanup)
+  process.on("SIGTERM", () => { cleanup(); process.exit(0) })
+  process.on("SIGINT",  () => { cleanup(); process.exit(0) })
+}
+
+/** Release the runner lock explicitly. Used by `startRunner`'s graceful
+ *  `stop()` path. Idempotent: no-op if the lock is not held. The signal
+ *  cleanup handlers installed by acquireRunnerLock remain in place. */
+export async function releaseRunnerLock(): Promise<void> {
+  if (!release) return
+  const r = release
+  release = null
+  try {
+    await r()
+  } catch {
+    // best-effort — surfacing this would mask the actual stop reason
+  }
+}