@toist/aja 0.5.0

package/CHANGELOG.md

@@ -0,0 +1,69 @@
+# Changelog
+
+All notable changes to `@toist/aja` are recorded here.
+
+## 0.5.0 — 2026-05-05
+
+Lockstep version bump. No functional changes in this package.
+
+## 0.4.0 — 2026-05-05
+
+**Breaking rename:** package was `@toist/run`; it is now `@toist/aja`.
+Update your `package.json` dependency and any `import ... from "@toist/run"`
+statements to `"@toist/aja"`. No API or behaviour changes.
+
+## 0.3.0 — 2026-05-05
+
+Lockstep version bump alongside the new `@toist/up` package. No
+functional changes in this package.
+
+## 0.2.2 — 2026-05-05
+
+**Fix:** UI assets now ship with explicit `Content-Type` headers. The
+previous version relied on `new Response(Bun.file(path))` to carry the
+MIME type through automatically, which works in some hosts but not
+others — observed in published installs (consumer's `node_modules/@toist/ui`)
+the header arrived empty, tripping the browser's strict MIME check on
+JS module scripts (`Failed to load module script: MIME type ""`). The
+runner now sets `Content-Type` from `BunFile.type` explicitly. SPA
+fallback uses the same path.
+
+## 0.2.1 — 2026-05-05
+
+Lockstep version bump alongside `@toist/in@0.2.1`. No functional changes
+in this package.
+
+## 0.2.0 — 2026-05-05
+
+Rebrand: `@2121/runner` → `@toist/aja`. Republished to **npmjs.com**
+(public scope). See `@toist/spec` CHANGELOG for the full rationale.
+
+No API or behaviour changes from 0.1.0; the rename is the only breaking
+change. Workspace deps now reference `@toist/spec` and `@toist/ui`.
+
+## 0.1.0 — 2026-05-05 — abandoned (@2121/runner on Gitea)
+
+Initial release. The pipeline runtime, extracted from the platform
+monorepo per Phase F W2 and shipped to the Gitea npm registry per W3.
+
+- Public API: `startRunner(options): Promise<RunnerHandle>`,
+  `register(...kinds)`, `getKind(id)`, `manifest()`. See instance-spec at
+  https://toist.in for the full contract.
+- HTTP server (Hono on Bun) — mounts API at `/api/*` and serves the
+  bundled UI from `@toist/ui`'s `distDir` at `/`.
+- Pipeline dispatcher with HITL-aware suspendable executor (per-node
+  memoization for resume-skip; tasks queue with single-use response
+  tokens).
+- Kind registry with built-in kinds: `if`, `for`, `try`, `parallel`,
+  `human.input`, `js.eval`, `http.request`, `pipeline.invoke`,
+  `chat.completion`, and platform-internal helpers. Custom kinds register
+  via `register(...)` before `startRunner`.
+- Resource resolution per resource-spec three-tier: ENV > DB > YAML.
+- Persistence: SQLite — `runtime.db` (run ledger, logs, HITL tasks,
+  resources), `cache.db` (TTL'd cache), `data.db` (host-owned domain
+  state). All under `dataDir`.
+- Migration runner: linear, checksum-tracked SQL files in `migrations/`.
+  This release ships a single squashed `001_initial.sql` baseline.
+- StartRunner options: `disableUi`, `disableMcp`, `disableWatch`,
+  `corsOrigins`. MCP server surface is reserved (instance-spec §12);
+  not yet served from this process.

package/migrations/001_initial.sql

@@ -0,0 +1,111 @@
+-- 2121 platform migration 001: initial runtime schema (v0.1 baseline).
+--
+-- Squashed from the pre-publish migrations (initial + HITL + resources). For
+-- the first published version of @2121/runner this is the single baseline a
+-- fresh instance applies on first start. Future migrations stack on top.
+--
+-- All statements use IF NOT EXISTS so a surgical re-baseline (DELETE FROM
+-- _migrations on a populated DB) is a safe no-op against existing schema.
+--
+-- Cache lives in its own cache.db (see cache.ts) and is not migration-managed
+-- — its TTL semantics make drop-recreate safe. Host domain data lives in
+-- data.db, schema-owned by the host repo.
+
+-- ---------------------------------------------------------------------------
+-- runs — the run ledger. Every pipeline invocation gets a row here, kept for
+-- the lifetime of the runtime DB. current_node + updated_at + trigger support
+-- the suspendable executor (HITL pause/resume).
+
+CREATE TABLE IF NOT EXISTS runs (
+  id           INTEGER PRIMARY KEY AUTOINCREMENT,
+  pipeline     TEXT NOT NULL,
+  status       TEXT NOT NULL DEFAULT 'pending',
+  payload      TEXT,
+  result       TEXT,
+  steps        TEXT,
+  error        TEXT,
+  started_at   TEXT DEFAULT (datetime('now')),
+  finished_at  TEXT,
+  current_node TEXT,
+  updated_at   TEXT,
+  trigger      TEXT NOT NULL DEFAULT 'api'
+);
+
+CREATE INDEX IF NOT EXISTS idx_runs_pipeline ON runs(pipeline);
+CREATE INDEX IF NOT EXISTS idx_runs_started  ON runs(started_at DESC);
+
+-- ---------------------------------------------------------------------------
+-- logs — append-only per-run log lines. Cascades on run deletion.
+
+CREATE TABLE IF NOT EXISTS logs (
+  id      INTEGER PRIMARY KEY AUTOINCREMENT,
+  run_id  INTEGER REFERENCES runs(id) ON DELETE CASCADE,
+  ts      TEXT DEFAULT (datetime('now')),
+  level   TEXT DEFAULT 'info',
+  msg     TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_logs_run ON logs(run_id);
+
+-- ---------------------------------------------------------------------------
+-- node_outputs — per-node memoization for resume-skip. On resume, the
+-- executor hydrates results from this table and skips any node that already
+-- has an output recorded; only the suspended node and its successors actually
+-- re-execute.
+
+CREATE TABLE IF NOT EXISTS node_outputs (
+  run_id      INTEGER NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
+  node_id     TEXT    NOT NULL,
+  output_json TEXT,
+  started_at  TEXT    NOT NULL DEFAULT (datetime('now')),
+  finished_at TEXT    NOT NULL DEFAULT (datetime('now')),
+  PRIMARY KEY (run_id, node_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_node_outputs_run ON node_outputs(run_id);
+
+-- ---------------------------------------------------------------------------
+-- tasks — HITL pending-input queue. One row per human.input occurrence in a
+-- run. response_token is single-use (UNIQUE), giving idempotency for retried
+-- resume requests for free. assignee is polymorphic: 'user:<email>' or
+-- 'agent:<id>'.
+
+CREATE TABLE IF NOT EXISTS tasks (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  run_id          INTEGER NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
+  node_id         TEXT    NOT NULL,
+  kind            TEXT    NOT NULL,            -- e.g. 'human.input'
+  prompt          TEXT,
+  schema_json     TEXT,
+  assignee        TEXT,
+  status          TEXT    NOT NULL DEFAULT 'open',  -- open|answered|expired|cancelled
+  response_json   TEXT,
+  response_token  TEXT    NOT NULL UNIQUE,
+  responded_by    TEXT,
+  responded_at    TEXT,
+  created_at      TEXT    NOT NULL DEFAULT (datetime('now')),
+  expires_at      TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_tasks_status   ON tasks(status);
+CREATE INDEX IF NOT EXISTS idx_tasks_run      ON tasks(run_id);
+CREATE INDEX IF NOT EXISTS idx_tasks_assignee ON tasks(assignee);
+
+-- ---------------------------------------------------------------------------
+-- resources — UI/API-managed resource instances. YAML-file and ENV-VAR
+-- resources are not stored here; this table holds only the "UI/API mutation"
+-- tier of the three-tier override chain. Priority: ENV > DB (this table) >
+-- YAML files. Sensitive fields are stored clear-text in v1 with a load-time
+-- warning; encryption at rest is deferred (pipeline-spec.md §16.1).
+
+CREATE TABLE IF NOT EXISTS resources (
+  id          INTEGER PRIMARY KEY AUTOINCREMENT,
+  name        TEXT    NOT NULL UNIQUE,         -- access key in ctx.resource.<name>
+  type        TEXT    NOT NULL,                -- ResourceType name (e.g. "AnthropicApi")
+  fields_json TEXT    NOT NULL DEFAULT '{}',   -- JSON object of field values
+  created_at  TEXT    NOT NULL DEFAULT (datetime('now')),
+  updated_at  TEXT    NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_resources_name ON resources(name);
+CREATE INDEX IF NOT EXISTS idx_resources_type ON resources(type);

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@toist/aja",
+  "version": "0.5.0",
+  "type": "module",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "files": [
+    "src/",
+    "migrations/",
+    "CHANGELOG.md"
+  ],
+  "scripts": {
+    "dev": "bun --watch src/server.ts"
+  },
+  "dependencies": {
+    "@toist/spec": "0.5.0",
+    "@toist/ui": "0.5.0",
+    "hono": "^4.7.7",
+    "proper-lockfile": "^4.1.2"
+  },
+  "devDependencies": {
+    "@types/proper-lockfile": "^4.1.4"
+  }
+}

package/src/cache-db.ts

@@ -0,0 +1,17 @@
+// 2121
+// Cache database — separate file from runtime.db so its drop-recreate
+// semantics (TTL makes content valueless) do not entangle runtime durability.
+//
+// Schema is owned and bootstrapped by makeCache() in cache.ts. No migrations
+// here: cache table can be dropped or recreated freely.
+
+import { Database } from "bun:sqlite"
+import { mkdirSync } from "node:fs"
+import { dirname } from "node:path"
+import { cacheDbPath } from "./config.ts"
+
+export function openCacheDb(): Database {
+  const path = cacheDbPath()
+  mkdirSync(dirname(path), { recursive: true })
+  return new Database(path, { create: true })
+}

package/src/cache.ts

@@ -0,0 +1,67 @@
+// 2121
+// Sqlite-backed cache with TTL. Kindit memoisoivat kalliit fetchit transparentisti
+// pipelinelle: ctx.cache.get/set, key on canonical hash inputeista + paramsista.
+//
+// TTL accepts: "30s" | "5m" | "2h" | "1d" or seconds as number. Default: "1h".
+
+import type { Database } from "bun:sqlite"
+import { createHash } from "node:crypto"
+import type { Cache } from "@toist/spec"
+
+export type { Cache }
+
+const TTL_RE = /^(\d+)(s|m|h|d)$/
+const DEFAULT_TTL_SECONDS = 3600
+
+function ttlToSeconds(ttl: string | number | undefined): number {
+  if (ttl === undefined) return DEFAULT_TTL_SECONDS
+  if (typeof ttl === "number") return ttl
+  const m = TTL_RE.exec(ttl)
+  if (!m) throw new Error(`Invalid TTL: ${ttl}`)
+  const n = Number(m[1])
+  const unit = m[2]
+  return unit === "s" ? n : unit === "m" ? n * 60 : unit === "h" ? n * 3600 : n * 86400
+}
+
+export function makeCache(db: Database): Cache {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS cache (
+      key        TEXT PRIMARY KEY,
+      value      TEXT NOT NULL,
+      expires_at INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_cache_expires ON cache(expires_at);
+  `)
+
+  const get = db.prepare("SELECT value, expires_at FROM cache WHERE key = ?")
+  const set = db.prepare("INSERT OR REPLACE INTO cache (key, value, expires_at) VALUES (?, ?, ?)")
+  const del = db.prepare("DELETE FROM cache WHERE key = ?")
+  const prune = db.prepare("DELETE FROM cache WHERE expires_at < ?")
+
+  return {
+    key(...parts) {
+      const canon = JSON.stringify(parts)
+      return createHash("sha1").update(canon).digest("hex")
+    },
+    get<T>(key: string): T | undefined {
+      const row = get.get(key) as { value: string; expires_at: number } | undefined
+      if (!row) return undefined
+      if (row.expires_at < Date.now()) {
+        del.run(key)
+        return undefined
+      }
+      try { return JSON.parse(row.value) as T } catch { return undefined }
+    },
+    set(key, value, opts) {
+      const expires = Date.now() + ttlToSeconds(opts?.ttl) * 1000
+      set.run(key, JSON.stringify(value), expires)
+    },
+    delete(key) {
+      del.run(key)
+    },
+    prune() {
+      const r = prune.run(Date.now())
+      return Number(r.changes ?? 0)
+    },
+  }
+}

package/src/config.ts

@@ -0,0 +1,129 @@
+// 2121
+// Single source of truth for instance-relative paths. Per
+// context/instance-spec.md §3 + §5:
+//
+//   <rootDir>/
+//     instance.json          — optional metadata (§6)
+//     pipelines/             — pipeline YAML files
+//     resources/             — resource YAML files
+//     data/                  — runner-managed runtime state (gitignored):
+//       data.db              — host's domain data (schema-on-write)
+//       runtime.db           — platform ledger (runs/logs/tasks/node_outputs)
+//       cache.db             — TTL-driven cache
+//       .lock                — proper-lockfile holder (single-leader guard)
+//
+// rootDir resolution order:
+//   1. process.env.PLATFORM_ROOT_DIR       — explicit override (used by hosts
+//                                             that bootstrap @toist/aja from
+//                                             a non-cwd-aligned location)
+//   2. join(process.cwd(), "..")            — fallback matching the historic
+//                                             "server cwd is <root>/server"
+//                                             convention
+//
+// Note: the legacy `<rootDir>/.platform/` directory is platform-internal
+// dev-mode coordination (pm2 port handoff in runtime.json, CLI tooling output
+// in generated/). It is NOT part of the instance contract and is not exposed
+// from this module — start.ts and ecosystem.config.cjs handle it on their own.
+
+import { join } from "node:path"
+
+let rootDirOverride: string | null = null
+let uiDisabled = false
+let watchDisabled = false
+let corsOriginsConfig: string | string[] | null = null
+
+/** Set the rootDir explicitly. Used by `startRunner({rootDir, ...})` to
+ *  configure paths before the db modules' static imports trigger their
+ *  module-load init. Once set, the override wins over PLATFORM_ROOT_DIR
+ *  and the cwd-parent fallback. Calling twice is an error — rootDir is
+ *  once-set per process. */
+export function setRootDir(dir: string): void {
+  if (rootDirOverride !== null && rootDirOverride !== dir) {
+    throw new Error(
+      `[config] setRootDir(${JSON.stringify(dir)}) called after rootDir ` +
+      `was already set to ${JSON.stringify(rootDirOverride)}. ` +
+      `rootDir is once-set per process.`,
+    )
+  }
+  rootDirOverride = dir
+}
+
+/** Set the disableUi flag. Used by `startRunner({disableUi: true, ...})` to
+ *  opt out of the bundled static-UI mount before server.ts's outer app
+ *  construction reads the flag. Default: UI is enabled when the dist/
+ *  directory in @toist/ui exists. */
+export function setDisableUi(v: boolean): void {
+  uiDisabled = v
+}
+
+export function isUiDisabled(): boolean {
+  return uiDisabled
+}
+
+/** Set the disableWatch flag. Used by `startRunner({disableWatch: true, ...})`
+ *  to opt out of the filesystem watcher on pipelineDir/resourceDir. Default:
+ *  watcher is enabled. Production deployments may set true to avoid
+ *  inotify/fsevent overhead and accidental hot-reloads. */
+export function setDisableWatch(v: boolean): void {
+  watchDisabled = v
+}
+
+export function isWatchDisabled(): boolean {
+  return watchDisabled
+}
+
+/** Set the CORS allow-origin configuration. `string` for a single origin,
+ *  `string[]` for an allowlist, `null` for default (permissive `*`).
+ *  Per Hono's cors() middleware semantics. */
+export function setCorsOrigins(origins: string | string[] | null | undefined): void {
+  corsOriginsConfig = origins ?? null
+}
+
+export function getCorsOrigins(): string | string[] | null {
+  return corsOriginsConfig
+}
+
+export function resolveRootDir(): string {
+  if (rootDirOverride !== null) return rootDirOverride
+  const override = process.env.PLATFORM_ROOT_DIR
+  if (override && override.length > 0) return override
+  return join(process.cwd(), "..")
+}
+
+export function dataDir(): string {
+  const override = process.env.PLATFORM_DATA_DIR
+  if (override && override.length > 0) return override
+  return join(resolveRootDir(), "data")
+}
+
+export function dataDbPath(): string {
+  return join(dataDir(), "data.db")
+}
+
+export function runtimeDbPath(): string {
+  return join(dataDir(), "runtime.db")
+}
+
+export function cacheDbPath(): string {
+  return join(dataDir(), "cache.db")
+}
+
+export function lockfilePath(): string {
+  return join(dataDir(), ".lock")
+}
+
+export function instanceFilePath(): string {
+  return join(resolveRootDir(), "instance.json")
+}
+
+export function pipelinesDir(): string {
+  const override = process.env.PIPELINES_DIR
+  if (override && override.length > 0) return override
+  return join(resolveRootDir(), "pipelines")
+}
+
+export function resourcesDir(): string {
+  const override = process.env.RESOURCES_DIR
+  if (override && override.length > 0) return override
+  return join(resolveRootDir(), "resources")
+}

package/src/data-db.ts

@@ -0,0 +1,21 @@
+// 2121
+// Domain data store. The instance owns the schema entirely — every table here
+// is created by a pipeline (via db.insert) or by a domain migration the project
+// chooses to add. Platform never writes to this DB directly.
+//
+// Kinds receive this as `ctx.db`. Pipelines write to it (db.insert), read
+// from it (db.query), or any custom domain kind interacts with it as needed.
+//
+// Location: <dataDir>/data.db (per context/instance-spec.md §3 — the host's
+// schema-on-write domain DB lives in data/ alongside runtime.db and cache.db).
+
+import { Database } from "bun:sqlite"
+import { mkdirSync } from "node:fs"
+import { dirname } from "node:path"
+import { dataDbPath } from "./config.ts"
+
+export function openDataDb(): Database {
+  const path = dataDbPath()
+  mkdirSync(dirname(path), { recursive: true })
+  return new Database(path, { create: true })
+}

package/src/db-handles.ts

@@ -0,0 +1,70 @@
+// 2121
+// Singleton db handle holder + lifecycle orchestration. Decouples the db
+// modules (which are now pure factories per `openX()`) from the consumers
+// (index.ts, pipeline.ts) that need stable handle references throughout the
+// HTTP request lifetime.
+//
+// Lifecycle (matches context/instance-spec.md §7):
+//
+//   initDbs()     — acquire runner lock, migrate runtime.db, open all three
+//                   handles. Idempotent: second call is a no-op when handles
+//                   are already populated.
+//   runtimeDb()   — getter; throws when init has not run
+//   dataDb()      — getter; throws when init has not run
+//   cacheDb()     — getter; throws when init has not run
+//   closeDbs()    — close all three handles, release the lock. Used by
+//                   `startRunner`'s graceful stop path.
+//
+// Why getters and not exported instances: the db modules used to do their
+// init at module load via top-level await, which forced lifecycle to happen
+// during the import graph. That works for source-mod single-instance runs
+// but breaks the moment startRunner() needs to set rootDir before init runs.
+// Getters defer the resolution to call time; init runs once, before any
+// HTTP request fires.
+
+import type { Database } from "bun:sqlite"
+import { openRuntimeDb } from "./runtime-db.ts"
+import { openDataDb } from "./data-db.ts"
+import { openCacheDb } from "./cache-db.ts"
+import { acquireRunnerLock, releaseRunnerLock } from "./lock.ts"
+
+let _runtimeDb: Database | null = null
+let _dataDb: Database | null = null
+let _cacheDb: Database | null = null
+
+export async function initDbs(): Promise<void> {
+  if (_runtimeDb !== null) return  // idempotent
+
+  await acquireRunnerLock()
+  _runtimeDb = openRuntimeDb()
+  _dataDb = openDataDb()
+  _cacheDb = openCacheDb()
+}
+
+export async function closeDbs(): Promise<void> {
+  if (_runtimeDb) { _runtimeDb.close(); _runtimeDb = null }
+  if (_dataDb)    { _dataDb.close();    _dataDb = null }
+  if (_cacheDb)   { _cacheDb.close();   _cacheDb = null }
+  await releaseRunnerLock()
+}
+
+export function runtimeDb(): Database {
+  if (_runtimeDb === null) {
+    throw new Error("[db-handles] runtimeDb() called before initDbs() — lifecycle violation")
+  }
+  return _runtimeDb
+}
+
+export function dataDb(): Database {
+  if (_dataDb === null) {
+    throw new Error("[db-handles] dataDb() called before initDbs() — lifecycle violation")
+  }
+  return _dataDb
+}
+
+export function cacheDb(): Database {
+  if (_cacheDb === null) {
+    throw new Error("[db-handles] cacheDb() called before initDbs() — lifecycle violation")
+  }
+  return _cacheDb
+}