@toist/aja 0.6.1 → 0.8.0

package/CHANGELOG.md

@@ -2,6 +2,62 @@
 
 All notable changes to `@toist/aja` are recorded here.
 
+## 0.8.0 — 2026-05-08
+
+Major refactor: `@toist/aja` is now the SQLite host package, with the
+execution kernel extracted to `@toist/core`.
+
+**New:**
+
+- `createSqliteRuntime({ rootDir, ... })` — durable runtime constructor.
+  Owns lock acquisition, migrations, db handle opening, store wiring,
+  registry projection, resource resolution. Returns a `ToistRuntime`
+  with `close()`.
+- `mountApi(runtime, options): Hono` factory replaces module-level
+  state in `server.ts`. `startRunner` builds a runtime then calls
+  `mountApi`.
+- `loadFilesystemResources(rootDir)` — reads `toist.yml`, performs
+  `${env:VAR}` and `${env:VAR:-default}` substitution, fails closed
+  at config-load time on missing required vars.
+- `kindAllowlist` field in `instance.json` filters the registry
+  projection at construction time. Glob support (`agent.*`).
+- `StartRunnerOptions.storage: "sqlite" | "memory"` flag.
+- `StartRunnerOptions.runtime?: ToistRuntime` late-bound injection.
+- `HostedRuntime` interface (extends `ToistRuntime` with optional
+  `close`, `adminDb`, `pipelinesDir`).
+- `createFilesystemPipelineStore({ rootDir, registry, watch })` for
+  YAML pipeline loading + watching.
+- `POST /api/runs` async ad-hoc spec-run endpoint.
+- `GET /api/runs/:id/events` SSE event stream.
+- `client.pipelines.runSpec(spec, payload)` and `client.runs.events(runId)`.
+- `/manifest` response carries `version` + `schema` URL for stable
+  downstream consumption.
+
+**Removed:**
+
+- `db-handles.ts`, `config.ts` (process-global `setRootDir` etc.),
+  `runtime.ts` (`createProcessRuntime`). Replaced by explicit
+  `createSqliteRuntime`.
+- Local `pipeline.ts`, `hitl.ts`, `runs.ts`, `kinds/{control,data,db,hitl,
+  http,runs,transform,types}.ts` — all moved to `@toist/core`.
+  `@toist/aja` re-exports the kernel surface.
+- Filesystem `resources/*.yaml` loader (replaced by `toist.yml`).
+
+**Renamed:**
+
+- `instance.ts` → `instance-metadata.ts` (frees the name for a possible
+  future `@toist/instance` package).
+
+**New dependency:** `@toist/core@0.8.0`.
+
+## 0.7.1 — 2026-05-06
+
+Fix @toist/aja cross-package dependency versions not being bumped by release script
+
+## 0.7.0 — 2026-05-06
+
+Add @toist/aja CLI (bunx @toist/aja --config toist.yml) and @toist/in interactive setup wizard (bunx @toist/in). Establishes aja=runner, in=adoption tool split. No forced directory layout — user chooses paths via wizard, config drives the runner.
+
 ## Unreleased
 
 ## 0.6.1 — 2026-05-05
@@ -69,7 +125,7 @@ monorepo per Phase F W2 and shipped to the Gitea npm registry per W3.
 
 - Public API: `startRunner(options): Promise<RunnerHandle>`,
   `register(...kinds)`, `getKind(id)`, `manifest()`. See instance-spec at
-  https://toist.in for the full contract.
+  https://toist.in/docs for the full contract.
 - HTTP server (Hono on Bun) — mounts API at `/api/*` and serves the
   bundled UI from `@toist/ui`'s `distDir` at `/`.
 - Pipeline dispatcher with HITL-aware suspendable executor (per-node

package/README.md

@@ -0,0 +1,62 @@
+# `@toist/aja`
+
+Durable Toist host surface: SQLite runtime, HTTP runner, UI mount, filesystem helpers, and the typed runner client.
+
+```ts
+import { startRunner } from "@toist/aja"
+
+await startRunner({
+  rootDir: import.meta.dir,
+  port: 3000,
+})
+```
+
+## Main exports
+
+- `startRunner(options)`
+- `createSqliteRuntime(options)`
+- `createFilesystemPipelineStore({ rootDir, watch? })`
+- `loadFilesystemResources(rootDir)`
+- `createRunnerClient({ baseUrl })`
+- convenience re-exports from `@toist/core`
+
+## Client quickstart
+
+```ts
+import { createRunnerClient } from "@toist/aja"
+
+const client = createRunnerClient({ baseUrl: "http://localhost:3000" })
+const started = await client.pipelines.runSpec(spec, { name: "world" })
+for await (const event of client.runs.events(started.id)) {
+  console.log(event.type)
+}
+```
+
+## Runtime layout
+
+With `startRunner({ rootDir, port })`, the runner:
+
+- loads pipelines from `<rootDir>/pipelines`
+- stores runtime state under `<rootDir>/data`
+- reads resources from `<rootDir>/toist.yml`
+- serves UI + API from one Bun process
+
+## `/manifest`
+
+`GET /api/manifest` returns stable JSON for tooling:
+
+```json
+{
+  "version": 1,
+  "schema": "https://toist.in/schemas/manifest-v1.json",
+  "kinds": []
+}
+```
+
+## Resources
+
+`loadFilesystemResources(rootDir)` reads `toist.yml` and resolves `${env:VAR}` and `${env:VAR:-fallback}` substitutions.
+
+## When to use `@toist/aja`
+
+Use it when you want a durable single-tenant runner with HTTP APIs. For in-process execution without the server, import `@toist/core` directly.

package/package.json

@@ -1,9 +1,12 @@
 {
   "name": "@toist/aja",
-  "version": "0.6.1",
+  "version": "0.8.0",
   "type": "module",
   "main": "./src/index.ts",
   "types": "./src/index.ts",
+  "bin": {
+    "toist-aja": "./src/cli.ts"
+  },
   "exports": {
     ".": "./src/index.ts",
     "./client": "./src/client.ts"
@@ -11,17 +14,20 @@
   "files": [
     "src/",
     "migrations/",
-    "CHANGELOG.md"
+    "CHANGELOG.md",
+    "README.md"
   ],
   "scripts": {
     "dev": "bun --watch src/server.ts",
     "smoke:client": "bun test/client-smoke.ts"
   },
   "dependencies": {
-    "@toist/spec": "0.6.1",
-    "@toist/ui": "0.6.1",
+    "@toist/core": "0.8.0",
+    "@toist/spec": "0.8.0",
+    "@toist/ui": "0.8.0",
     "hono": "^4.7.7",
-    "proper-lockfile": "^4.1.2"
+    "proper-lockfile": "^4.1.2",
+    "yaml": "^2.8.4"
   },
   "devDependencies": {
     "@types/proper-lockfile": "^4.1.4"

package/src/cache-db.ts

@@ -1,17 +1,9 @@
 // 2121
-// Cache database — separate file from runtime.db so its drop-recreate
-// semantics (TTL makes content valueless) do not entangle runtime durability.
-//
-// Schema is owned and bootstrapped by makeCache() in cache.ts. No migrations
-// here: cache table can be dropped or recreated freely.
-
 import { Database } from "bun:sqlite"
 import { mkdirSync } from "node:fs"
 import { dirname } from "node:path"
-import { cacheDbPath } from "./config.ts"
 
-export function openCacheDb(): Database {
-  const path = cacheDbPath()
+export function openCacheDb(path: string): Database {
   mkdirSync(dirname(path), { recursive: true })
   return new Database(path, { create: true })
 }

package/src/cli.ts

@@ -0,0 +1,153 @@
+#!/usr/bin/env bun
+// 2121 toist
+// @toist/aja CLI — start a Toist runner from a config file.
+//
+// Usage:
+//   bunx @toist/aja                       auto-discover toist.yml in cwd
+//   bunx @toist/aja --config <path>       explicit config file
+
+import { existsSync, readFileSync } from "node:fs"
+import { dirname, isAbsolute, resolve } from "node:path"
+import YAML from "yaml"
+import { createSqliteRuntime, register, startRunner } from "./index.ts"
+
+// ---------------------------------------------------------------------------
+// Config schema
+// ---------------------------------------------------------------------------
+
+interface ToistConfig {
+  /** TCP port. Env PORT or TOIST_PORT takes precedence. */
+  port?: number
+  /** Root directory for default path resolution. Defaults to config file dir. */
+  root?: string
+  /** Pipeline YAML directory. Default: <root>/pipelines */
+  pipelines?: string
+  /** Resource path override (legacy). Default: <root>/resources */
+  resources?: string | Record<string, Record<string, unknown>>
+  /** Data directory (SQLite files). Default: <root>/data */
+  data?: string
+  /** TypeScript kind files to import and register before starting. */
+  kinds?: string[]
+  /** Disable the bundled UI. */
+  disableUi?: boolean
+  /** Disable filesystem watch on pipelines/resources. */
+  disableWatch?: boolean
+  /** Disable the MCP server. */
+  disableMcp?: boolean
+}
+
+// ---------------------------------------------------------------------------
+// CLI args
+// ---------------------------------------------------------------------------
+
+const args = process.argv.slice(2)
+
+if (args.includes("--help") || args.includes("-h")) {
+  console.log("Usage:")
+  console.log("  bunx @toist/aja                     auto-discover toist.yml in cwd")
+  console.log("  bunx @toist/aja --config <path>     explicit config file")
+  console.log("")
+  console.log("Config file format (YAML):")
+  console.log("  port: 3000")
+  console.log("  root: .                  # base for all relative paths")
+  console.log("  pipelines: ./pipelines   # override pipeline dir")
+  console.log("  resources: ./resources   # override resource dir")
+  console.log("  data: ./data             # override data dir")
+  console.log("  kinds:                   # TypeScript kind files to register")
+  console.log("    - ./src/kinds/index.ts")
+  console.log("")
+  console.log("Environment:")
+  console.log("  TOIST_PORT / PORT        Override port from config")
+  console.log("")
+  console.log("Full docs: https://toist.in/docs")
+  process.exit(0)
+}
+
+const configIdx = args.indexOf("--config")
+const configArg = configIdx !== -1 ? args[configIdx + 1] : null
+
+// ---------------------------------------------------------------------------
+// Config discovery
+// ---------------------------------------------------------------------------
+
+const CONFIG_NAMES = ["toist.yml", "toist.yaml"]
+
+function findConfig(): string {
+  if (configArg) {
+    const p = resolve(process.cwd(), configArg)
+    if (!existsSync(p)) {
+      console.error(`Config file not found: ${p}`)
+      process.exit(1)
+    }
+    return p
+  }
+  for (const name of CONFIG_NAMES) {
+    const p = resolve(process.cwd(), name)
+    if (existsSync(p)) return p
+  }
+  console.error("No toist.yml found in current directory.")
+  console.error("")
+  console.error("Create one, or use:  bunx @toist/aja --config <path>")
+  console.error("To scaffold a new Toist setup: bunx @toist/in")
+  process.exit(1)
+}
+
+// ---------------------------------------------------------------------------
+// Path resolution — all paths relative to config file directory
+// ---------------------------------------------------------------------------
+
+function resolvePath(configDir: string, root: string, p: string | undefined, defaultSub: string): string {
+  if (p) return isAbsolute(p) ? p : resolve(configDir, p)
+  return resolve(configDir, root, defaultSub)
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+const configPath = findConfig()
+const configDir  = dirname(configPath)
+const raw        = readFileSync(configPath, "utf8")
+const cfg        = (YAML.parse(raw) ?? {}) as ToistConfig
+
+const root = cfg.root ? resolve(configDir, cfg.root) : configDir
+
+const pipelineDir = resolvePath(configDir, root, cfg.pipelines, "pipelines")
+const resourceDir = resolvePath(configDir, root, typeof cfg.resources === "string" ? cfg.resources : undefined, "resources")
+const dataDir     = resolvePath(configDir, root, cfg.data,      "data")
+const port        = Number(process.env.TOIST_PORT ?? process.env.PORT ?? cfg.port ?? 3000)
+
+// Register kinds from config — dynamic imports resolved relative to config dir.
+if (cfg.kinds && cfg.kinds.length > 0) {
+  for (const kindPath of cfg.kinds) {
+    const abs = isAbsolute(kindPath) ? kindPath : resolve(configDir, kindPath)
+    if (!existsSync(abs)) {
+      console.error(`Kind file not found: ${abs}`)
+      process.exit(1)
+    }
+    const mod = await import(abs)
+    const kinds = Object.values(mod).filter(Boolean)
+    if (kinds.length === 0) {
+      console.warn(`[toist] warning: no exports found in kind file ${abs}`)
+    } else {
+      register(...(kinds as Parameters<typeof register>))
+    }
+  }
+}
+
+void cfg.disableMcp
+
+const runtime = await createSqliteRuntime({
+  rootDir: root,
+  pipelinesDir: pipelineDir,
+  resourcesDir: resourceDir,
+  dataDir,
+})
+
+await startRunner({
+  port,
+  rootDir: root,
+  runtime,
+  disableUi: cfg.disableUi ?? false,
+  disableWatch: cfg.disableWatch ?? false,
+})

package/src/client.ts

@@ -5,6 +5,7 @@
 // that want to drive the runner over its HTTP API without hard-coding paths.
 
 import type { NodeKindManifest, ValidateResult } from "@toist/spec"
+import type { ToistEvent } from "@toist/core"
 
 export interface RunnerClientOptions {
   /** Runner API base URL. Accepts either the API root
@@ -74,6 +75,12 @@ export type PipelineRunResponse =
   | { id: number; pipeline: string; status: "suspended"; suspendedAt: string; task: PipelineTaskRef; steps: StepResult[] }
   | { id: number; pipeline: string; status: "error"; error: string }
 
+export interface AsyncRunStartResponse {
+  id: number
+  pipeline: string
+  status: "running"
+}
+
 export interface RunListItem {
   id: number
   pipeline: string
@@ -161,12 +168,14 @@ export interface RunnerClient {
     create(yaml: string): Promise<PipelineSaveResult>
     update(id: string, yaml: string): Promise<PipelineSaveResult>
     run(id: string, payload?: Record<string, unknown>): Promise<PipelineRunResponse>
+    runSpec(spec: unknown, payload?: Record<string, unknown>): Promise<AsyncRunStartResponse>
   }
   runs: {
     list(opts?: { pipeline?: string; limit?: number }): Promise<RunListItem[]>
     get(id: number): Promise<RunListItem>
     nodes(id: number): Promise<RunNodeOutput[]>
     logs(id: number): Promise<RunLogLine[]>
+    events(id: number): AsyncIterable<ToistEvent>
   }
   tasks: {
     list(opts?: { status?: string; runId?: number; assignee?: string; limit?: number }): Promise<TaskListItem[]>
@@ -234,6 +243,10 @@ export function createRunnerClient(options: RunnerClientOptions): RunnerClient {
         method: "POST",
         body: json(payload),
       }),
+      runSpec: (spec, payload = {}) => request<AsyncRunStartResponse>("/runs", {
+        method: "POST",
+        body: json({ spec, payload }),
+      }),
     },
 
     runs: {
@@ -241,6 +254,7 @@ export function createRunnerClient(options: RunnerClientOptions): RunnerClient {
       get:   (id) => request<RunListItem>(`/runs/${id}`),
       nodes: (id) => request<RunNodeOutput[]>(`/runs/${id}/nodes`),
       logs:  (id) => request<RunLogLine[]>(`/runs/${id}/logs`),
+      events: (id) => streamEvents(f, `${base}/runs/${id}/events`, options.headers),
     },
 
     tasks: {
@@ -303,3 +317,65 @@ async function readJsonOrText(response: Response): Promise<unknown> {
   try { return JSON.parse(text) }
   catch { return text }
 }
+
+function streamEvents(
+  f: typeof fetch,
+  url: string,
+  extraHeaders?: HeadersInit,
+): AsyncIterable<ToistEvent> {
+  return {
+    [Symbol.asyncIterator]() {
+      let reader: ReadableStreamDefaultReader<Uint8Array> | null = null
+      let buffer = ""
+      const decoder = new TextDecoder()
+      const queue: ToistEvent[] = []
+      let finished = false
+
+      async function ensureReader() {
+        if (reader) return reader
+        const headers = new Headers(extraHeaders)
+        headers.set("accept", "text/event-stream")
+        const response = await f(url, { headers })
+        if (!response.ok) {
+          const body = await readJsonOrText(response)
+          throw new RunnerHttpError(response.status, response.statusText, body)
+        }
+        if (!response.body) throw new Error("runner returned no SSE body")
+        reader = response.body.getReader()
+        return reader
+      }
+
+      async function pump(): Promise<void> {
+        const r = await ensureReader()
+        while (queue.length === 0 && !finished) {
+          const { value, done } = await r.read()
+          if (done) {
+            finished = true
+            return
+          }
+          buffer += decoder.decode(value, { stream: true })
+          let boundary = buffer.indexOf("\n\n")
+          while (boundary !== -1) {
+            const frame = buffer.slice(0, boundary)
+            buffer = buffer.slice(boundary + 2)
+            const data = frame
+              .split("\n")
+              .filter((line) => line.startsWith("data:"))
+              .map((line) => line.slice(5).trim())
+              .join("\n")
+            if (data) queue.push(JSON.parse(data) as ToistEvent)
+            boundary = buffer.indexOf("\n\n")
+          }
+        }
+      }
+
+      return {
+        async next(): Promise<IteratorResult<ToistEvent>> {
+          await pump()
+          if (queue.length > 0) return { value: queue.shift()!, done: false }
+          return { value: undefined as never, done: true }
+        },
+      }
+    },
+  }
+}

package/src/data-db.ts

@@ -1,21 +1,9 @@
 // 2121
-// Domain data store. The instance owns the schema entirely — every table here
-// is created by a pipeline (via db.insert) or by a domain migration the project
-// chooses to add. Platform never writes to this DB directly.
-//
-// Kinds receive this as `ctx.db`. Pipelines write to it (db.insert), read
-// from it (db.query), or any custom domain kind interacts with it as needed.
-//
-// Location: <dataDir>/data.db (per context/instance-spec.md §3 — the host's
-// schema-on-write domain DB lives in data/ alongside runtime.db and cache.db).
-
 import { Database } from "bun:sqlite"
 import { mkdirSync } from "node:fs"
 import { dirname } from "node:path"
-import { dataDbPath } from "./config.ts"
 
-export function openDataDb(): Database {
-  const path = dataDbPath()
+export function openDataDb(path: string): Database {
   mkdirSync(dirname(path), { recursive: true })
   return new Database(path, { create: true })
 }

package/src/index.ts

@@ -9,6 +9,9 @@
 
 // Lifecycle entry — what hosts call to start an instance.
 export { startRunner, type StartRunnerOptions, type RunnerHandle } from "./startRunner.ts"
+export { createSqliteRuntime, type CreateSqliteRuntimeOptions } from "./sqlite-runtime.ts"
+export { loadFilesystemResources } from "./resources-fs.ts"
+export { createFilesystemPipelineStore, type FilesystemPipelineStore } from "./pipeline-store.ts"
 
 // Typed HTTP client — what external hosts/tools use to drive an instance.
 export {
@@ -22,6 +25,7 @@ export {
   type StepResult,
   type PipelineTaskRef,
   type PipelineRunResponse,
+  type AsyncRunStartResponse,
   type RunListItem,
   type RunNodeOutput,
   type RunLogLine,
@@ -31,15 +35,44 @@ export {
   type ResourceRecord,
 } from "./client.ts"
 
-// Kind registration — must be called before startRunner.
-export { register, getKind, manifest } from "./kinds/index.ts"
+// Kernel convenience re-exports.
+export {
+  builtinKinds,
+  createRegistry,
+  createMemoryRuntime,
+  createEventStream,
+  runSpec,
+  register,
+  getKind,
+  manifest,
+} from "@toist/core"
 
 // Type re-exports from @toist/spec, so a host that imports only
 // @toist/aja doesn't need a second package on its dependency list when
 // it only writes kinds (rather than authoring pipeline-format tooling).
+export type {
+  ToistRuntime,
+  ResumableRuntime,
+  RunLedger,
+  TaskStore,
+  LogStore,
+  NodeOutputStore,
+  ResourceResolver,
+  KindRegistry,
+  RunCtx,
+  RunOutcome,
+  RunOptions,
+  ToistEvent,
+  EventSink,
+  RunSpecOptions,
+  RunSpecResult,
+} from "@toist/core"
+
 export type {
   NodeKind,
   NodeKindManifest,
+  DataHandle,
+  DataStatement,
   ParamDef,
   PortDef,
   ExecContext,

package/src/instance-metadata.ts

@@ -0,0 +1,48 @@
+// 2121
+// Per-instance metadata: what kind of platform-instance this is (which
+// customer, what tier, what teasers to surface), distinct from the runtime
+// state in runtime.db.
+
+import { existsSync, readFileSync } from "node:fs"
+import { join } from "node:path"
+
+export interface Teaser {
+  id:           string
+  title:        string
+  description?: string
+  category?:    string
+}
+
+export interface Instance {
+  platformVersion?: string
+  instanceName?:    string
+  tier?:            string
+  kindAllowlist?:   string[]
+  teasers:          Teaser[]
+}
+
+const DEFAULTS: Instance = { teasers: [] }
+
+// Re-read on every call. instance.json is tiny and changes rarely; the cost
+// is negligible and avoids stale-cache surprises when the file is edited.
+export function loadInstance(rootDir: string): Instance {
+  const path = join(rootDir, "instance.json")
+  if (!existsSync(path)) return DEFAULTS
+
+  try {
+    const raw = readFileSync(path, "utf8")
+    const parsed = JSON.parse(raw) as Partial<Instance>
+    return {
+      platformVersion: parsed.platformVersion,
+      instanceName: parsed.instanceName,
+      tier: parsed.tier,
+      kindAllowlist: Array.isArray(parsed.kindAllowlist)
+        ? parsed.kindAllowlist.filter((value): value is string => typeof value === "string")
+        : undefined,
+      teasers: Array.isArray(parsed.teasers) ? parsed.teasers : [],
+    }
+  } catch (err) {
+    console.warn(`[instance] failed to read ${path}:`, (err as Error).message)
+    return DEFAULTS
+  }
+}