@toist/aja 0.7.1 → 0.8.0

package/src/migrate.ts

@@ -103,7 +103,7 @@ function backupAndPrune(dbPath: string): string | null {
   const backupPath = join(dir, `${base}.bak-${ts}`)
 
   copyFileSync(dbPath, backupPath)
-  console.log(`[migrate] backup ${backupPath}`)
+  console.error(`[migrate] backup ${backupPath}`)
 
   const prefix = `${base}.bak-`
   const all = readdirSync(dir)
@@ -113,7 +113,7 @@ function backupAndPrune(dbPath: string): string | null {
 
   for (const stale of all.slice(BACKUP_RETAIN)) {
     unlinkSync(join(dir, stale.name))
-    console.log(`[migrate] pruned old backup ${stale.name}`)
+    console.error(`[migrate] pruned old backup ${stale.name}`)
   }
 
   return backupPath
@@ -191,7 +191,7 @@ export function runMigrations(dbPath: string, opts: MigrateOptions = {}): Migrat
       try {
         apply()
         result.applied.push(file.filename)
-        console.log(`[migrate] applied ${file.filename}`)
+        console.error(`[migrate] applied ${file.filename}`)
       } catch (err) {
         const msg = err instanceof Error ? err.message : String(err)
         throw new Error(`[migrate] failed applying "${file.filename}": ${msg}`)

package/src/pipeline-store.ts

@@ -0,0 +1,31 @@
+import { join } from "node:path"
+import { getPipeline, getPipelines, loadAll, watchAll, type KindRegistry } from "@toist/core"
+import type { PipelineSpec } from "@toist/spec"
+
+export interface FilesystemPipelineStore {
+  get(id: string): PipelineSpec | null
+  list(): PipelineSpec[]
+  close(): void
+}
+
+export function createFilesystemPipelineStore(options: {
+  rootDir: string
+  watch?: boolean
+  registry?: KindRegistry
+}): FilesystemPipelineStore {
+  const pipelinesDir = join(options.rootDir, "pipelines")
+  loadAll(pipelinesDir, { registry: options.registry })
+  const watcher = options.watch === false ? null : watchAll(pipelinesDir, undefined, { registry: options.registry })
+
+  return {
+    get(id) {
+      return getPipeline(id) ?? null
+    },
+    list() {
+      return getPipelines()
+    },
+    close() {
+      watcher?.close()
+    },
+  }
+}

package/src/resources-fs.ts

@@ -0,0 +1,43 @@
+// 2121
+import { existsSync, readFileSync } from "node:fs"
+import { join } from "node:path"
+import YAML from "yaml"
+
+const ENV_EXPR = /\$\{env:([A-Za-z_][A-Za-z0-9_]*)(?::-(.*?))?\}/g
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === "object" && !Array.isArray(value)
+}
+
+function substituteEnvInString(value: string): string {
+  return value.replace(ENV_EXPR, (_match, name: string, fallback: string | undefined) => {
+    const resolved = process.env[name]
+    if (resolved !== undefined) return resolved
+    if (fallback !== undefined) return fallback
+    throw new Error(`[resources-fs] missing required environment variable ${name}`)
+  })
+}
+
+function substituteEnv(value: unknown): unknown {
+  if (typeof value === "string") return substituteEnvInString(value)
+  if (Array.isArray(value)) return value.map((entry) => substituteEnv(entry))
+  if (!isPlainObject(value)) return value
+  return Object.fromEntries(
+    Object.entries(value).map(([key, entry]) => [key, substituteEnv(entry)]),
+  )
+}
+
+export async function loadFilesystemResources(
+  rootDir: string,
+  _opts: { env?: string } = {},
+): Promise<Record<string, Record<string, unknown>>> {
+  const candidates = [join(rootDir, "toist.yml"), join(rootDir, "toist.yaml")]
+  const path = candidates.find((candidate) => existsSync(candidate))
+  if (!path) return {}
+
+  const parsed = (YAML.parse(readFileSync(path, "utf8")) ?? {}) as Record<string, unknown>
+  const resources = parsed.resources
+  if (!isPlainObject(resources)) return {}
+
+  return substituteEnv(resources) as Record<string, Record<string, unknown>>
+}

package/src/resources.ts

@@ -1,29 +1,14 @@
 // 2121
-// Resource system — typed external-system handles.
+// Resource type registry + runtime DB CRUD for the admin API.
 //
-// Three-tier override chain (highest priority first):
-//   1. ENV-VAR:  PLATFORM_RESOURCE_<NAME>_<FIELD>=value
-//   2. DB:       runtime.db resources table (set via UI or POST /resources)
-//   3. YAML:     <runner>/resources/*.yaml (committable; secrets use { $env: "VAR" })
-//
-// ctx.resource.<name>.<field> is the pipeline-side surface.
-// Fields from higher tiers override lower tiers field-by-field.
-//
-// Sensitive fields (x-sensitive:true in the Type schema) are stored
-// clear-text in v1. A load-time warning surfaces this. Encryption at
-// rest is deferred to pipeline-spec.md §16.1.
+// Note: execution-time resource resolution no longer reads from the resources
+// table. Phase 4 resolves resources from <rootDir>/toist.yml at runtime
+// construction time via resources-fs.ts. The DB table remains for the admin
+// surface / legacy UI editor until a later UX pass decides its fate.
 
-import { existsSync, readdirSync, readFileSync } from "node:fs"
-import { basename, extname, join, dirname } from "node:path"
-import { fileURLToPath } from "node:url"
 import type { Database } from "bun:sqlite"
-import { parseYaml, YamlError } from "@toist/spec"
 import type { ResourceTypeDef } from "@toist/spec"
 
-const __dir = dirname(fileURLToPath(import.meta.url))
-// CWD is apps/runner/server/; resources/*.yaml lives at apps/runner/resources/
-const RESOURCES_DIR = join(__dir, "..", "..", "resources")
-
 // ─── Resource Type registry ───────────────────────────────────────────────────
 
 const typeRegistry = new Map<string, ResourceTypeDef>()
@@ -49,7 +34,7 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
       type: "object",
       properties: {
         apiKey: { type: "string", "x-sensitive": true, description: "Anthropic API key (sk-ant-…)" },
-        model:  { type: "string", default: "claude-sonnet-4-6", description: "Default model ID" },
+        model: { type: "string", default: "claude-sonnet-4-6", description: "Default model ID" },
       },
       required: ["apiKey"],
     },
@@ -61,8 +46,8 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
       $schema: "https://json-schema.org/draft/2020-12/schema",
       type: "object",
       properties: {
-        apiKey:  { type: "string", "x-sensitive": true },
-        model:   { type: "string", default: "gpt-4o" },
+        apiKey: { type: "string", "x-sensitive": true },
+        model: { type: "string", default: "gpt-4o" },
         baseUrl: { type: "string", description: "Override for OpenAI-compatible endpoints" },
       },
       required: ["apiKey"],
@@ -75,12 +60,12 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
       $schema: "https://json-schema.org/draft/2020-12/schema",
       type: "object",
       properties: {
-        host:     { type: "string" },
-        port:     { type: "number", default: 5432 },
+        host: { type: "string" },
+        port: { type: "number", default: 5432 },
         database: { type: "string" },
-        user:     { type: "string" },
+        user: { type: "string" },
         password: { type: "string", "x-sensitive": true },
-        ssl:      { type: "boolean", default: false },
+        ssl: { type: "boolean", default: false },
       },
       required: ["host", "database", "user"],
     },
@@ -92,11 +77,11 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
       $schema: "https://json-schema.org/draft/2020-12/schema",
       type: "object",
       properties: {
-        bucket:          { type: "string" },
-        region:          { type: "string" },
-        accessKeyId:     { type: "string", "x-sensitive": true },
+        bucket: { type: "string" },
+        region: { type: "string" },
+        accessKeyId: { type: "string", "x-sensitive": true },
         secretAccessKey: { type: "string", "x-sensitive": true },
-        endpoint:        { type: "string", description: "S3-compatible endpoint URL (optional)" },
+        endpoint: { type: "string", description: "S3-compatible endpoint URL (optional)" },
       },
       required: ["bucket", "accessKeyId", "secretAccessKey"],
     },
@@ -108,8 +93,8 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
       $schema: "https://json-schema.org/draft/2020-12/schema",
       type: "object",
       properties: {
-        baseUrl:     { type: "string" },
-        apiKey:      { type: "string", "x-sensitive": true },
+        baseUrl: { type: "string" },
+        apiKey: { type: "string", "x-sensitive": true },
         bearerToken: { type: "string", "x-sensitive": true },
       },
       required: ["baseUrl"],
@@ -119,82 +104,7 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
 
 registerResourceType(...BUILTIN_TYPES)
 
-// ─── Sensitive-field detection ────────────────────────────────────────────────
-
-function sensitiveFields(typeName: string): Set<string> {
-  const type = typeRegistry.get(typeName)
-  if (!type) return new Set()
-  const props = (type.schema?.properties ?? {}) as Record<string, { "x-sensitive"?: boolean }>
-  return new Set(Object.entries(props).filter(([, v]) => v?.["x-sensitive"]).map(([k]) => k))
-}
-
-function warnSensitive(name: string, typeName: string, fields: Record<string, unknown>): void {
-  const sensitive = sensitiveFields(typeName)
-  const exposed = Object.keys(fields).filter((k) => sensitive.has(k))
-  if (exposed.length > 0) {
-    console.warn(
-      `[resources] resource "${name}" (${typeName}) has sensitive fields stored unencrypted: ${exposed.join(", ")}. Encryption pending (§16.1).`,
-    )
-  }
-}
-
-// ─── $env placeholder resolver ────────────────────────────────────────────────
-
-function resolveEnvPlaceholders(val: unknown, source: string): unknown {
-  if (val === null || typeof val !== "object") return val
-  if (Array.isArray(val)) return val.map((v) => resolveEnvPlaceholders(v, source))
-  const obj = val as Record<string, unknown>
-  if ("$env" in obj && typeof obj["$env"] === "string") {
-    const envVal = process.env[obj["$env"]]
-    if (envVal === undefined) {
-      throw new Error(`[resources] ${source}: $env placeholder "${obj["$env"]}" is not set at startup`)
-    }
-    return envVal
-  }
-  return Object.fromEntries(
-    Object.entries(obj).map(([k, v]) => [k, resolveEnvPlaceholders(v, source)]),
-  )
-}
-
-// ─── YAML resource loader ─────────────────────────────────────────────────────
-
-interface ResourceYaml {
-  type: string
-  name?: string
-  fields?: Record<string, unknown>
-}
-
-function loadResourceYamls(): Map<string, { type: string; fields: Record<string, unknown> }> {
-  const out = new Map<string, { type: string; fields: Record<string, unknown> }>()
-  if (!existsSync(RESOURCES_DIR)) return out
-
-  const files = readdirSync(RESOURCES_DIR).filter((f) => /\.ya?ml$/.test(f))
-  for (const file of files) {
-    const path = join(RESOURCES_DIR, file)
-    try {
-      const raw = readFileSync(path, "utf8")
-      const parsed = parseYaml(raw) as ResourceYaml
-      if (!parsed || typeof parsed !== "object" || !parsed.type) {
-        console.warn(`[resources] ${file}: missing required "type" field — skipped`)
-        continue
-      }
-      const stem  = basename(file, extname(file))
-      const name  = parsed.name ?? stem
-      const rawFields = parsed.fields ?? {}
-      const fields = resolveEnvPlaceholders(rawFields, file) as Record<string, unknown>
-      out.set(name, { type: parsed.type, fields })
-    } catch (err) {
-      if (err instanceof YamlError) {
-        console.warn(`[resources] ${file}: YAML parse error — ${err.message}`)
-      } else {
-        console.warn(`[resources] ${file}: ${(err as Error).message}`)
-      }
-    }
-  }
-  return out
-}
-
-// ─── DB resource loader ───────────────────────────────────────────────────────
+// ─── DB CRUD (used by API routes) ─────────────────────────────────────────────
 
 interface ResourceRow {
   id: number
@@ -205,79 +115,6 @@ interface ResourceRow {
   updated_at: string
 }
 
-function loadResourcesFromDb(db: Database): Map<string, { type: string; fields: Record<string, unknown> }> {
-  const out = new Map<string, { type: string; fields: Record<string, unknown> }>()
-  try {
-    const rows = db.prepare("SELECT name, type, fields_json FROM resources").all() as
-      { name: string; type: string; fields_json: string }[]
-    for (const row of rows) {
-      try {
-        const fields = JSON.parse(row.fields_json) as Record<string, unknown>
-        out.set(row.name, { type: row.type, fields })
-      } catch {
-        console.warn(`[resources] DB row "${row.name}": invalid fields_json — skipped`)
-      }
-    }
-  } catch (err) {
-    // Table may not exist yet (migration pending). Log and continue.
-    console.warn(`[resources] DB read failed: ${(err as Error).message}`)
-  }
-  return out
-}
-
-// ─── ENV-VAR resolver ─────────────────────────────────────────────────────────
-// Scans process.env for PLATFORM_RESOURCE_<NAME>_<FIELD>=value.
-// NAME and FIELD are uppercased in the env key; both are lowercased when
-// applied to ctx.resource. Supports single-level fields only (no nested ENV).
-
-function applyEnvOverrides(result: Map<string, { type: string; fields: Record<string, unknown> }>): void {
-  for (const [key, value] of Object.entries(process.env)) {
-    if (!value) continue
-    const m = key.match(/^PLATFORM_RESOURCE_([A-Z][A-Z0-9_]*)_([A-Z][A-Z0-9_]*)$/)
-    if (!m) continue
-    const name  = m[1].toLowerCase()
-    const field = m[2].toLowerCase()
-    const entry = result.get(name)
-    if (entry) {
-      entry.fields[field] = value
-    } else {
-      // ENV-only resource with no YAML or DB entry — create a minimal record.
-      result.set(name, { type: "Unknown", fields: { [field]: value } })
-    }
-  }
-}
-
-// ─── Main: build ctx.resource ─────────────────────────────────────────────────
-
-/** Merges all three tiers and returns the ctx.resource namespace for runPipeline. */
-export function buildResourceCtx(db: Database): Record<string, Record<string, unknown>> {
-  // Tier 3 (lowest): YAML files
-  const merged = loadResourceYamls()
-
-  // Tier 2: DB overrides YAML field-by-field
-  for (const [name, dbEntry] of loadResourcesFromDb(db)) {
-    const existing = merged.get(name)
-    if (existing) {
-      existing.fields = { ...existing.fields, ...dbEntry.fields }
-    } else {
-      merged.set(name, dbEntry)
-    }
-  }
-
-  // Tier 1 (highest): ENV-VAR overrides
-  applyEnvOverrides(merged)
-
-  // Flatten to Record<name, fields> and warn on exposed sensitive fields
-  const out: Record<string, Record<string, unknown>> = {}
-  for (const [name, { type, fields }] of merged) {
-    warnSensitive(name, type, fields)
-    out[name] = fields
-  }
-  return out
-}
-
-// ─── DB CRUD (used by API routes) ─────────────────────────────────────────────
-
 export interface ResourceRecord {
   id: number
   name: string
@@ -307,9 +144,12 @@ export function getResource(db: Database, name: string): ResourceRecord | null {
   ).get(name) as ResourceRow | undefined
   if (!row) return null
   return {
-    id: row.id, name: row.name, type: row.type,
+    id: row.id,
+    name: row.name,
+    type: row.type,
     fields: JSON.parse(row.fields_json) as Record<string, unknown>,
-    created_at: row.created_at, updated_at: row.updated_at,
+    created_at: row.created_at,
+    updated_at: row.updated_at,
   }
 }
 
@@ -338,13 +178,10 @@ export function patchResource(
   const existing = getResource(db, name)
   if (!existing) return null
   const merged = { ...existing.fields, ...fields }
-  db.prepare(
-    "UPDATE resources SET fields_json = ?, updated_at = datetime('now') WHERE name = ?",
-  ).run(JSON.stringify(merged), name)
-  return getResource(db, name)!
+  return upsertResource(db, name, existing.type, merged)
 }
 
 export function deleteResource(db: Database, name: string): boolean {
-  const result = db.prepare("DELETE FROM resources WHERE name = ?").run(name)
-  return result.changes > 0
+  const changed = db.prepare("DELETE FROM resources WHERE name = ?").run(name).changes
+  return changed > 0
 }

package/src/run-events.ts

@@ -0,0 +1,42 @@
+import type { ToistEvent } from "@toist/core"
+
+export interface RunEventBroker {
+  emit(event: ToistEvent): Promise<void>
+  finish(): void
+  subscribe(): AsyncIterable<ToistEvent>
+}
+
+export function createRunEventBroker(): RunEventBroker {
+  const events: ToistEvent[] = []
+  let finished = false
+  const waiters = new Set<() => void>()
+
+  return {
+    async emit(event) {
+      events.push(event)
+      for (const notify of waiters) notify()
+      waiters.clear()
+    },
+    finish() {
+      finished = true
+      for (const notify of waiters) notify()
+      waiters.clear()
+    },
+    subscribe() {
+      let index = 0
+      return {
+        [Symbol.asyncIterator]() {
+          return {
+            async next(): Promise<IteratorResult<ToistEvent>> {
+              while (index >= events.length) {
+                if (finished) return { value: undefined as never, done: true }
+                await new Promise<void>((resolve) => waiters.add(resolve))
+              }
+              return { value: events[index++], done: false }
+            },
+          }
+        },
+      }
+    },
+  }
+}

package/src/runtime-db.ts

@@ -1,44 +1,25 @@
 // 2121
-// Platform-internal runtime ledger. Holds:
-//   - runs        — every pipeline execution (status, payload, result, steps, timing)
-//   - logs        — log lines emitted from kinds during a run, scoped to run_id
-//   - tasks       — HITL pending tasks (added in migration 002)
-//   - node_outputs — checkpoint-between-nodes memoization (added in migration 002)
-//
-// Schema is owned by the platform and evolved via numbered SQL migrations
-// in ../migrations/. See migrate.ts for the runner.
-//
-// Kinds never receive a handle to this DB — they only see ctx.db (data
-// store) and ctx.cache (capability), so they cannot accidentally touch
-// runtime tables.
-//
-// Migration policy: by default the runner applies pending migrations at
-// startup. RUNTIME_AUTO_MIGRATE=false flips to fail-loud — startup aborts if
-// anything is pending.
-//
-// Lifecycle: openRuntimeDb() is a pure factory. The runner lock is acquired
-// separately by db-handles.initDbs() before this is called, so two callers
-// cannot race on migrations.
-
 import { Database } from "bun:sqlite"
 import { mkdirSync } from "node:fs"
 import { dirname } from "node:path"
 import { runMigrations, pendingMigrations } from "./migrate.ts"
-import { runtimeDbPath } from "./config.ts"
 
-export function openRuntimeDb(): Database {
-  const path = runtimeDbPath()
+export function openRuntimeDb(path: string, opts: { skipMigrations?: boolean } = {}): Database {
   mkdirSync(dirname(path), { recursive: true })
 
-  if (process.env.RUNTIME_AUTO_MIGRATE === "false") {
+  if (opts.skipMigrations) {
+    const pending = pendingMigrations(path)
+    if (pending.length > 0) {
+      throw new Error(
+        `[runtime-db] ${pending.length} pending migration(s) with skipMigrations=true: ${pending.join(", ")}`,
+      )
+    }
+  } else if (process.env.RUNTIME_AUTO_MIGRATE === "false") {
     const pending = pendingMigrations(path)
     if (pending.length > 0) {
-      console.error(
-        `[runtime-db] ${pending.length} pending migration(s); RUNTIME_AUTO_MIGRATE=false.`,
+      throw new Error(
+        `[runtime-db] ${pending.length} pending migration(s); RUNTIME_AUTO_MIGRATE=false. Pending: ${pending.join(", ")}`,
       )
-      console.error(`[runtime-db] pending: ${pending.join(", ")}`)
-      console.error(`[runtime-db] run migrations explicitly, or unset RUNTIME_AUTO_MIGRATE.`)
-      process.exit(1)
     }
   } else {
     runMigrations(path)