@toist/aja 0.6.1 → 0.8.0

package/src/kinds/index.ts

@@ -1,66 +1,28 @@
 // 2121
-import type { NodeKind, NodeKindManifest } from "./types.ts"
-import { dataJson, dataMerge } from "./data.ts"
-import { transformFilter, transformMap, transformSort, transformAggregate } from "./transform.ts"
-import { dbInsert, dbQuery } from "./db.ts"
-import { httpFetch, httpPost } from "./http.ts"
-import { trigger, sink } from "./control.ts"
-import { humanInput } from "./hitl.ts"
-import { runsLastOutput, runsNodeOutput, pipelineRun } from "./runs.ts"
+// Aja-side kind registry shim.
+//
+// Builtins and registry implementation now live in @toist/core. Aja keeps
+// this module so existing local imports and custom kind registration patterns
+// continue to work during the package carve-out.
 
-const builtins: NodeKind<any, any>[] = [
-  trigger,
-  sink,
-  dataJson,
-  dataMerge,
-  transformFilter,
-  transformMap,
-  transformSort,
-  transformAggregate,
-  dbInsert,
-  dbQuery,
-  httpFetch,
-  httpPost,
-  humanInput,
-  runsLastOutput,
-  runsNodeOutput,
-  pipelineRun,
-]
+export {
+  builtinKinds,
+  createRegistry,
+  register,
+  getKind,
+  manifest,
+} from "@toist/core"
 
-export const registry: Map<string, NodeKind<any, any>> = new Map(builtins.map((k) => [k.id, k]))
+export type {
+  NodeKind,
+  NodeKindManifest,
+  DataHandle,
+  DataStatement,
+  ParamDef,
+  PortDef,
+  ExecContext,
+  PlatformCtx,
+} from "@toist/spec"
 
-/**
- * Register one or more custom domain kinds. Domain projects edit
- * `runner/src/kinds/custom.ts` (created by `platform init`) and call
- * `register(...)` there with their own kinds. Builtin kinds and domain kinds
- * live in the same registry — `getKind(id)` resolves both transparently.
- *
- * Re-registering an existing id replaces the previous entry; useful for hot
- * iteration but warn loudly so accidental shadows are visible.
- */
-export function register(...kinds: NodeKind<any, any>[]): void {
-  for (const k of kinds) {
-    if (registry.has(k.id) && !builtins.find((b) => b.id === k.id)) {
-      console.warn(`[kinds] re-registering "${k.id}" — previous registration replaced`)
-    }
-    registry.set(k.id, k)
-  }
-}
-
-export function getKind(id: string): NodeKind<any, any> | undefined {
-  return registry.get(id)
-}
-
-export function manifest(): NodeKindManifest[] {
-  return [...registry.values()].map(({ run: _e, ...rest }) => rest)
-}
-
-export type { NodeKind, NodeKindManifest, ParamDef, PortDef, ExecContext, PlatformCtx } from "./types.ts"
-
-// Side-effect import: domain kinds register themselves when this module loads.
-// Dynamic import + top-level await defers custom.ts evaluation until AFTER
-// `registry` and `register` are initialised — a static `import "./custom.ts"`
-// gets hoisted ahead of the const init, putting `registry` in the temporal
-// dead zone when custom.ts calls register(). The dynamic form runs at
-// statement position, after init, breaking the cycle.
+// Preserve the old side-effect registration hook for domain kinds.
 await import("./custom.ts")

package/src/lock.ts

@@ -1,64 +1,38 @@
 // 2121
-// Single-leader runner lock. Prevents two runner processes from racing on
-// migrations, scheduled cron-wheel firings, or HITL resume dispatch.
-//
-// The lock guards the data/ directory using a sentinel lockfile at
-// data/.lock. proper-lockfile creates a holder directory that includes mtime;
-// stale locks (from a crashed previous process) are detected after `stale` ms
-// and overtaken automatically.
-//
-// Watch-mode reload tolerance: bun --watch SIGTERMs the old process before
-// spawning the new one. The cleanup handlers below release the lock on
-// SIGTERM/SIGINT/exit. Async lock acquisition with retries handles the brief
-// window where the old process has not yet released.
-
 import lockfile from "proper-lockfile"
 import { mkdirSync } from "node:fs"
-import { dataDir, lockfilePath } from "./config.ts"
 
-let release: (() => Promise<void>) | null = null
+export interface AcquireRunnerLockOptions {
+  dir: string
+  lockfilePath: string
+}
 
-export async function acquireRunnerLock(): Promise<void> {
-  const dir = dataDir()
-  const lockPath = lockfilePath()
-  mkdirSync(dir, { recursive: true })
+export async function acquireRunnerLock(opts: AcquireRunnerLockOptions): Promise<() => Promise<void>> {
+  mkdirSync(opts.dir, { recursive: true })
 
-  try {
-    release = await lockfile.lock(dir, {
-      lockfilePath: lockPath,
-      realpath: false,
-      stale: 10_000,
-      retries: { retries: 5, factor: 2, minTimeout: 100, maxTimeout: 1000 },
-    })
-  } catch (err) {
-    const msg = err instanceof Error ? err.message : String(err)
-    console.error(`[runner] failed to acquire ${lockPath}: ${msg}`)
-    console.error(`[runner] another runner appears live. Stop it first; only one runner per project is allowed.`)
-    process.exit(1)
-  }
+  let release = await lockfile.lock(opts.dir, {
+    lockfilePath: opts.lockfilePath,
+    realpath: false,
+    stale: 10_000,
+    retries: { retries: 5, factor: 2, minTimeout: 100, maxTimeout: 1000 },
+  })
 
-  const cleanup = (): void => {
-    if (!release) return
-    const r = release
-    release = null
-    r().catch(() => { /* best-effort */ })
+  let released = false
+  const cleanup = async () => {
+    if (released) return
+    released = true
+    const current = release
+    release = async () => {}
+    try {
+      await current()
+    } catch {
+      // best-effort cleanup on process exit / stop
+    }
   }
 
-  process.on("exit",    cleanup)
-  process.on("SIGTERM", () => { cleanup(); process.exit(0) })
-  process.on("SIGINT",  () => { cleanup(); process.exit(0) })
-}
+  process.on("exit", () => { void cleanup() })
+  process.on("SIGTERM", () => { void cleanup() })
+  process.on("SIGINT", () => { void cleanup() })
 
-/** Release the runner lock explicitly. Used by `startRunner`'s graceful
- *  `stop()` path. Idempotent: no-op if the lock is not held. The signal
- *  cleanup handlers installed by acquireRunnerLock remain in place. */
-export async function releaseRunnerLock(): Promise<void> {
-  if (!release) return
-  const r = release
-  release = null
-  try {
-    await r()
-  } catch {
-    // best-effort — surfacing this would mask the actual stop reason
-  }
+  return cleanup
 }

package/src/migrate.ts

@@ -103,7 +103,7 @@ function backupAndPrune(dbPath: string): string | null {
   const backupPath = join(dir, `${base}.bak-${ts}`)
 
   copyFileSync(dbPath, backupPath)
-  console.log(`[migrate] backup ${backupPath}`)
+  console.error(`[migrate] backup ${backupPath}`)
 
   const prefix = `${base}.bak-`
   const all = readdirSync(dir)
@@ -113,7 +113,7 @@ function backupAndPrune(dbPath: string): string | null {
 
   for (const stale of all.slice(BACKUP_RETAIN)) {
     unlinkSync(join(dir, stale.name))
-    console.log(`[migrate] pruned old backup ${stale.name}`)
+    console.error(`[migrate] pruned old backup ${stale.name}`)
   }
 
   return backupPath
@@ -191,7 +191,7 @@ export function runMigrations(dbPath: string, opts: MigrateOptions = {}): Migrat
       try {
         apply()
         result.applied.push(file.filename)
-        console.log(`[migrate] applied ${file.filename}`)
+        console.error(`[migrate] applied ${file.filename}`)
       } catch (err) {
         const msg = err instanceof Error ? err.message : String(err)
         throw new Error(`[migrate] failed applying "${file.filename}": ${msg}`)

package/src/pipeline-store.ts

@@ -0,0 +1,31 @@
+import { join } from "node:path"
+import { getPipeline, getPipelines, loadAll, watchAll, type KindRegistry } from "@toist/core"
+import type { PipelineSpec } from "@toist/spec"
+
+export interface FilesystemPipelineStore {
+  get(id: string): PipelineSpec | null
+  list(): PipelineSpec[]
+  close(): void
+}
+
+export function createFilesystemPipelineStore(options: {
+  rootDir: string
+  watch?: boolean
+  registry?: KindRegistry
+}): FilesystemPipelineStore {
+  const pipelinesDir = join(options.rootDir, "pipelines")
+  loadAll(pipelinesDir, { registry: options.registry })
+  const watcher = options.watch === false ? null : watchAll(pipelinesDir, undefined, { registry: options.registry })
+
+  return {
+    get(id) {
+      return getPipeline(id) ?? null
+    },
+    list() {
+      return getPipelines()
+    },
+    close() {
+      watcher?.close()
+    },
+  }
+}

package/src/resources-fs.ts

@@ -0,0 +1,43 @@
+// 2121
+import { existsSync, readFileSync } from "node:fs"
+import { join } from "node:path"
+import YAML from "yaml"
+
+const ENV_EXPR = /\$\{env:([A-Za-z_][A-Za-z0-9_]*)(?::-(.*?))?\}/g
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === "object" && !Array.isArray(value)
+}
+
+function substituteEnvInString(value: string): string {
+  return value.replace(ENV_EXPR, (_match, name: string, fallback: string | undefined) => {
+    const resolved = process.env[name]
+    if (resolved !== undefined) return resolved
+    if (fallback !== undefined) return fallback
+    throw new Error(`[resources-fs] missing required environment variable ${name}`)
+  })
+}
+
+function substituteEnv(value: unknown): unknown {
+  if (typeof value === "string") return substituteEnvInString(value)
+  if (Array.isArray(value)) return value.map((entry) => substituteEnv(entry))
+  if (!isPlainObject(value)) return value
+  return Object.fromEntries(
+    Object.entries(value).map(([key, entry]) => [key, substituteEnv(entry)]),
+  )
+}
+
+export async function loadFilesystemResources(
+  rootDir: string,
+  _opts: { env?: string } = {},
+): Promise<Record<string, Record<string, unknown>>> {
+  const candidates = [join(rootDir, "toist.yml"), join(rootDir, "toist.yaml")]
+  const path = candidates.find((candidate) => existsSync(candidate))
+  if (!path) return {}
+
+  const parsed = (YAML.parse(readFileSync(path, "utf8")) ?? {}) as Record<string, unknown>
+  const resources = parsed.resources
+  if (!isPlainObject(resources)) return {}
+
+  return substituteEnv(resources) as Record<string, Record<string, unknown>>
+}

package/src/resources.ts

@@ -1,29 +1,14 @@
 // 2121
-// Resource system — typed external-system handles.
+// Resource type registry + runtime DB CRUD for the admin API.
 //
-// Three-tier override chain (highest priority first):
-//   1. ENV-VAR:  PLATFORM_RESOURCE_<NAME>_<FIELD>=value
-//   2. DB:       runtime.db resources table (set via UI or POST /resources)
-//   3. YAML:     <runner>/resources/*.yaml (committable; secrets use { $env: "VAR" })
-//
-// ctx.resource.<name>.<field> is the pipeline-side surface.
-// Fields from higher tiers override lower tiers field-by-field.
-//
-// Sensitive fields (x-sensitive:true in the Type schema) are stored
-// clear-text in v1. A load-time warning surfaces this. Encryption at
-// rest is deferred to pipeline-spec.md §16.1.
+// Note: execution-time resource resolution no longer reads from the resources
+// table. Phase 4 resolves resources from <rootDir>/toist.yml at runtime
+// construction time via resources-fs.ts. The DB table remains for the admin
+// surface / legacy UI editor until a later UX pass decides its fate.
 
-import { existsSync, readdirSync, readFileSync } from "node:fs"
-import { basename, extname, join, dirname } from "node:path"
-import { fileURLToPath } from "node:url"
 import type { Database } from "bun:sqlite"
-import { parseYaml, YamlError } from "@toist/spec"
 import type { ResourceTypeDef } from "@toist/spec"
 
-const __dir = dirname(fileURLToPath(import.meta.url))
-// CWD is apps/runner/server/; resources/*.yaml lives at apps/runner/resources/
-const RESOURCES_DIR = join(__dir, "..", "..", "resources")
-
 // ─── Resource Type registry ───────────────────────────────────────────────────
 
 const typeRegistry = new Map<string, ResourceTypeDef>()
@@ -49,7 +34,7 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
       type: "object",
       properties: {
         apiKey: { type: "string", "x-sensitive": true, description: "Anthropic API key (sk-ant-…)" },
-        model:  { type: "string", default: "claude-sonnet-4-6", description: "Default model ID" },
+        model: { type: "string", default: "claude-sonnet-4-6", description: "Default model ID" },
       },
       required: ["apiKey"],
     },
@@ -61,8 +46,8 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
       $schema: "https://json-schema.org/draft/2020-12/schema",
       type: "object",
       properties: {
-        apiKey:  { type: "string", "x-sensitive": true },
-        model:   { type: "string", default: "gpt-4o" },
+        apiKey: { type: "string", "x-sensitive": true },
+        model: { type: "string", default: "gpt-4o" },
         baseUrl: { type: "string", description: "Override for OpenAI-compatible endpoints" },
       },
       required: ["apiKey"],
@@ -75,12 +60,12 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
       $schema: "https://json-schema.org/draft/2020-12/schema",
       type: "object",
       properties: {
-        host:     { type: "string" },
-        port:     { type: "number", default: 5432 },
+        host: { type: "string" },
+        port: { type: "number", default: 5432 },
         database: { type: "string" },
-        user:     { type: "string" },
+        user: { type: "string" },
         password: { type: "string", "x-sensitive": true },
-        ssl:      { type: "boolean", default: false },
+        ssl: { type: "boolean", default: false },
       },
       required: ["host", "database", "user"],
     },
@@ -92,11 +77,11 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
       $schema: "https://json-schema.org/draft/2020-12/schema",
       type: "object",
       properties: {
-        bucket:          { type: "string" },
-        region:          { type: "string" },
-        accessKeyId:     { type: "string", "x-sensitive": true },
+        bucket: { type: "string" },
+        region: { type: "string" },
+        accessKeyId: { type: "string", "x-sensitive": true },
         secretAccessKey: { type: "string", "x-sensitive": true },
-        endpoint:        { type: "string", description: "S3-compatible endpoint URL (optional)" },
+        endpoint: { type: "string", description: "S3-compatible endpoint URL (optional)" },
       },
       required: ["bucket", "accessKeyId", "secretAccessKey"],
     },
@@ -108,8 +93,8 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
       $schema: "https://json-schema.org/draft/2020-12/schema",
       type: "object",
       properties: {
-        baseUrl:     { type: "string" },
-        apiKey:      { type: "string", "x-sensitive": true },
+        baseUrl: { type: "string" },
+        apiKey: { type: "string", "x-sensitive": true },
         bearerToken: { type: "string", "x-sensitive": true },
       },
       required: ["baseUrl"],
@@ -119,82 +104,7 @@ const BUILTIN_TYPES: ResourceTypeDef[] = [
 
 registerResourceType(...BUILTIN_TYPES)
 
-// ─── Sensitive-field detection ────────────────────────────────────────────────
-
-function sensitiveFields(typeName: string): Set<string> {
-  const type = typeRegistry.get(typeName)
-  if (!type) return new Set()
-  const props = (type.schema?.properties ?? {}) as Record<string, { "x-sensitive"?: boolean }>
-  return new Set(Object.entries(props).filter(([, v]) => v?.["x-sensitive"]).map(([k]) => k))
-}
-
-function warnSensitive(name: string, typeName: string, fields: Record<string, unknown>): void {
-  const sensitive = sensitiveFields(typeName)
-  const exposed = Object.keys(fields).filter((k) => sensitive.has(k))
-  if (exposed.length > 0) {
-    console.warn(
-      `[resources] resource "${name}" (${typeName}) has sensitive fields stored unencrypted: ${exposed.join(", ")}. Encryption pending (§16.1).`,
-    )
-  }
-}
-
-// ─── $env placeholder resolver ────────────────────────────────────────────────
-
-function resolveEnvPlaceholders(val: unknown, source: string): unknown {
-  if (val === null || typeof val !== "object") return val
-  if (Array.isArray(val)) return val.map((v) => resolveEnvPlaceholders(v, source))
-  const obj = val as Record<string, unknown>
-  if ("$env" in obj && typeof obj["$env"] === "string") {
-    const envVal = process.env[obj["$env"]]
-    if (envVal === undefined) {
-      throw new Error(`[resources] ${source}: $env placeholder "${obj["$env"]}" is not set at startup`)
-    }
-    return envVal
-  }
-  return Object.fromEntries(
-    Object.entries(obj).map(([k, v]) => [k, resolveEnvPlaceholders(v, source)]),
-  )
-}
-
-// ─── YAML resource loader ─────────────────────────────────────────────────────
-
-interface ResourceYaml {
-  type: string
-  name?: string
-  fields?: Record<string, unknown>
-}
-
-function loadResourceYamls(): Map<string, { type: string; fields: Record<string, unknown> }> {
-  const out = new Map<string, { type: string; fields: Record<string, unknown> }>()
-  if (!existsSync(RESOURCES_DIR)) return out
-
-  const files = readdirSync(RESOURCES_DIR).filter((f) => /\.ya?ml$/.test(f))
-  for (const file of files) {
-    const path = join(RESOURCES_DIR, file)
-    try {
-      const raw = readFileSync(path, "utf8")
-      const parsed = parseYaml(raw) as ResourceYaml
-      if (!parsed || typeof parsed !== "object" || !parsed.type) {
-        console.warn(`[resources] ${file}: missing required "type" field — skipped`)
-        continue
-      }
-      const stem  = basename(file, extname(file))
-      const name  = parsed.name ?? stem
-      const rawFields = parsed.fields ?? {}
-      const fields = resolveEnvPlaceholders(rawFields, file) as Record<string, unknown>
-      out.set(name, { type: parsed.type, fields })
-    } catch (err) {
-      if (err instanceof YamlError) {
-        console.warn(`[resources] ${file}: YAML parse error — ${err.message}`)
-      } else {
-        console.warn(`[resources] ${file}: ${(err as Error).message}`)
-      }
-    }
-  }
-  return out
-}
-
-// ─── DB resource loader ───────────────────────────────────────────────────────
+// ─── DB CRUD (used by API routes) ─────────────────────────────────────────────
 
 interface ResourceRow {
   id: number
@@ -205,79 +115,6 @@ interface ResourceRow {
   updated_at: string
 }
 
-function loadResourcesFromDb(db: Database): Map<string, { type: string; fields: Record<string, unknown> }> {
-  const out = new Map<string, { type: string; fields: Record<string, unknown> }>()
-  try {
-    const rows = db.prepare("SELECT name, type, fields_json FROM resources").all() as
-      { name: string; type: string; fields_json: string }[]
-    for (const row of rows) {
-      try {
-        const fields = JSON.parse(row.fields_json) as Record<string, unknown>
-        out.set(row.name, { type: row.type, fields })
-      } catch {
-        console.warn(`[resources] DB row "${row.name}": invalid fields_json — skipped`)
-      }
-    }
-  } catch (err) {
-    // Table may not exist yet (migration pending). Log and continue.
-    console.warn(`[resources] DB read failed: ${(err as Error).message}`)
-  }
-  return out
-}
-
-// ─── ENV-VAR resolver ─────────────────────────────────────────────────────────
-// Scans process.env for PLATFORM_RESOURCE_<NAME>_<FIELD>=value.
-// NAME and FIELD are uppercased in the env key; both are lowercased when
-// applied to ctx.resource. Supports single-level fields only (no nested ENV).
-
-function applyEnvOverrides(result: Map<string, { type: string; fields: Record<string, unknown> }>): void {
-  for (const [key, value] of Object.entries(process.env)) {
-    if (!value) continue
-    const m = key.match(/^PLATFORM_RESOURCE_([A-Z][A-Z0-9_]*)_([A-Z][A-Z0-9_]*)$/)
-    if (!m) continue
-    const name  = m[1].toLowerCase()
-    const field = m[2].toLowerCase()
-    const entry = result.get(name)
-    if (entry) {
-      entry.fields[field] = value
-    } else {
-      // ENV-only resource with no YAML or DB entry — create a minimal record.
-      result.set(name, { type: "Unknown", fields: { [field]: value } })
-    }
-  }
-}
-
-// ─── Main: build ctx.resource ─────────────────────────────────────────────────
-
-/** Merges all three tiers and returns the ctx.resource namespace for runPipeline. */
-export function buildResourceCtx(db: Database): Record<string, Record<string, unknown>> {
-  // Tier 3 (lowest): YAML files
-  const merged = loadResourceYamls()
-
-  // Tier 2: DB overrides YAML field-by-field
-  for (const [name, dbEntry] of loadResourcesFromDb(db)) {
-    const existing = merged.get(name)
-    if (existing) {
-      existing.fields = { ...existing.fields, ...dbEntry.fields }
-    } else {
-      merged.set(name, dbEntry)
-    }
-  }
-
-  // Tier 1 (highest): ENV-VAR overrides
-  applyEnvOverrides(merged)
-
-  // Flatten to Record<name, fields> and warn on exposed sensitive fields
-  const out: Record<string, Record<string, unknown>> = {}
-  for (const [name, { type, fields }] of merged) {
-    warnSensitive(name, type, fields)
-    out[name] = fields
-  }
-  return out
-}
-
-// ─── DB CRUD (used by API routes) ─────────────────────────────────────────────
-
 export interface ResourceRecord {
   id: number
   name: string
@@ -307,9 +144,12 @@ export function getResource(db: Database, name: string): ResourceRecord | null {
   ).get(name) as ResourceRow | undefined
   if (!row) return null
   return {
-    id: row.id, name: row.name, type: row.type,
+    id: row.id,
+    name: row.name,
+    type: row.type,
     fields: JSON.parse(row.fields_json) as Record<string, unknown>,
-    created_at: row.created_at, updated_at: row.updated_at,
+    created_at: row.created_at,
+    updated_at: row.updated_at,
   }
 }
 
@@ -338,13 +178,10 @@ export function patchResource(
   const existing = getResource(db, name)
   if (!existing) return null
   const merged = { ...existing.fields, ...fields }
-  db.prepare(
-    "UPDATE resources SET fields_json = ?, updated_at = datetime('now') WHERE name = ?",
-  ).run(JSON.stringify(merged), name)
-  return getResource(db, name)!
+  return upsertResource(db, name, existing.type, merged)
 }
 
 export function deleteResource(db: Database, name: string): boolean {
-  const result = db.prepare("DELETE FROM resources WHERE name = ?").run(name)
-  return result.changes > 0
+  const changed = db.prepare("DELETE FROM resources WHERE name = ?").run(name).changes
+  return changed > 0
 }

package/src/run-events.ts

@@ -0,0 +1,42 @@
+import type { ToistEvent } from "@toist/core"
+
+export interface RunEventBroker {
+  emit(event: ToistEvent): Promise<void>
+  finish(): void
+  subscribe(): AsyncIterable<ToistEvent>
+}
+
+export function createRunEventBroker(): RunEventBroker {
+  const events: ToistEvent[] = []
+  let finished = false
+  const waiters = new Set<() => void>()
+
+  return {
+    async emit(event) {
+      events.push(event)
+      for (const notify of waiters) notify()
+      waiters.clear()
+    },
+    finish() {
+      finished = true
+      for (const notify of waiters) notify()
+      waiters.clear()
+    },
+    subscribe() {
+      let index = 0
+      return {
+        [Symbol.asyncIterator]() {
+          return {
+            async next(): Promise<IteratorResult<ToistEvent>> {
+              while (index >= events.length) {
+                if (finished) return { value: undefined as never, done: true }
+                await new Promise<void>((resolve) => waiters.add(resolve))
+              }
+              return { value: events[index++], done: false }
+            },
+          }
+        },
+      }
+    },
+  }
+}