@toist/aja 0.5.0

package/src/resources.ts

@@ -0,0 +1,350 @@
+// 2121
+// Resource system — typed external-system handles.
+//
+// Three-tier override chain (highest priority first):
+//   1. ENV-VAR:  PLATFORM_RESOURCE_<NAME>_<FIELD>=value
+//   2. DB:       runtime.db resources table (set via UI or POST /resources)
+//   3. YAML:     <runner>/resources/*.yaml (committable; secrets use { $env: "VAR" })
+//
+// ctx.resource.<name>.<field> is the pipeline-side surface.
+// Fields from higher tiers override lower tiers field-by-field.
+//
+// Sensitive fields (x-sensitive:true in the Type schema) are stored
+// clear-text in v1. A load-time warning surfaces this. Encryption at
+// rest is deferred to pipeline-spec.md §16.1.
+
+import { existsSync, readdirSync, readFileSync } from "node:fs"
+import { basename, extname, join, dirname } from "node:path"
+import { fileURLToPath } from "node:url"
+import type { Database } from "bun:sqlite"
+import { parseYaml, YamlError } from "@toist/spec"
+import type { ResourceTypeDef } from "@toist/spec"
+
+const __dir = dirname(fileURLToPath(import.meta.url))
+// CWD is apps/runner/server/; resources/*.yaml lives at apps/runner/resources/
+const RESOURCES_DIR = join(__dir, "..", "..", "resources")
+
+// ─── Resource Type registry ───────────────────────────────────────────────────
+
+const typeRegistry = new Map<string, ResourceTypeDef>()
+
+export function registerResourceType(...types: ResourceTypeDef[]): void {
+  for (const t of types) typeRegistry.set(t.name, t)
+}
+
+export function getResourceType(name: string): ResourceTypeDef | undefined {
+  return typeRegistry.get(name)
+}
+
+export function listResourceTypes(): ResourceTypeDef[] {
+  return [...typeRegistry.values()]
+}
+
+const BUILTIN_TYPES: ResourceTypeDef[] = [
+  {
+    name: "AnthropicApi",
+    description: "Anthropic API credentials and default model config",
+    schema: {
+      $schema: "https://json-schema.org/draft/2020-12/schema",
+      type: "object",
+      properties: {
+        apiKey: { type: "string", "x-sensitive": true, description: "Anthropic API key (sk-ant-…)" },
+        model:  { type: "string", default: "claude-sonnet-4-6", description: "Default model ID" },
+      },
+      required: ["apiKey"],
+    },
+  },
+  {
+    name: "OpenAI",
+    description: "OpenAI API credentials",
+    schema: {
+      $schema: "https://json-schema.org/draft/2020-12/schema",
+      type: "object",
+      properties: {
+        apiKey:  { type: "string", "x-sensitive": true },
+        model:   { type: "string", default: "gpt-4o" },
+        baseUrl: { type: "string", description: "Override for OpenAI-compatible endpoints" },
+      },
+      required: ["apiKey"],
+    },
+  },
+  {
+    name: "Postgres",
+    description: "PostgreSQL database connection",
+    schema: {
+      $schema: "https://json-schema.org/draft/2020-12/schema",
+      type: "object",
+      properties: {
+        host:     { type: "string" },
+        port:     { type: "number", default: 5432 },
+        database: { type: "string" },
+        user:     { type: "string" },
+        password: { type: "string", "x-sensitive": true },
+        ssl:      { type: "boolean", default: false },
+      },
+      required: ["host", "database", "user"],
+    },
+  },
+  {
+    name: "S3",
+    description: "AWS S3 or S3-compatible object storage",
+    schema: {
+      $schema: "https://json-schema.org/draft/2020-12/schema",
+      type: "object",
+      properties: {
+        bucket:          { type: "string" },
+        region:          { type: "string" },
+        accessKeyId:     { type: "string", "x-sensitive": true },
+        secretAccessKey: { type: "string", "x-sensitive": true },
+        endpoint:        { type: "string", description: "S3-compatible endpoint URL (optional)" },
+      },
+      required: ["bucket", "accessKeyId", "secretAccessKey"],
+    },
+  },
+  {
+    name: "HttpApi",
+    description: "Generic HTTP API with base URL and optional auth token",
+    schema: {
+      $schema: "https://json-schema.org/draft/2020-12/schema",
+      type: "object",
+      properties: {
+        baseUrl:     { type: "string" },
+        apiKey:      { type: "string", "x-sensitive": true },
+        bearerToken: { type: "string", "x-sensitive": true },
+      },
+      required: ["baseUrl"],
+    },
+  },
+]
+
+registerResourceType(...BUILTIN_TYPES)
+
+// ─── Sensitive-field detection ────────────────────────────────────────────────
+
+function sensitiveFields(typeName: string): Set<string> {
+  const type = typeRegistry.get(typeName)
+  if (!type) return new Set()
+  const props = (type.schema?.properties ?? {}) as Record<string, { "x-sensitive"?: boolean }>
+  return new Set(Object.entries(props).filter(([, v]) => v?.["x-sensitive"]).map(([k]) => k))
+}
+
+function warnSensitive(name: string, typeName: string, fields: Record<string, unknown>): void {
+  const sensitive = sensitiveFields(typeName)
+  const exposed = Object.keys(fields).filter((k) => sensitive.has(k))
+  if (exposed.length > 0) {
+    console.warn(
+      `[resources] resource "${name}" (${typeName}) has sensitive fields stored unencrypted: ${exposed.join(", ")}. Encryption pending (§16.1).`,
+    )
+  }
+}
+
+// ─── $env placeholder resolver ────────────────────────────────────────────────
+
+function resolveEnvPlaceholders(val: unknown, source: string): unknown {
+  if (val === null || typeof val !== "object") return val
+  if (Array.isArray(val)) return val.map((v) => resolveEnvPlaceholders(v, source))
+  const obj = val as Record<string, unknown>
+  if ("$env" in obj && typeof obj["$env"] === "string") {
+    const envVal = process.env[obj["$env"]]
+    if (envVal === undefined) {
+      throw new Error(`[resources] ${source}: $env placeholder "${obj["$env"]}" is not set at startup`)
+    }
+    return envVal
+  }
+  return Object.fromEntries(
+    Object.entries(obj).map(([k, v]) => [k, resolveEnvPlaceholders(v, source)]),
+  )
+}
+
+// ─── YAML resource loader ─────────────────────────────────────────────────────
+
+interface ResourceYaml {
+  type: string
+  name?: string
+  fields?: Record<string, unknown>
+}
+
+function loadResourceYamls(): Map<string, { type: string; fields: Record<string, unknown> }> {
+  const out = new Map<string, { type: string; fields: Record<string, unknown> }>()
+  if (!existsSync(RESOURCES_DIR)) return out
+
+  const files = readdirSync(RESOURCES_DIR).filter((f) => /\.ya?ml$/.test(f))
+  for (const file of files) {
+    const path = join(RESOURCES_DIR, file)
+    try {
+      const raw = readFileSync(path, "utf8")
+      const parsed = parseYaml(raw) as ResourceYaml
+      if (!parsed || typeof parsed !== "object" || !parsed.type) {
+        console.warn(`[resources] ${file}: missing required "type" field — skipped`)
+        continue
+      }
+      const stem  = basename(file, extname(file))
+      const name  = parsed.name ?? stem
+      const rawFields = parsed.fields ?? {}
+      const fields = resolveEnvPlaceholders(rawFields, file) as Record<string, unknown>
+      out.set(name, { type: parsed.type, fields })
+    } catch (err) {
+      if (err instanceof YamlError) {
+        console.warn(`[resources] ${file}: YAML parse error — ${err.message}`)
+      } else {
+        console.warn(`[resources] ${file}: ${(err as Error).message}`)
+      }
+    }
+  }
+  return out
+}
+
+// ─── DB resource loader ───────────────────────────────────────────────────────
+
+interface ResourceRow {
+  id: number
+  name: string
+  type: string
+  fields_json: string
+  created_at: string
+  updated_at: string
+}
+
+function loadResourcesFromDb(db: Database): Map<string, { type: string; fields: Record<string, unknown> }> {
+  const out = new Map<string, { type: string; fields: Record<string, unknown> }>()
+  try {
+    const rows = db.prepare("SELECT name, type, fields_json FROM resources").all() as
+      { name: string; type: string; fields_json: string }[]
+    for (const row of rows) {
+      try {
+        const fields = JSON.parse(row.fields_json) as Record<string, unknown>
+        out.set(row.name, { type: row.type, fields })
+      } catch {
+        console.warn(`[resources] DB row "${row.name}": invalid fields_json — skipped`)
+      }
+    }
+  } catch (err) {
+    // Table may not exist yet (migration pending). Log and continue.
+    console.warn(`[resources] DB read failed: ${(err as Error).message}`)
+  }
+  return out
+}
+
+// ─── ENV-VAR resolver ─────────────────────────────────────────────────────────
+// Scans process.env for PLATFORM_RESOURCE_<NAME>_<FIELD>=value.
+// NAME and FIELD are uppercased in the env key; both are lowercased when
+// applied to ctx.resource. Supports single-level fields only (no nested ENV).
+
+function applyEnvOverrides(result: Map<string, { type: string; fields: Record<string, unknown> }>): void {
+  for (const [key, value] of Object.entries(process.env)) {
+    if (!value) continue
+    const m = key.match(/^PLATFORM_RESOURCE_([A-Z][A-Z0-9_]*)_([A-Z][A-Z0-9_]*)$/)
+    if (!m) continue
+    const name  = m[1].toLowerCase()
+    const field = m[2].toLowerCase()
+    const entry = result.get(name)
+    if (entry) {
+      entry.fields[field] = value
+    } else {
+      // ENV-only resource with no YAML or DB entry — create a minimal record.
+      result.set(name, { type: "Unknown", fields: { [field]: value } })
+    }
+  }
+}
+
+// ─── Main: build ctx.resource ─────────────────────────────────────────────────
+
+/** Merges all three tiers and returns the ctx.resource namespace for runPipeline. */
+export function buildResourceCtx(db: Database): Record<string, Record<string, unknown>> {
+  // Tier 3 (lowest): YAML files
+  const merged = loadResourceYamls()
+
+  // Tier 2: DB overrides YAML field-by-field
+  for (const [name, dbEntry] of loadResourcesFromDb(db)) {
+    const existing = merged.get(name)
+    if (existing) {
+      existing.fields = { ...existing.fields, ...dbEntry.fields }
+    } else {
+      merged.set(name, dbEntry)
+    }
+  }
+
+  // Tier 1 (highest): ENV-VAR overrides
+  applyEnvOverrides(merged)
+
+  // Flatten to Record<name, fields> and warn on exposed sensitive fields
+  const out: Record<string, Record<string, unknown>> = {}
+  for (const [name, { type, fields }] of merged) {
+    warnSensitive(name, type, fields)
+    out[name] = fields
+  }
+  return out
+}
+
+// ─── DB CRUD (used by API routes) ─────────────────────────────────────────────
+
+export interface ResourceRecord {
+  id: number
+  name: string
+  type: string
+  fields: Record<string, unknown>
+  created_at: string
+  updated_at: string
+}
+
+export function listResources(db: Database): ResourceRecord[] {
+  const rows = db.prepare(
+    "SELECT id, name, type, fields_json, created_at, updated_at FROM resources ORDER BY name",
+  ).all() as ResourceRow[]
+  return rows.map((r) => ({
+    id: r.id,
+    name: r.name,
+    type: r.type,
+    fields: JSON.parse(r.fields_json) as Record<string, unknown>,
+    created_at: r.created_at,
+    updated_at: r.updated_at,
+  }))
+}
+
+export function getResource(db: Database, name: string): ResourceRecord | null {
+  const row = db.prepare(
+    "SELECT id, name, type, fields_json, created_at, updated_at FROM resources WHERE name = ?",
+  ).get(name) as ResourceRow | undefined
+  if (!row) return null
+  return {
+    id: row.id, name: row.name, type: row.type,
+    fields: JSON.parse(row.fields_json) as Record<string, unknown>,
+    created_at: row.created_at, updated_at: row.updated_at,
+  }
+}
+
+export function upsertResource(
+  db: Database,
+  name: string,
+  type: string,
+  fields: Record<string, unknown>,
+): ResourceRecord {
+  db.prepare(`
+    INSERT INTO resources (name, type, fields_json)
+    VALUES (?, ?, ?)
+    ON CONFLICT(name) DO UPDATE SET
+      type        = excluded.type,
+      fields_json = excluded.fields_json,
+      updated_at  = datetime('now')
+  `).run(name, type, JSON.stringify(fields))
+  return getResource(db, name)!
+}
+
+export function patchResource(
+  db: Database,
+  name: string,
+  fields: Record<string, unknown>,
+): ResourceRecord | null {
+  const existing = getResource(db, name)
+  if (!existing) return null
+  const merged = { ...existing.fields, ...fields }
+  db.prepare(
+    "UPDATE resources SET fields_json = ?, updated_at = datetime('now') WHERE name = ?",
+  ).run(JSON.stringify(merged), name)
+  return getResource(db, name)!
+}
+
+export function deleteResource(db: Database, name: string): boolean {
+  const result = db.prepare("DELETE FROM resources WHERE name = ?").run(name)
+  return result.changes > 0
+}

package/src/runs.ts

@@ -0,0 +1,53 @@
+// 2121
+// Read-only capability over the runtime ledger, exposed to kinds as
+// ctx.runs. Powers the runs.lastOutput and runs.nodeOutput kinds (M3, M4
+// in pipeline-spec.md §12). Kept here rather than in runtime-db.ts because
+// the latter is intentionally not exposed to kinds — kinds see ctx.db
+// (data store) and ctx.cache (capability), not the runtime tables.
+
+import type { Database } from "bun:sqlite"
+import type { RunStore } from "@toist/spec"
+
+interface RunRow {
+  id: number
+  result: string | null
+}
+
+interface NodeOutputRow {
+  output_json: string | null
+}
+
+export function makeRunStore(runtimeDb: Database): RunStore {
+  return {
+    lastOutput(pipeline, opts) {
+      const status = opts?.status ?? "done"
+      const row = runtimeDb.prepare(
+        "SELECT id, result FROM runs WHERE pipeline = ? AND status = ? ORDER BY id DESC LIMIT 1",
+      ).get(pipeline, status) as RunRow | undefined
+      if (!row || row.result === null) return null
+      try {
+        return JSON.parse(row.result) as unknown
+      } catch {
+        return null
+      }
+    },
+
+    nodeOutput(pipeline, node, opts) {
+      const status = opts?.status ?? "done"
+      const row = runtimeDb.prepare(
+        `SELECT no.output_json
+           FROM node_outputs no
+           JOIN runs r ON no.run_id = r.id
+          WHERE r.pipeline = ? AND r.status = ? AND no.node_id = ?
+          ORDER BY r.id DESC
+          LIMIT 1`,
+      ).get(pipeline, status, node) as NodeOutputRow | undefined
+      if (!row || row.output_json === null) return null
+      try {
+        return JSON.parse(row.output_json) as unknown
+      } catch {
+        return null
+      }
+    },
+  }
+}

package/src/runtime-db.ts

@@ -0,0 +1,48 @@
+// 2121
+// Platform-internal runtime ledger. Holds:
+//   - runs        — every pipeline execution (status, payload, result, steps, timing)
+//   - logs        — log lines emitted from kinds during a run, scoped to run_id
+//   - tasks       — HITL pending tasks (added in migration 002)
+//   - node_outputs — checkpoint-between-nodes memoization (added in migration 002)
+//
+// Schema is owned by the platform and evolved via numbered SQL migrations
+// in ../migrations/. See migrate.ts for the runner.
+//
+// Kinds never receive a handle to this DB — they only see ctx.db (data
+// store) and ctx.cache (capability), so they cannot accidentally touch
+// runtime tables.
+//
+// Migration policy: by default the runner applies pending migrations at
+// startup. RUNTIME_AUTO_MIGRATE=false flips to fail-loud — startup aborts if
+// anything is pending.
+//
+// Lifecycle: openRuntimeDb() is a pure factory. The runner lock is acquired
+// separately by db-handles.initDbs() before this is called, so two callers
+// cannot race on migrations.
+
+import { Database } from "bun:sqlite"
+import { mkdirSync } from "node:fs"
+import { dirname } from "node:path"
+import { runMigrations, pendingMigrations } from "./migrate.ts"
+import { runtimeDbPath } from "./config.ts"
+
+export function openRuntimeDb(): Database {
+  const path = runtimeDbPath()
+  mkdirSync(dirname(path), { recursive: true })
+
+  if (process.env.RUNTIME_AUTO_MIGRATE === "false") {
+    const pending = pendingMigrations(path)
+    if (pending.length > 0) {
+      console.error(
+        `[runtime-db] ${pending.length} pending migration(s); RUNTIME_AUTO_MIGRATE=false.`,
+      )
+      console.error(`[runtime-db] pending: ${pending.join(", ")}`)
+      console.error(`[runtime-db] run migrations explicitly, or unset RUNTIME_AUTO_MIGRATE.`)
+      process.exit(1)
+    }
+  } else {
+    runMigrations(path)
+  }
+
+  return new Database(path, { create: true })
+}