@toist/in 0.7.1 → 0.8.0

package/src/index.ts

@@ -1,37 +1,2 @@
 #!/usr/bin/env bun
-// 2121 toist
-// @toist/in — interactive setup wizard for Toist.
-//
-// Usage:
-//   bunx @toist/in              run wizard → writes toist.yml + creates dirs
-//   bunx @toist/in upgrade      upgrade @toist/* deps in current directory
-
-import { wizard } from "./wizard.ts"
-import { upgrade } from "./upgrade.ts"
-
-const args = process.argv.slice(2)
-const arg = args[0]
-
-if (arg === "--help" || arg === "-h") {
-  console.log("bunx @toist/in              run setup wizard → writes toist.yml")
-  console.log("bunx @toist/in upgrade      upgrade @toist/* deps")
-  console.log("")
-  console.log("To start the runner after setup:")
-  console.log("  bunx @toist/aja")
-  console.log("")
-  console.log("Full docs: https://toist.in")
-  process.exit(0)
-}
-
-if (!arg) {
-  await wizard()
-} else if (arg === "upgrade") {
-  await upgrade()
-} else {
-  console.error(`Unknown command: ${arg}`)
-  console.error("")
-  console.error("Usage:")
-  console.error("  bunx @toist/in            run setup wizard")
-  console.error("  bunx @toist/in upgrade    upgrade @toist/* deps")
-  process.exit(1)
-}
+import "./cli.ts"

package/src/runtime.ts

@@ -0,0 +1,75 @@
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { join, resolve, dirname } from "node:path"
+import { homedir } from "node:os"
+import { createMemoryRuntime, runSpec, type ResumableRuntime, type RunSpecResult, type ToistRuntime } from "@toist/core"
+import { createSqliteRuntime } from "@toist/aja"
+
+interface RuntimeWithClose extends ToistRuntime {
+  close?: () => Promise<void>
+}
+
+export interface StoredRunDescriptor {
+  spec: unknown
+  payload: Record<string, unknown>
+}
+
+export function getCliStateDir(): string {
+  const xdg = process.env.XDG_STATE_HOME
+  if (xdg) return resolve(xdg, "toist")
+
+  const home = process.env.HOME || homedir()
+  if (home) return resolve(home, ".toist")
+
+  return resolve(process.cwd(), ".toist")
+}
+
+export async function getCliRuntime(options: { ephemeral?: boolean } = {}): Promise<RuntimeWithClose> {
+  if (options.ephemeral) return createMemoryRuntime()
+
+  const rootDir = getCliStateDir()
+  mkdirSync(rootDir, { recursive: true })
+  return await createSqliteRuntime({ rootDir, dataDir: rootDir })
+}
+
+export function getRunDescriptorPath(runId: number): string {
+  return join(getCliStateDir(), "runs", `${runId}.json`)
+}
+
+export function storeRunDescriptor(runId: number, descriptor: StoredRunDescriptor): void {
+  const path = getRunDescriptorPath(runId)
+  mkdirSync(dirname(path), { recursive: true })
+  writeFileSync(path, JSON.stringify(descriptor, null, 2))
+}
+
+export function loadRunDescriptor(runId: number): StoredRunDescriptor | null {
+  const path = getRunDescriptorPath(runId)
+  if (!existsSync(path)) return null
+  return JSON.parse(readFileSync(path, "utf8")) as StoredRunDescriptor
+}
+
+export function deleteRunDescriptor(runId: number): void {
+  const path = getRunDescriptorPath(runId)
+  if (existsSync(path)) rmSync(path, { force: true })
+}
+
+export function isResumableRuntime(runtime: ToistRuntime): runtime is ResumableRuntime {
+  return typeof (runtime as ResumableRuntime).resumeRun === "function"
+}
+
+export async function resumeLocalStoredRun(
+  runtime: ToistRuntime,
+  runId: number,
+  descriptor: StoredRunDescriptor,
+  response: { taskId: number; token: string; response: unknown; respondedBy?: string },
+): Promise<RunSpecResult> {
+  await runtime.tasks.answer(response.taskId, response.token, response.response, response.respondedBy ?? "cli")
+  const resumeOutputs = await runtime.outputs.list(runId)
+  await runtime.runs.markRunning(runId, { trigger: "resumed", clearCurrentNode: true })
+  return await runSpec(descriptor.spec, descriptor.payload, {
+    runtime,
+    resumeRunId: runId,
+    resumeOutputs,
+    trigger: "resumed",
+    events: true,
+  })
+}

package/src/wizard.ts

@@ -1,144 +1,91 @@
 // 2121 toist
-// Interactive setup wizard for @toist/in.
-// Asks a few questions, writes toist.yml, creates directories.
 
-import { existsSync, mkdirSync, writeFileSync } from "node:fs"
-import { createInterface } from "node:readline"
-import { basename, resolve } from "node:path"
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"
+import { basename, dirname, join, resolve } from "node:path"
+import { createInterface } from "node:readline/promises"
 
-// Line-by-line reader that works with both TTY and piped stdin.
-function makeReader() {
-  const iface = createInterface({ input: process.stdin, output: process.stdout, terminal: false })
-  const lines: string[] = []
-  let resolve: ((v: string) => void) | null = null
+const TEMPLATES = [
+  { id: "default", label: "Durable runner (default)" },
+  { id: "embedded", label: "Embedded `.toist/` runner" },
+  { id: "embedded-memory", label: "Memory-only (library use)" },
+] as const
 
-  iface.on("line", (line) => {
-    if (resolve) {
-      const r = resolve
-      resolve = null
-      r(line)
-    } else {
-      lines.push(line)
-    }
-  })
-
-  iface.on("close", () => {
-    if (resolve) resolve("")
-  })
+type TemplateId = typeof TEMPLATES[number]["id"]
 
-  return {
-    read(): Promise<string> {
-      if (lines.length > 0) return Promise.resolve(lines.shift()!)
-      return new Promise((r) => { resolve = r })
-    },
-    close() { iface.close() },
-  }
+function templatesDir(): string {
+  return resolve(import.meta.dir, "..", "templates")
 }
 
-async function prompt(reader: ReturnType<typeof makeReader>, label: string, def: string): Promise<string> {
-  process.stdout.write(`${label} [${def}] `)
-  const v = (await reader.read()).trim()
-  return v || def
+function suggestTemplate(dest: string): TemplateId {
+  if (basename(dest) === ".toist") return "embedded"
+  return "default"
 }
 
-async function yn(reader: ReturnType<typeof makeReader>, label: string, def = false): Promise<boolean> {
-  const hint = def ? "Y/n" : "y/N"
-  process.stdout.write(`${label} [${hint}] `)
-  const v = (await reader.read()).trim().toLowerCase()
-  if (!v) return def
-  return v === "y" || v === "yes"
+function packageNameFor(dest: string): string {
+  const base = basename(dest)
+  if (base === ".toist") return `${basename(dirname(dest)) || "toist"}-toist`
+  return base || "toist"
 }
 
-export async function wizard(cwd = process.cwd()): Promise<void> {
-  const reader = makeReader()
-
-  console.log("")
-  console.log("Welcome to Toist. Let's set up your project.")
-  console.log("")
-
-  const defaultName = basename(cwd)
-  const name      = await prompt(reader, "Instance name?    ", defaultName)
-  const pipelines = await prompt(reader, "Pipelines dir?    ", "pipelines")
-  const resources = await prompt(reader, "Resources dir?    ", "resources")
-  const data      = await prompt(reader, "Data dir?         ", "data")
-  const port      = await prompt(reader, "Port?             ", "3000")
+function copyTemplate(src: string, dest: string) {
+  mkdirSync(dest, { recursive: true })
+  for (const entry of readdirSync(src)) {
+    const srcPath = join(src, entry)
+    const outName = entry === "_gitignore" ? ".gitignore" : entry === "_package.json" ? "package.json" : entry
+    const destPath = join(dest, outName)
+    const info = statSync(srcPath)
+    if (info.isDirectory()) {
+      copyTemplate(srcPath, destPath)
+    } else {
+      copyFileSync(srcPath, destPath)
+    }
+  }
+}
 
-  console.log("")
-  const hasKinds = await yn(reader, "Register custom TypeScript kinds?")
+function rewritePackageName(dest: string) {
+  const path = join(dest, "package.json")
+  if (!existsSync(path)) return
+  const pkg = JSON.parse(readFileSync(path, "utf8")) as { name?: string }
+  pkg.name = packageNameFor(dest)
+  writeFileSync(path, JSON.stringify(pkg, null, 2) + "\n")
+}
 
-  const kinds: string[] = []
-  if (hasKinds) {
+async function chooseTemplate(defaultTemplate: TemplateId): Promise<TemplateId> {
+  const rl = createInterface({ input: process.stdin, output: process.stdout })
+  try {
     console.log("")
-    let more = true
-    while (more) {
-      const kindFile = await prompt(reader, "  Kind file path? ", "src/kinds/index.ts")
-      kinds.push(kindFile)
-      more = await yn(reader, "  Add another?    ")
+    console.log("How will you run pipelines?")
+    for (const [index, template] of TEMPLATES.entries()) {
+      console.log(`  ${index + 1}) ${template.label}`)
     }
+    const defaultIndex = TEMPLATES.findIndex((template) => template.id === defaultTemplate) + 1
+    const answer = (await rl.question(`Choice [${defaultIndex}]: `)).trim()
+    const picked = Number(answer || defaultIndex)
+    return TEMPLATES[picked - 1]?.id ?? defaultTemplate
+  } finally {
+    rl.close()
   }
+}
 
-  reader.close()
-
-  // ---------------------------------------------------------------------------
-  // Write toist.yml
-  // ---------------------------------------------------------------------------
-
-  const kindsBlock = kinds.length > 0
-    ? `\nkinds:\n${kinds.map((k) => `  - ${k}`).join("\n")}`
-    : ""
-
-  const configContent = [
-    `# toist.yml — generated by bunx @toist/in`,
-    `# Run with: bunx @toist/aja`,
-    ``,
-    `instance: ${name}`,
-    `port: ${port}`,
-    `pipelines: ./${pipelines}`,
-    `resources: ./${resources}`,
-    `data: ./${data}`,
-    ...(kinds.length > 0 ? [``, `kinds:`, ...kinds.map((k) => `  - ${k}`)] : []),
-    ``,
-  ].join("\n")
-
-  const configPath = resolve(cwd, "toist.yml")
-  const alreadyExists = existsSync(configPath)
-
-  console.log("")
+export async function wizard(cwd = process.cwd(), destArg?: string): Promise<void> {
+  const destName = destArg ?? "toist"
+  const templateId = await chooseTemplate(suggestTemplate(destName))
+  const templateDir = join(templatesDir(), templateId)
+  const dest = resolve(cwd, destName)
 
-  if (alreadyExists) {
-    console.log("toist.yml already exists — skipping (remove it to re-run wizard).")
-  } else {
-    writeFileSync(configPath, configContent, "utf8")
-    console.log("Wrote toist.yml")
+  if (!existsSync(templateDir)) {
+    console.error(`Template not found: ${templateId}`)
+    process.exit(1)
   }
 
-  // ---------------------------------------------------------------------------
-  // Create directories
-  // ---------------------------------------------------------------------------
-
-  for (const dir of [pipelines, resources, data]) {
-    const abs = resolve(cwd, dir)
-    if (!existsSync(abs)) {
-      mkdirSync(abs, { recursive: true })
-      // Keep data/ out of git, commit the rest
-      if (dir === data) {
-        writeFileSync(resolve(abs, ".gitignore"), "*.db\n*.db-shm\n*.db-wal\n.lock/\n")
-      } else {
-        writeFileSync(resolve(abs, ".gitkeep"), "")
-      }
-      console.log(`Created ${dir}/`)
-    }
+  if (existsSync(dest)) {
+    console.error(`Destination already exists: ${dest}`)
+    process.exit(1)
   }
 
-  // ---------------------------------------------------------------------------
-  // Done
-  // ---------------------------------------------------------------------------
+  copyTemplate(templateDir, dest)
+  rewritePackageName(dest)
 
-  console.log("")
-  console.log("Done. Start with:")
-  console.log("  bunx @toist/aja")
-  console.log("")
-  console.log("Or add a script to package.json:")
-  console.log('  "toist": "bunx @toist/aja"')
-  console.log("")
+  const relative = dest.startsWith(cwd) ? `./${dest.slice(cwd.length).replace(/^\/+/, "") || basename(dest)}` : dest
+  console.log(`✓ created ${relative} (${templateId} template)`)
 }

package/templates/default/README.md

@@ -0,0 +1,35 @@
+# my-toist-instance
+
+Generated with [`bunx @toist/in`](https://toist.in/docs).
+
+This directory is a Toist home. It can be used either as a dedicated
+instance directory or as an embedded repo-owned home such as
+`<repo>/.toist/`.
+
+## First run
+
+1. `bun install`
+2. `bun start.ts` — runner boots on `http://localhost:3000`.
+
+Open `http://localhost:3000` to use the UI.
+
+## Layout
+
+- `start.ts` — bootstrap; calls `startRunner` from `@toist/aja`
+- `kinds/custom.ts` — register your domain-specific kinds here (or import them from normal workspace code)
+- `pipelines/*.yaml` — pipeline definitions (per pipeline-spec)
+- `toist.yml` — resource config for this Toist home
+- `data/` — runtime SQLite databases (gitignored)
+
+## Upgrading
+
+```sh
+bunx @toist/in upgrade
+```
+
+Run from the instance directory or host repo root. Bumps every
+`@toist/*` dep in `package.json` to the latest version.
+
+## More
+
+Full quickstart and docs: https://toist.in/docs

package/templates/default/_gitignore

@@ -0,0 +1,7 @@
+node_modules/
+bun.lock
+data/*.db
+data/*.db-shm
+data/*.db-wal
+data/.lock/
+.platform/

package/templates/default/_package.json

@@ -0,0 +1,13 @@
+{
+  "name": "my-toist-instance",
+  "version": "0.1.0",
+  "type": "module",
+  "scripts": {
+    "start": "bun start.ts",
+    "dev": "bun --watch start.ts"
+  },
+  "dependencies": {
+    "@toist/aja": "^0.8.0",
+    "@toist/spec": "^0.8.0"
+  }
+}

package/templates/default/kinds/custom.ts

@@ -0,0 +1,23 @@
+// 2121 toist — register your domain-specific kinds here.
+//
+// Each export should be a NodeKind<P, I>. See the @toist/aja built-ins for
+// reference shapes, and pipeline-spec / kind-spec at https://toist.in/docs for
+// the full contract.
+//
+// Example:
+//
+//   import type { NodeKind } from "@toist/spec"
+//
+//   export const greet: NodeKind<{ name: string }, { msg: string }> = {
+//     id: "demo.greet",
+//     params: { name: { type: "string", required: true } },
+//     async execute(ctx) {
+//       return { msg: `Hello, ${ctx.params.name}!` }
+//     },
+//   }
+//
+// Then in start.ts:
+//   import * as customKinds from "./kinds/custom.ts"
+//   register(...Object.values(customKinds))
+
+export {}

package/templates/default/start.ts

@@ -0,0 +1,14 @@
+// 2121 toist instance bootstrap.
+// See https://toist.in/docs for the full quickstart.
+
+import { startRunner } from "@toist/aja"
+
+// Uncomment to register domain-specific kinds:
+// import { register } from "@toist/aja"
+// import * as customKinds from "./kinds/custom.ts"
+// register(...Object.values(customKinds))
+
+await startRunner({
+  port:    Number(process.env.PORT ?? 3000),
+  rootDir: import.meta.dir,
+})

package/templates/default/toist.yml

@@ -0,0 +1,2 @@
+# toist.yml — resource config for this Toist home
+resources: {}

package/templates/embedded/README.md

@@ -0,0 +1,58 @@
+# my-repo/.toist
+
+Generated with [`bunx @toist/in .toist`](https://toist.in/docs).
+
+This directory is the embedded Toist home for an existing repo.
+`@toist/aja` runs pipelines; `.toist/` is the repo-owned local home for
+Toist config, pipeline definitions, and runtime state.
+
+## First run
+
+1. From the host repo root: `bun add @toist/aja @toist/spec`
+2. Optionally, add a script to your root `package.json`:
+   ```json
+   "scripts": { "toist": "bun .toist/start.ts" }
+   ```
+3. Start the embedded instance: `bun .toist/start.ts`
+
+Open `http://localhost:3000` to use the UI.
+
+## Layout
+
+```
+.toist/
+  start.ts        — embedded bootstrap; calls startRunner from @toist/aja
+  instance.json   — instance name and tier (optional)
+  pipelines/      — pipeline YAML files owned by this repo
+  toist.yml       — resource config for this embedded home
+  data/           — runtime SQLite databases (gitignored)
+```
+
+Custom TypeScript kinds stay in your host repo's normal source tree or
+workspace packages. Import and register them from `.toist/start.ts`:
+
+```ts
+import { register } from "@toist/aja"
+import * as myKinds from "../src/toist-kinds.ts"
+register(...Object.values(myKinds))
+```
+
+## Environment
+
+| Variable | Default | Notes |
+|---|---|---|
+| `TOIST_PORT` | `3000` | Port the runner listens on |
+| `PORT` | `3000` | Fallback if `TOIST_PORT` is unset |
+
+## Upgrading
+
+```sh
+bunx @toist/in upgrade
+```
+
+Run from `.toist/` or from the host repo root. Finds `package.json`
+and bumps every `@toist/*` dependency to the latest version.
+
+## More
+
+Full quickstart and docs: https://toist.in/docs

package/templates/embedded/_gitignore

@@ -0,0 +1,5 @@
+data/*.db
+data/*.db-shm
+data/*.db-wal
+data/.lock/
+.platform/

package/templates/embedded/instance.json

@@ -0,0 +1,4 @@
+{
+  "instanceName": "my-project",
+  "tier": "local"
+}

package/templates/embedded/start.ts

@@ -0,0 +1,24 @@
+// 2121 toist — embedded bootstrap for .toist/
+// See https://toist.in/docs for the full quickstart.
+//
+// rootDir is set to this directory (.toist/).
+// The runner resolves all paths relative to it:
+//   .toist/pipelines/   — pipeline YAML files
+//   .toist/toist.yml    — resource config
+//   .toist/data/        — runtime SQLite databases (gitignored)
+//
+// Custom kinds belong in your host repo's normal source tree.
+// Import and register them here before calling startRunner.
+//
+// Examples:
+//   import * as myKinds from "../src/toist-kinds.ts"
+//   import * as myKinds from "../packages/my-kinds/src/index.ts"
+//   import { register } from "@toist/aja"
+//   register(...Object.values(myKinds))
+
+import { startRunner } from "@toist/aja"
+
+await startRunner({
+  port:    Number(process.env.TOIST_PORT ?? process.env.PORT ?? 3000),
+  rootDir: import.meta.dir,  // = <repo>/.toist/
+})

package/templates/embedded/toist.yml

@@ -0,0 +1,2 @@
+# toist.yml — resource config for this embedded Toist home
+resources: {}

package/templates/embedded-memory/README.md

@@ -0,0 +1,12 @@
+# memory-only Toist host
+
+Generated with `bunx @toist/in` using the memory-only template.
+
+This template uses `runSpec()` and `createMemoryRuntime()` directly. There is no HTTP runner, no UI, and no durable SQLite state.
+
+## First run
+
+1. Add `@toist/core` and `@toist/spec` to your host repo.
+2. Run `bun start.ts`.
+
+The example reads `./pipelines/hello.yml` and prints the result JSON.

package/templates/embedded-memory/_gitignore

@@ -0,0 +1 @@
+node_modules

package/templates/embedded-memory/pipelines/hello.yml

@@ -0,0 +1,12 @@
+apiVersion: "2121.fi/v1"
+id: hello
+nodes:
+  - id: greet
+    kind: data.json
+    params:
+      value: { expr: '"hello " + ctx.params.name' }
+  - id: out
+    kind: sink
+    dependsOn: [greet]
+    input:
+      value: { expr: "ctx.results.greet" }

package/templates/embedded-memory/start.ts

@@ -0,0 +1,16 @@
+import { readFileSync } from "node:fs"
+import { join } from "node:path"
+import { parseYaml } from "@toist/spec"
+import { createMemoryRuntime, runSpec } from "@toist/core"
+
+const spec = parseYaml(readFileSync(join(import.meta.dir, "pipelines", "hello.yml"), "utf8"))
+const runtime = createMemoryRuntime()
+const payload = process.argv[2] ? JSON.parse(process.argv[2]) : { name: "world" }
+const result = await runSpec(spec, payload, { runtime, events: true })
+
+for await (const event of result.events ?? []) {
+  console.error(`[event] ${event.type}`)
+}
+
+const { events: _events, ...printable } = result
+console.log(JSON.stringify(printable, null, 2))