@toist/in 0.6.1 → 0.8.0

package/src/index.ts

@@ -1,38 +1,2 @@
 #!/usr/bin/env bun
-// 2121 toist
-// @toist/in — scaffold a new toist instance, or upgrade an existing one.
-//
-// Usage:
-//   bunx @toist/in <path>   scaffold a new instance at <path>
-//   bunx @toist/in          upgrade @toist/* deps in the current directory
-
-import { existsSync } from "node:fs"
-import { resolve } from "node:path"
-import { scaffold } from "./scaffold.ts"
-import { upgrade } from "./upgrade.ts"
-
-const args = process.argv.slice(2)
-const arg = args[0]
-
-if (arg === "--help" || arg === "-h") {
-  console.log("bunx @toist/in <path>   scaffold a new toist instance at <path>")
-  console.log("bunx @toist/in          upgrade @toist/* deps in the current directory")
-  console.log("")
-  console.log("Full docs: https://toist.in")
-  process.exit(0)
-}
-
-if (!arg) {
-  // No path given — upgrade if we're inside a toist instance, else error.
-  const pkgPath = resolve(process.cwd(), "package.json")
-  if (!existsSync(pkgPath)) {
-    console.error("No package.json found in the current directory.")
-    console.error("")
-    console.error("To scaffold a new instance:  bunx @toist/in <path>")
-    console.error("To upgrade an existing one:  run from the instance directory")
-    process.exit(1)
-  }
-  await upgrade()
-} else {
-  await scaffold(arg)
-}
+import "./cli.ts"

package/src/runtime.ts

@@ -0,0 +1,75 @@
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { join, resolve, dirname } from "node:path"
+import { homedir } from "node:os"
+import { createMemoryRuntime, runSpec, type ResumableRuntime, type RunSpecResult, type ToistRuntime } from "@toist/core"
+import { createSqliteRuntime } from "@toist/aja"
+
+interface RuntimeWithClose extends ToistRuntime {
+  close?: () => Promise<void>
+}
+
+export interface StoredRunDescriptor {
+  spec: unknown
+  payload: Record<string, unknown>
+}
+
+export function getCliStateDir(): string {
+  const xdg = process.env.XDG_STATE_HOME
+  if (xdg) return resolve(xdg, "toist")
+
+  const home = process.env.HOME || homedir()
+  if (home) return resolve(home, ".toist")
+
+  return resolve(process.cwd(), ".toist")
+}
+
+export async function getCliRuntime(options: { ephemeral?: boolean } = {}): Promise<RuntimeWithClose> {
+  if (options.ephemeral) return createMemoryRuntime()
+
+  const rootDir = getCliStateDir()
+  mkdirSync(rootDir, { recursive: true })
+  return await createSqliteRuntime({ rootDir, dataDir: rootDir })
+}
+
+export function getRunDescriptorPath(runId: number): string {
+  return join(getCliStateDir(), "runs", `${runId}.json`)
+}
+
+export function storeRunDescriptor(runId: number, descriptor: StoredRunDescriptor): void {
+  const path = getRunDescriptorPath(runId)
+  mkdirSync(dirname(path), { recursive: true })
+  writeFileSync(path, JSON.stringify(descriptor, null, 2))
+}
+
+export function loadRunDescriptor(runId: number): StoredRunDescriptor | null {
+  const path = getRunDescriptorPath(runId)
+  if (!existsSync(path)) return null
+  return JSON.parse(readFileSync(path, "utf8")) as StoredRunDescriptor
+}
+
+export function deleteRunDescriptor(runId: number): void {
+  const path = getRunDescriptorPath(runId)
+  if (existsSync(path)) rmSync(path, { force: true })
+}
+
+export function isResumableRuntime(runtime: ToistRuntime): runtime is ResumableRuntime {
+  return typeof (runtime as ResumableRuntime).resumeRun === "function"
+}
+
+export async function resumeLocalStoredRun(
+  runtime: ToistRuntime,
+  runId: number,
+  descriptor: StoredRunDescriptor,
+  response: { taskId: number; token: string; response: unknown; respondedBy?: string },
+): Promise<RunSpecResult> {
+  await runtime.tasks.answer(response.taskId, response.token, response.response, response.respondedBy ?? "cli")
+  const resumeOutputs = await runtime.outputs.list(runId)
+  await runtime.runs.markRunning(runId, { trigger: "resumed", clearCurrentNode: true })
+  return await runSpec(descriptor.spec, descriptor.payload, {
+    runtime,
+    resumeRunId: runId,
+    resumeOutputs,
+    trigger: "resumed",
+    events: true,
+  })
+}

package/src/upgrade.ts

@@ -5,17 +5,31 @@
 
 import { readFile } from "node:fs/promises"
 import { existsSync } from "node:fs"
-import { resolve } from "node:path"
+import { basename, dirname, resolve } from "node:path"
 import { spawnSync } from "node:child_process"
 
+function resolvePackageDir(cwd: string): string | null {
+  const localPkg = resolve(cwd, "package.json")
+  if (existsSync(localPkg)) return cwd
+
+  if (basename(cwd) === ".toist") {
+    const parent = dirname(cwd)
+    if (existsSync(resolve(parent, "package.json"))) return parent
+  }
+
+  return null
+}
+
 export async function upgrade(cwd = process.cwd()): Promise<void> {
-  const pkgPath = resolve(cwd, "package.json")
-  if (!existsSync(pkgPath)) {
+  const packageDir = resolvePackageDir(cwd)
+  if (!packageDir) {
     console.error(`No package.json in ${cwd}.`)
-    console.error("Run from your toist instance directory.")
+    console.error("Run from your host repo root or .toist/ directory.")
     process.exit(1)
   }
 
+  const pkgPath = resolve(packageDir, "package.json")
+
   const pkg = JSON.parse(await readFile(pkgPath, "utf8")) as {
     dependencies?: Record<string, string>
     devDependencies?: Record<string, string>
@@ -39,7 +53,7 @@ export async function upgrade(cwd = process.cwd()): Promise<void> {
   console.log(`Upgrading: ${targets.join(", ")}\n`)
   const result = spawnSync("bun", ["update", "--latest", ...targets], {
     stdio: "inherit",
-    cwd,
+    cwd: packageDir,
   })
   process.exit(result.status ?? 0)
 }

package/src/wizard.ts

@@ -0,0 +1,91 @@
+// 2121 toist
+
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"
+import { basename, dirname, join, resolve } from "node:path"
+import { createInterface } from "node:readline/promises"
+
+const TEMPLATES = [
+  { id: "default", label: "Durable runner (default)" },
+  { id: "embedded", label: "Embedded `.toist/` runner" },
+  { id: "embedded-memory", label: "Memory-only (library use)" },
+] as const
+
+type TemplateId = typeof TEMPLATES[number]["id"]
+
+function templatesDir(): string {
+  return resolve(import.meta.dir, "..", "templates")
+}
+
+function suggestTemplate(dest: string): TemplateId {
+  if (basename(dest) === ".toist") return "embedded"
+  return "default"
+}
+
+function packageNameFor(dest: string): string {
+  const base = basename(dest)
+  if (base === ".toist") return `${basename(dirname(dest)) || "toist"}-toist`
+  return base || "toist"
+}
+
+function copyTemplate(src: string, dest: string) {
+  mkdirSync(dest, { recursive: true })
+  for (const entry of readdirSync(src)) {
+    const srcPath = join(src, entry)
+    const outName = entry === "_gitignore" ? ".gitignore" : entry === "_package.json" ? "package.json" : entry
+    const destPath = join(dest, outName)
+    const info = statSync(srcPath)
+    if (info.isDirectory()) {
+      copyTemplate(srcPath, destPath)
+    } else {
+      copyFileSync(srcPath, destPath)
+    }
+  }
+}
+
+function rewritePackageName(dest: string) {
+  const path = join(dest, "package.json")
+  if (!existsSync(path)) return
+  const pkg = JSON.parse(readFileSync(path, "utf8")) as { name?: string }
+  pkg.name = packageNameFor(dest)
+  writeFileSync(path, JSON.stringify(pkg, null, 2) + "\n")
+}
+
+async function chooseTemplate(defaultTemplate: TemplateId): Promise<TemplateId> {
+  const rl = createInterface({ input: process.stdin, output: process.stdout })
+  try {
+    console.log("")
+    console.log("How will you run pipelines?")
+    for (const [index, template] of TEMPLATES.entries()) {
+      console.log(`  ${index + 1}) ${template.label}`)
+    }
+    const defaultIndex = TEMPLATES.findIndex((template) => template.id === defaultTemplate) + 1
+    const answer = (await rl.question(`Choice [${defaultIndex}]: `)).trim()
+    const picked = Number(answer || defaultIndex)
+    return TEMPLATES[picked - 1]?.id ?? defaultTemplate
+  } finally {
+    rl.close()
+  }
+}
+
+export async function wizard(cwd = process.cwd(), destArg?: string): Promise<void> {
+  const destName = destArg ?? "toist"
+  const templateId = await chooseTemplate(suggestTemplate(destName))
+  const templateDir = join(templatesDir(), templateId)
+  const dest = resolve(cwd, destName)
+
+  if (!existsSync(templateDir)) {
+    console.error(`Template not found: ${templateId}`)
+    process.exit(1)
+  }
+
+  if (existsSync(dest)) {
+    console.error(`Destination already exists: ${dest}`)
+    process.exit(1)
+  }
+
+  copyTemplate(templateDir, dest)
+  rewritePackageName(dest)
+
+  const relative = dest.startsWith(cwd) ? `./${dest.slice(cwd.length).replace(/^\/+/, "") || basename(dest)}` : dest
+  console.log(`✓ created ${relative} (${templateId} template)`)
+}

package/templates/default/README.md

@@ -1,6 +1,10 @@
 # my-toist-instance
 
-Generated with [`bunx @toist/in`](https://toist.in).
+Generated with [`bunx @toist/in`](https://toist.in/docs).
+
+This directory is a Toist home. It can be used either as a dedicated
+instance directory or as an embedded repo-owned home such as
+`<repo>/.toist/`.
 
 ## First run
 
@@ -12,20 +16,20 @@ Open `http://localhost:3000` to use the UI.
 ## Layout
 
 - `start.ts` — bootstrap; calls `startRunner` from `@toist/aja`
-- `kinds/custom.ts` — register your domain-specific kinds here
+- `kinds/custom.ts` — register your domain-specific kinds here (or import them from normal workspace code)
 - `pipelines/*.yaml` — pipeline definitions (per pipeline-spec)
-- `resources/*.yaml` — resource definitions (per resource-spec)
+- `toist.yml` — resource config for this Toist home
 - `data/` — runtime SQLite databases (gitignored)
 
 ## Upgrading
 
 ```sh
-bunx @toist/in
+bunx @toist/in upgrade
 ```
 
-Run without arguments from the instance directory. Auto-detects every
-`@toist/*` dep in `package.json` and bumps them to the latest version.
+Run from the instance directory or host repo root. Bumps every
+`@toist/*` dep in `package.json` to the latest version.
 
 ## More
 
-Full quickstart and docs: https://toist.in
+Full quickstart and docs: https://toist.in/docs

package/templates/default/_package.json

@@ -7,7 +7,7 @@
     "dev": "bun --watch start.ts"
   },
   "dependencies": {
-    "@toist/aja": "^0.6.1",
-    "@toist/spec": "^0.6.1"
+    "@toist/aja": "^0.8.0",
+    "@toist/spec": "^0.8.0"
   }
 }

package/templates/default/kinds/custom.ts

@@ -1,7 +1,7 @@
 // 2121 toist — register your domain-specific kinds here.
 //
 // Each export should be a NodeKind<P, I>. See the @toist/aja built-ins for
-// reference shapes, and pipeline-spec / kind-spec at https://toist.in for
+// reference shapes, and pipeline-spec / kind-spec at https://toist.in/docs for
 // the full contract.
 //
 // Example:

package/templates/default/start.ts

@@ -1,5 +1,5 @@
 // 2121 toist instance bootstrap.
-// See https://toist.in for the full quickstart.
+// See https://toist.in/docs for the full quickstart.
 
 import { startRunner } from "@toist/aja"
 

package/templates/default/toist.yml

@@ -0,0 +1,2 @@
+# toist.yml — resource config for this Toist home
+resources: {}

package/templates/embedded/README.md

@@ -0,0 +1,58 @@
+# my-repo/.toist
+
+Generated with [`bunx @toist/in .toist`](https://toist.in/docs).
+
+This directory is the embedded Toist home for an existing repo.
+`@toist/aja` runs pipelines; `.toist/` is the repo-owned local home for
+Toist config, pipeline definitions, and runtime state.
+
+## First run
+
+1. From the host repo root: `bun add @toist/aja @toist/spec`
+2. Optionally, add a script to your root `package.json`:
+   ```json
+   "scripts": { "toist": "bun .toist/start.ts" }
+   ```
+3. Start the embedded instance: `bun .toist/start.ts`
+
+Open `http://localhost:3000` to use the UI.
+
+## Layout
+
+```
+.toist/
+  start.ts        — embedded bootstrap; calls startRunner from @toist/aja
+  instance.json   — instance name and tier (optional)
+  pipelines/      — pipeline YAML files owned by this repo
+  toist.yml       — resource config for this embedded home
+  data/           — runtime SQLite databases (gitignored)
+```
+
+Custom TypeScript kinds stay in your host repo's normal source tree or
+workspace packages. Import and register them from `.toist/start.ts`:
+
+```ts
+import { register } from "@toist/aja"
+import * as myKinds from "../src/toist-kinds.ts"
+register(...Object.values(myKinds))
+```
+
+## Environment
+
+| Variable | Default | Notes |
+|---|---|---|
+| `TOIST_PORT` | `3000` | Port the runner listens on |
+| `PORT` | `3000` | Fallback if `TOIST_PORT` is unset |
+
+## Upgrading
+
+```sh
+bunx @toist/in upgrade
+```
+
+Run from `.toist/` or from the host repo root. Finds `package.json`
+and bumps every `@toist/*` dependency to the latest version.
+
+## More
+
+Full quickstart and docs: https://toist.in/docs

package/templates/embedded/_gitignore

@@ -0,0 +1,5 @@
+data/*.db
+data/*.db-shm
+data/*.db-wal
+data/.lock/
+.platform/

package/templates/embedded/instance.json

@@ -0,0 +1,4 @@
+{
+  "instanceName": "my-project",
+  "tier": "local"
+}

package/templates/embedded/start.ts

@@ -0,0 +1,24 @@
+// 2121 toist — embedded bootstrap for .toist/
+// See https://toist.in/docs for the full quickstart.
+//
+// rootDir is set to this directory (.toist/).
+// The runner resolves all paths relative to it:
+//   .toist/pipelines/   — pipeline YAML files
+//   .toist/toist.yml    — resource config
+//   .toist/data/        — runtime SQLite databases (gitignored)
+//
+// Custom kinds belong in your host repo's normal source tree.
+// Import and register them here before calling startRunner.
+//
+// Examples:
+//   import * as myKinds from "../src/toist-kinds.ts"
+//   import * as myKinds from "../packages/my-kinds/src/index.ts"
+//   import { register } from "@toist/aja"
+//   register(...Object.values(myKinds))
+
+import { startRunner } from "@toist/aja"
+
+await startRunner({
+  port:    Number(process.env.TOIST_PORT ?? process.env.PORT ?? 3000),
+  rootDir: import.meta.dir,  // = <repo>/.toist/
+})

package/templates/embedded/toist.yml

@@ -0,0 +1,2 @@
+# toist.yml — resource config for this embedded Toist home
+resources: {}

package/templates/embedded-memory/README.md

@@ -0,0 +1,12 @@
+# memory-only Toist host
+
+Generated with `bunx @toist/in` using the memory-only template.
+
+This template uses `runSpec()` and `createMemoryRuntime()` directly. There is no HTTP runner, no UI, and no durable SQLite state.
+
+## First run
+
+1. Add `@toist/core` and `@toist/spec` to your host repo.
+2. Run `bun start.ts`.
+
+The example reads `./pipelines/hello.yml` and prints the result JSON.

package/templates/embedded-memory/_gitignore

@@ -0,0 +1 @@
+node_modules

package/templates/embedded-memory/pipelines/hello.yml

@@ -0,0 +1,12 @@
+apiVersion: "2121.fi/v1"
+id: hello
+nodes:
+  - id: greet
+    kind: data.json
+    params:
+      value: { expr: '"hello " + ctx.params.name' }
+  - id: out
+    kind: sink
+    dependsOn: [greet]
+    input:
+      value: { expr: "ctx.results.greet" }

package/templates/embedded-memory/start.ts

@@ -0,0 +1,16 @@
+import { readFileSync } from "node:fs"
+import { join } from "node:path"
+import { parseYaml } from "@toist/spec"
+import { createMemoryRuntime, runSpec } from "@toist/core"
+
+const spec = parseYaml(readFileSync(join(import.meta.dir, "pipelines", "hello.yml"), "utf8"))
+const runtime = createMemoryRuntime()
+const payload = process.argv[2] ? JSON.parse(process.argv[2]) : { name: "world" }
+const result = await runSpec(spec, payload, { runtime, events: true })
+
+for await (const event of result.events ?? []) {
+  console.error(`[event] ${event.type}`)
+}
+
+const { events: _events, ...printable } = result
+console.log(JSON.stringify(printable, null, 2))

package/src/scaffold.ts

@@ -1,67 +0,0 @@
-// 2121 toist
-// Scaffold logic for @toist/in. Recursively copies templates/default/* into
-// the target directory, with two `_`-prefixed filenames renamed at copy time:
-//
-//   - `_gitignore`   -> `.gitignore`
-//   - `_package.json` -> `package.json`
-//
-// The underscore prefix exists because npm strips files like `.gitignore` and
-// (in some configurations) `package.json` from publishable subdirs. Prefixing
-// dodges those rules; the runtime rename produces the real names in the host.
-//
-// Templates live in `<package>/templates/default/`, relative to this file's
-// runtime location. Works whether the package is consumed as a workspace
-// link (in this repo) or from node_modules (in a host repo).
-
-import { mkdir, copyFile, readdir, stat } from "node:fs/promises"
-import { existsSync } from "node:fs"
-import { dirname, join, resolve } from "node:path"
-import { fileURLToPath } from "node:url"
-
-const __dir        = dirname(fileURLToPath(import.meta.url))
-const TEMPLATE_DIR = resolve(__dir, "..", "templates", "default")
-
-const RENAMES: Record<string, string> = {
-  _gitignore:     ".gitignore",
-  "_package.json": "package.json",
-}
-
-export async function scaffold(target: string): Promise<void> {
-  const dest = resolve(process.cwd(), target)
-
-  if (existsSync(dest)) {
-    const entries = await readdir(dest)
-    if (entries.length > 0) {
-      console.error(`Target directory '${dest}' is not empty. Aborting.`)
-      process.exit(1)
-    }
-  } else {
-    await mkdir(dest, { recursive: true })
-  }
-
-  await copyTree(TEMPLATE_DIR, dest)
-
-  console.log(`\nScaffolded toist instance at ${dest}\n`)
-  console.log("Next:")
-  console.log(`  cd ${target}`)
-  console.log(`  bun install`)
-  console.log(`  bun start.ts`)
-  console.log("")
-  console.log(`Full docs: https://toist.in`)
-  console.log("")
-}
-
-async function copyTree(src: string, dest: string): Promise<void> {
-  for (const entry of await readdir(src)) {
-    const srcPath  = join(src, entry)
-    const destName = RENAMES[entry] ?? entry
-    const destPath = join(dest, destName)
-    const info     = await stat(srcPath)
-    if (info.isDirectory()) {
-      await mkdir(destPath, { recursive: true })
-      await copyTree(srcPath, destPath)
-    } else {
-      await copyFile(srcPath, destPath)
-    }
-  }
-}