@toist/aja 0.6.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -1
- package/README.md +62 -0
- package/package.json +11 -5
- package/src/cache-db.ts +1 -9
- package/src/cli.ts +153 -0
- package/src/client.ts +76 -0
- package/src/data-db.ts +1 -13
- package/src/index.ts +35 -2
- package/src/instance-metadata.ts +48 -0
- package/src/kinds/index.ts +23 -61
- package/src/lock.ts +27 -53
- package/src/migrate.ts +3 -3
- package/src/pipeline-store.ts +31 -0
- package/src/resources-fs.ts +43 -0
- package/src/resources.ts +27 -190
- package/src/run-events.ts +42 -0
- package/src/runtime-db.ts +11 -30
- package/src/server.ts +506 -496
- package/src/sqlite-runtime.ts +135 -0
- package/src/startRunner.ts +56 -70
- package/src/stores/sqlite.ts +243 -0
- package/src/stores/types.ts +18 -0
- package/src/config.ts +0 -129
- package/src/db-handles.ts +0 -70
- package/src/hitl.ts +0 -257
- package/src/instance.ts +0 -64
- package/src/kinds/control.ts +0 -26
- package/src/kinds/data.ts +0 -30
- package/src/kinds/db.ts +0 -92
- package/src/kinds/hitl.ts +0 -56
- package/src/kinds/http.ts +0 -134
- package/src/kinds/runs.ts +0 -130
- package/src/kinds/transform.ts +0 -123
- package/src/kinds/types.ts +0 -16
- package/src/pipeline.ts +0 -605
- package/src/runs.ts +0 -53
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
// 2121
|
|
2
|
+
import type { Database } from "bun:sqlite"
|
|
3
|
+
import { join } from "node:path"
|
|
4
|
+
import { defaultRegistry, type KindRegistry, type ToistRuntime } from "@toist/core"
|
|
5
|
+
import { makeCache } from "./cache.ts"
|
|
6
|
+
import { openRuntimeDb } from "./runtime-db.ts"
|
|
7
|
+
import { openDataDb } from "./data-db.ts"
|
|
8
|
+
import { openCacheDb } from "./cache-db.ts"
|
|
9
|
+
import { acquireRunnerLock } from "./lock.ts"
|
|
10
|
+
import { loadInstance } from "./instance-metadata.ts"
|
|
11
|
+
import { loadFilesystemResources } from "./resources-fs.ts"
|
|
12
|
+
import {
|
|
13
|
+
createSqliteLogStore,
|
|
14
|
+
createSqliteNodeOutputStore,
|
|
15
|
+
createSqliteRunLedger,
|
|
16
|
+
createSqliteTaskStore,
|
|
17
|
+
} from "./stores/sqlite.ts"
|
|
18
|
+
|
|
19
|
+
export interface CreateSqliteRuntimeOptions {
|
|
20
|
+
rootDir: string
|
|
21
|
+
dataDir?: string
|
|
22
|
+
pipelinesDir?: string
|
|
23
|
+
resourcesDir?: string
|
|
24
|
+
registry?: KindRegistry
|
|
25
|
+
skipMigrations?: boolean
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
interface InternalSqliteRuntime extends ToistRuntime {
|
|
29
|
+
close(): Promise<void>
|
|
30
|
+
adminDb: Database
|
|
31
|
+
rootDir: string
|
|
32
|
+
pipelinesDir: string
|
|
33
|
+
dataDir: string
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function globToRegExp(glob: string): RegExp {
|
|
37
|
+
const escaped = glob.replace(/[|\\{}()[\]^$+?.]/g, "\\$&").replace(/\*/g, ".*")
|
|
38
|
+
return new RegExp(`^${escaped}$`)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function filterKindsByGlob(baseRegistry: KindRegistry, allowlist: string[]) {
|
|
42
|
+
const matchers = allowlist.map(globToRegExp)
|
|
43
|
+
return baseRegistry.manifest()
|
|
44
|
+
.filter((kind) => matchers.some((matcher) => matcher.test(kind.id)))
|
|
45
|
+
.map((kind) => baseRegistry.get(kind.id))
|
|
46
|
+
.filter((kind): kind is NonNullable<typeof kind> => !!kind)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function createProjectedRegistry(baseRegistry: KindRegistry, allowlist: string[]): KindRegistry {
|
|
50
|
+
const registry = new Map(filterKindsByGlob(baseRegistry, allowlist).map((kind) => [kind.id, kind]))
|
|
51
|
+
return {
|
|
52
|
+
exclusive: true,
|
|
53
|
+
register(...kinds) {
|
|
54
|
+
for (const kind of kinds) registry.set(kind.id, kind)
|
|
55
|
+
},
|
|
56
|
+
get(id) {
|
|
57
|
+
return registry.get(id)
|
|
58
|
+
},
|
|
59
|
+
manifest() {
|
|
60
|
+
return [...registry.values()].map(({ run: _run, ...manifest }) => manifest)
|
|
61
|
+
},
|
|
62
|
+
has(id) {
|
|
63
|
+
return registry.has(id)
|
|
64
|
+
},
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export async function createSqliteRuntime(
|
|
69
|
+
opts: CreateSqliteRuntimeOptions,
|
|
70
|
+
): Promise<ToistRuntime & { close(): Promise<void> }> {
|
|
71
|
+
const dataDir = opts.dataDir ?? join(opts.rootDir, "data")
|
|
72
|
+
const pipelinesDir = opts.pipelinesDir ?? join(opts.rootDir, "pipelines")
|
|
73
|
+
const ledgerPath = join(dataDir, "runtime.db")
|
|
74
|
+
const domainPath = join(dataDir, "data.db")
|
|
75
|
+
const cachePath = join(dataDir, "cache.db")
|
|
76
|
+
const lockfilePath = join(dataDir, ".lock")
|
|
77
|
+
|
|
78
|
+
const releaseLock = await acquireRunnerLock({ dir: dataDir, lockfilePath })
|
|
79
|
+
|
|
80
|
+
let ledgerHandle: Database | null = null
|
|
81
|
+
let domainHandle: Database | null = null
|
|
82
|
+
let cacheHandle: Database | null = null
|
|
83
|
+
|
|
84
|
+
try {
|
|
85
|
+
ledgerHandle = openRuntimeDb(ledgerPath, { skipMigrations: opts.skipMigrations })
|
|
86
|
+
domainHandle = openDataDb(domainPath)
|
|
87
|
+
cacheHandle = openCacheDb(cachePath)
|
|
88
|
+
|
|
89
|
+
const cache = makeCache(cacheHandle)
|
|
90
|
+
const baseRegistry = opts.registry ?? defaultRegistry
|
|
91
|
+
const allowlist = loadInstance(opts.rootDir).kindAllowlist
|
|
92
|
+
const registry = allowlist
|
|
93
|
+
? createProjectedRegistry(baseRegistry, allowlist)
|
|
94
|
+
: baseRegistry
|
|
95
|
+
const resources = await loadFilesystemResources(opts.rootDir)
|
|
96
|
+
|
|
97
|
+
const runtime: InternalSqliteRuntime = {
|
|
98
|
+
data: domainHandle,
|
|
99
|
+
cache,
|
|
100
|
+
runs: createSqliteRunLedger(ledgerHandle),
|
|
101
|
+
tasks: createSqliteTaskStore(ledgerHandle),
|
|
102
|
+
logs: createSqliteLogStore(ledgerHandle),
|
|
103
|
+
outputs: createSqliteNodeOutputStore(ledgerHandle),
|
|
104
|
+
resources: { resolve: () => resources },
|
|
105
|
+
registry,
|
|
106
|
+
adminDb: ledgerHandle,
|
|
107
|
+
rootDir: opts.rootDir,
|
|
108
|
+
pipelinesDir,
|
|
109
|
+
dataDir,
|
|
110
|
+
async close() {
|
|
111
|
+
if (ledgerHandle) {
|
|
112
|
+
ledgerHandle.close()
|
|
113
|
+
ledgerHandle = null
|
|
114
|
+
}
|
|
115
|
+
if (domainHandle) {
|
|
116
|
+
domainHandle.close()
|
|
117
|
+
domainHandle = null
|
|
118
|
+
}
|
|
119
|
+
if (cacheHandle) {
|
|
120
|
+
cacheHandle.close()
|
|
121
|
+
cacheHandle = null
|
|
122
|
+
}
|
|
123
|
+
await releaseLock()
|
|
124
|
+
},
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return runtime
|
|
128
|
+
} catch (error) {
|
|
129
|
+
if (ledgerHandle) ledgerHandle.close()
|
|
130
|
+
if (domainHandle) domainHandle.close()
|
|
131
|
+
if (cacheHandle) cacheHandle.close()
|
|
132
|
+
await releaseLock()
|
|
133
|
+
throw error
|
|
134
|
+
}
|
|
135
|
+
}
|
package/src/startRunner.ts
CHANGED
|
@@ -1,92 +1,78 @@
|
|
|
1
1
|
// 2121
|
|
2
|
-
// Public API entry per context/instance-spec.md §4. Hosts that consume
|
|
3
|
-
// `@toist/aja` (post-extraction) call this with options to start a runner
|
|
4
|
-
// instance. The legacy CLI path — `bun src/index.ts` invoked directly by pm2
|
|
5
|
-
// in this repo's dev mode — still works unchanged: index.ts triggers initDbs
|
|
6
|
-
// on its own when import.meta.main is true.
|
|
7
|
-
//
|
|
8
|
-
// Lifecycle (matches instance-spec.md §7, scoped to what this iteration
|
|
9
|
-
// covers):
|
|
10
|
-
//
|
|
11
|
-
// 1. setRootDir(options.rootDir) — config knows the host's root
|
|
12
|
-
// 2. initDbs() — acquire lock, migrate, open dbs
|
|
13
|
-
// 3. dynamic import("./index.ts") — builds the Hono app (its own
|
|
14
|
-
// initDbs guard is a no-op since
|
|
15
|
-
// import.meta.main is false here)
|
|
16
|
-
// 4. Bun.serve({ port, fetch }) — bind app to host's port
|
|
17
|
-
// 5. return { port, stop }
|
|
18
|
-
//
|
|
19
|
-
// On stop():
|
|
20
|
-
// - server.stop() — close the HTTP server
|
|
21
|
-
// - closeDbs() — close all three handles, release the runner lock
|
|
22
|
-
//
|
|
23
|
-
// What this iteration does NOT yet do (Roadmap, Phase F W2 follow-ups):
|
|
24
|
-
// - Per-path overrides (pipelineDir, resourceDir, dataDir) as options —
|
|
25
|
-
// env vars cover them today
|
|
26
|
-
// - disableUi, disableMcp, disableWatch options — no UI/MCP yet served
|
|
27
|
-
// from this process; watch is always on
|
|
28
|
-
// - Custom logger injection — current code goes through console.log
|
|
29
|
-
|
|
30
2
|
import type { Server } from "bun"
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
3
|
+
import { join } from "node:path"
|
|
4
|
+
import { createMemoryRuntime, type ToistRuntime } from "@toist/core"
|
|
5
|
+
import { mountApi } from "./server.ts"
|
|
6
|
+
import { createFilesystemPipelineStore } from "./pipeline-store.ts"
|
|
7
|
+
import { createSqliteRuntime } from "./sqlite-runtime.ts"
|
|
33
8
|
|
|
34
9
|
export interface StartRunnerOptions {
|
|
35
|
-
/** TCP port for the HTTP server. Required — no default. */
|
|
36
10
|
port: number
|
|
37
|
-
/** Root directory for all instance-relative paths. Required.
|
|
38
|
-
* Resolves: pipelines/, resources/, data/, instance.json. */
|
|
39
11
|
rootDir: string
|
|
40
|
-
|
|
41
|
-
* from @toist/ui/dist when the build artefacts exist). Set true
|
|
42
|
-
* for headless deployments or when fronting with a separate UI server. */
|
|
12
|
+
storage?: "sqlite" | "memory"
|
|
43
13
|
disableUi?: boolean
|
|
44
|
-
/** Disable the filesystem watcher on pipelineDir/resourceDir. Default:
|
|
45
|
-
* false (watcher is on; pipelines hot-reload on file changes).
|
|
46
|
-
* Production deployments may set true to avoid inotify/fsevent overhead
|
|
47
|
-
* and prevent accidental reloads from concurrent edits. */
|
|
48
14
|
disableWatch?: boolean
|
|
49
|
-
/** CORS allow-origin configuration. `string` for a single origin,
|
|
50
|
-
* `string[]` for an allowlist, `undefined` for default (permissive `*`).
|
|
51
|
-
* Production deployments should typically pin this. */
|
|
52
15
|
corsOrigins?: string | string[]
|
|
16
|
+
runtime?: ToistRuntime
|
|
53
17
|
}
|
|
54
18
|
|
|
55
19
|
export interface RunnerHandle {
|
|
56
|
-
/** The port the runner is actually listening on. */
|
|
57
20
|
port: number
|
|
58
|
-
/** Stop the runner gracefully — close the HTTP server, close DB handles,
|
|
59
|
-
* release the runner lock. */
|
|
60
21
|
stop(): Promise<void>
|
|
61
22
|
}
|
|
62
23
|
|
|
24
|
+
interface HostedRuntime extends ToistRuntime {
|
|
25
|
+
close?: () => Promise<void>
|
|
26
|
+
adminDb?: import("bun:sqlite").Database
|
|
27
|
+
pipelinesDir?: string
|
|
28
|
+
}
|
|
29
|
+
|
|
63
30
|
export async function startRunner(options: StartRunnerOptions): Promise<RunnerHandle> {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
31
|
+
const runtime: HostedRuntime = options.runtime
|
|
32
|
+
? options.runtime as HostedRuntime
|
|
33
|
+
: options.storage === "memory"
|
|
34
|
+
? createMemoryRuntime()
|
|
35
|
+
: await createSqliteRuntime({ rootDir: options.rootDir }) as HostedRuntime
|
|
36
|
+
|
|
37
|
+
const pipelinesDir = runtime.pipelinesDir ?? join(options.rootDir, "pipelines")
|
|
38
|
+
let pipelineStore: ReturnType<typeof createFilesystemPipelineStore> | null = null
|
|
39
|
+
let cachePruneTimer: ReturnType<typeof setInterval> | null = null
|
|
40
|
+
|
|
41
|
+
try {
|
|
42
|
+
pipelineStore = createFilesystemPipelineStore({
|
|
43
|
+
rootDir: options.rootDir,
|
|
44
|
+
watch: !options.disableWatch,
|
|
45
|
+
registry: runtime.registry,
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
cachePruneTimer = setInterval(() => runtime.cache.prune(), 5 * 60 * 1000)
|
|
49
|
+
cachePruneTimer.unref?.()
|
|
73
50
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
51
|
+
const app = await mountApi(runtime, {
|
|
52
|
+
rootDir: options.rootDir,
|
|
53
|
+
pipelinesDir,
|
|
54
|
+
adminDb: runtime.adminDb,
|
|
55
|
+
disableUi: options.disableUi ?? false,
|
|
56
|
+
corsOrigins: options.corsOrigins,
|
|
57
|
+
})
|
|
80
58
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
59
|
+
const server: Server<undefined> = Bun.serve({ port: options.port, fetch: app.fetch })
|
|
60
|
+
const port = server.port ?? options.port
|
|
61
|
+
console.log(`[runner] http://localhost:${port}`)
|
|
84
62
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
63
|
+
return {
|
|
64
|
+
port,
|
|
65
|
+
async stop() {
|
|
66
|
+
pipelineStore?.close()
|
|
67
|
+
if (cachePruneTimer) clearInterval(cachePruneTimer)
|
|
68
|
+
server.stop()
|
|
69
|
+
await runtime.close?.()
|
|
70
|
+
},
|
|
71
|
+
}
|
|
72
|
+
} catch (error) {
|
|
73
|
+
pipelineStore?.close()
|
|
74
|
+
if (cachePruneTimer) clearInterval(cachePruneTimer)
|
|
75
|
+
await runtime.close?.()
|
|
76
|
+
throw error
|
|
91
77
|
}
|
|
92
78
|
}
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
// 2121
|
|
2
|
+
import type { Database } from "bun:sqlite"
|
|
3
|
+
import type {
|
|
4
|
+
LogStore,
|
|
5
|
+
NodeOutputStore,
|
|
6
|
+
RunLedger,
|
|
7
|
+
RunRow,
|
|
8
|
+
TaskDescriptor,
|
|
9
|
+
TaskListItem,
|
|
10
|
+
TaskStore,
|
|
11
|
+
} from "./types.ts"
|
|
12
|
+
|
|
13
|
+
export function createSqliteRunLedger(db: Database): RunLedger {
|
|
14
|
+
return {
|
|
15
|
+
async create(input) {
|
|
16
|
+
const row = db.prepare(
|
|
17
|
+
"INSERT INTO runs (pipeline, status, payload, trigger, updated_at) VALUES (?, 'running', ?, ?, datetime('now')) RETURNING id",
|
|
18
|
+
).get(input.pipeline, JSON.stringify(input.payload ?? {}), input.trigger) as { id: number }
|
|
19
|
+
return row.id
|
|
20
|
+
},
|
|
21
|
+
|
|
22
|
+
async get(runId) {
|
|
23
|
+
return (db.prepare("SELECT * FROM runs WHERE id = ?").get(runId) as RunRow | undefined) ?? null
|
|
24
|
+
},
|
|
25
|
+
|
|
26
|
+
async list(filter = {}) {
|
|
27
|
+
const limit = filter.limit ?? 50
|
|
28
|
+
const rows = filter.pipeline
|
|
29
|
+
? db.prepare("SELECT * FROM runs WHERE pipeline = ? ORDER BY id DESC LIMIT ?").all(filter.pipeline, limit)
|
|
30
|
+
: db.prepare("SELECT * FROM runs ORDER BY id DESC LIMIT ?").all(limit)
|
|
31
|
+
return rows as RunRow[]
|
|
32
|
+
},
|
|
33
|
+
|
|
34
|
+
async markDone(runId, output, steps) {
|
|
35
|
+
db.prepare(
|
|
36
|
+
`UPDATE runs SET status='done', result=?, steps=?, finished_at=datetime('now'),
|
|
37
|
+
updated_at=datetime('now'), current_node=NULL WHERE id=?`,
|
|
38
|
+
).run(JSON.stringify(output), JSON.stringify(steps), runId)
|
|
39
|
+
},
|
|
40
|
+
|
|
41
|
+
async markSuspended(runId, currentNode, steps) {
|
|
42
|
+
db.prepare(
|
|
43
|
+
`UPDATE runs SET status='suspended', steps=?, current_node=?, updated_at=datetime('now') WHERE id=?`,
|
|
44
|
+
).run(JSON.stringify(steps), currentNode, runId)
|
|
45
|
+
},
|
|
46
|
+
|
|
47
|
+
async markError(runId, error) {
|
|
48
|
+
db.prepare(
|
|
49
|
+
"UPDATE runs SET status='error', error=?, finished_at=datetime('now'), updated_at=datetime('now') WHERE id=?",
|
|
50
|
+
).run(error, runId)
|
|
51
|
+
},
|
|
52
|
+
|
|
53
|
+
async markRunning(runId, opts = {}) {
|
|
54
|
+
if (opts.clearCurrentNode ?? true) {
|
|
55
|
+
db.prepare(
|
|
56
|
+
"UPDATE runs SET status='running', trigger=?, current_node=NULL, updated_at=datetime('now') WHERE id=?",
|
|
57
|
+
).run(opts.trigger ?? 'resumed', runId)
|
|
58
|
+
} else {
|
|
59
|
+
db.prepare(
|
|
60
|
+
"UPDATE runs SET status='running', trigger=?, updated_at=datetime('now') WHERE id=?",
|
|
61
|
+
).run(opts.trigger ?? 'resumed', runId)
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
|
|
65
|
+
lastOutput(pipeline, opts) {
|
|
66
|
+
const status = opts?.status ?? "done"
|
|
67
|
+
const row = db.prepare(
|
|
68
|
+
"SELECT result FROM runs WHERE pipeline = ? AND status = ? ORDER BY id DESC LIMIT 1",
|
|
69
|
+
).get(pipeline, status) as { result: string | null } | undefined
|
|
70
|
+
if (!row || row.result === null) return null
|
|
71
|
+
try { return JSON.parse(row.result) as unknown } catch { return null }
|
|
72
|
+
},
|
|
73
|
+
|
|
74
|
+
nodeOutput(pipeline, node, opts) {
|
|
75
|
+
const status = opts?.status ?? "done"
|
|
76
|
+
const row = db.prepare(
|
|
77
|
+
`SELECT no.output_json
|
|
78
|
+
FROM node_outputs no
|
|
79
|
+
JOIN runs r ON no.run_id = r.id
|
|
80
|
+
WHERE r.pipeline = ? AND r.status = ? AND no.node_id = ?
|
|
81
|
+
ORDER BY r.id DESC
|
|
82
|
+
LIMIT 1`,
|
|
83
|
+
).get(pipeline, status, node) as { output_json: string | null } | undefined
|
|
84
|
+
if (!row || row.output_json === null) return null
|
|
85
|
+
try { return JSON.parse(row.output_json) as unknown } catch { return null }
|
|
86
|
+
},
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export function createSqliteTaskStore(db: Database): TaskStore {
|
|
91
|
+
function get(taskId: number): TaskDescriptor | null {
|
|
92
|
+
const row = db.prepare(
|
|
93
|
+
`SELECT t.id, t.run_id, t.node_id, t.kind, t.prompt, t.schema_json, t.assignee,
|
|
94
|
+
t.response_token, t.status, r.pipeline AS pipeline
|
|
95
|
+
FROM tasks t LEFT JOIN runs r ON t.run_id = r.id
|
|
96
|
+
WHERE t.id = ?`,
|
|
97
|
+
).get(taskId) as {
|
|
98
|
+
id: number; run_id: number; node_id: string; kind: string; prompt: string | null
|
|
99
|
+
schema_json: string | null; assignee: string | null
|
|
100
|
+
response_token: string; status: string; pipeline: string | null
|
|
101
|
+
} | undefined
|
|
102
|
+
|
|
103
|
+
if (!row) return null
|
|
104
|
+
return {
|
|
105
|
+
id: row.id,
|
|
106
|
+
runId: row.run_id,
|
|
107
|
+
pipeline: row.pipeline,
|
|
108
|
+
nodeId: row.node_id,
|
|
109
|
+
kind: row.kind as TaskDescriptor["kind"],
|
|
110
|
+
prompt: row.prompt ?? "",
|
|
111
|
+
schema: row.schema_json ? JSON.parse(row.schema_json) : null,
|
|
112
|
+
assignee: row.assignee,
|
|
113
|
+
responseToken: row.response_token,
|
|
114
|
+
status: row.status as TaskDescriptor["status"],
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
return {
|
|
119
|
+
async create(input) {
|
|
120
|
+
const inserted = db.prepare(
|
|
121
|
+
`INSERT INTO tasks (run_id, node_id, kind, prompt, schema_json, assignee, response_token)
|
|
122
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
123
|
+
RETURNING id`,
|
|
124
|
+
).get(
|
|
125
|
+
input.runId,
|
|
126
|
+
input.nodeId,
|
|
127
|
+
input.kind,
|
|
128
|
+
input.prompt,
|
|
129
|
+
input.schema != null ? JSON.stringify(input.schema) : null,
|
|
130
|
+
input.assignee,
|
|
131
|
+
input.token,
|
|
132
|
+
) as { id: number }
|
|
133
|
+
return inserted.id
|
|
134
|
+
},
|
|
135
|
+
|
|
136
|
+
async get(taskId) {
|
|
137
|
+
return get(taskId)
|
|
138
|
+
},
|
|
139
|
+
|
|
140
|
+
async list(filter = {}) {
|
|
141
|
+
const status = filter.status ?? "open"
|
|
142
|
+
const limit = filter.limit ?? 200
|
|
143
|
+
const where: string[] = []
|
|
144
|
+
const args: Array<string | number> = []
|
|
145
|
+
if (status !== "all") { where.push("t.status = ?"); args.push(status) }
|
|
146
|
+
if (filter.assignee) { where.push("t.assignee = ?"); args.push(filter.assignee) }
|
|
147
|
+
if (filter.runId) { where.push("t.run_id = ?"); args.push(filter.runId) }
|
|
148
|
+
const whereSql = where.length ? `WHERE ${where.join(" AND ")}` : ""
|
|
149
|
+
args.push(limit)
|
|
150
|
+
const sql =
|
|
151
|
+
`SELECT t.id, t.run_id, t.node_id, t.kind, t.prompt, t.assignee, t.status,
|
|
152
|
+
t.created_at, t.responded_at, r.pipeline AS pipeline
|
|
153
|
+
FROM tasks t LEFT JOIN runs r ON t.run_id = r.id
|
|
154
|
+
${whereSql}
|
|
155
|
+
ORDER BY t.id DESC LIMIT ?`
|
|
156
|
+
return db.prepare(sql).all(...args) as TaskListItem[]
|
|
157
|
+
},
|
|
158
|
+
|
|
159
|
+
async findOpenForNode(runId, nodeId) {
|
|
160
|
+
return (db.prepare(
|
|
161
|
+
"SELECT id FROM tasks WHERE run_id = ? AND node_id = ? AND status = 'open' ORDER BY id DESC LIMIT 1",
|
|
162
|
+
).get(runId, nodeId) as { id: number } | undefined) ?? null
|
|
163
|
+
},
|
|
164
|
+
|
|
165
|
+
async findAnsweredForNode(runId, nodeId) {
|
|
166
|
+
const row = db.prepare(
|
|
167
|
+
"SELECT response_json FROM tasks WHERE run_id = ? AND node_id = ? AND status = 'answered' ORDER BY id DESC LIMIT 1",
|
|
168
|
+
).get(runId, nodeId) as { response_json: string | null } | undefined
|
|
169
|
+
if (!row) return null
|
|
170
|
+
return { response: row.response_json ? JSON.parse(row.response_json) : null }
|
|
171
|
+
},
|
|
172
|
+
|
|
173
|
+
async answer(taskId, token, response, respondedBy) {
|
|
174
|
+
const task = get(taskId)
|
|
175
|
+
if (!task) throw new Error(`task ${taskId} not found`)
|
|
176
|
+
if (task.status !== "open") throw new Error(`task ${taskId} is ${task.status}, cannot answer`)
|
|
177
|
+
if (task.responseToken !== token) throw new Error(`task ${taskId}: invalid response token`)
|
|
178
|
+
db.prepare(
|
|
179
|
+
`UPDATE tasks
|
|
180
|
+
SET status = 'answered',
|
|
181
|
+
response_json = ?,
|
|
182
|
+
responded_by = ?,
|
|
183
|
+
responded_at = datetime('now')
|
|
184
|
+
WHERE id = ?`,
|
|
185
|
+
).run(JSON.stringify(response ?? null), respondedBy, taskId)
|
|
186
|
+
return { ...task, status: "answered" }
|
|
187
|
+
},
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export function createSqliteLogStore(db: Database): LogStore {
|
|
192
|
+
const stmt = db.prepare("INSERT INTO logs (run_id, level, msg) VALUES (?, ?, ?)")
|
|
193
|
+
return {
|
|
194
|
+
append(runId, level, msg) {
|
|
195
|
+
stmt.run(runId, level, msg)
|
|
196
|
+
},
|
|
197
|
+
async list(runId) {
|
|
198
|
+
return db.prepare(
|
|
199
|
+
"SELECT id, ts, level, msg FROM logs WHERE run_id = ? ORDER BY id ASC",
|
|
200
|
+
).all(runId) as Array<{ id: number; ts: string; level: string; msg: string | null }>
|
|
201
|
+
},
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export function createSqliteNodeOutputStore(db: Database): NodeOutputStore {
|
|
206
|
+
return {
|
|
207
|
+
async putMany(runId, outputs) {
|
|
208
|
+
const stmt = db.prepare(
|
|
209
|
+
`INSERT OR REPLACE INTO node_outputs (run_id, node_id, output_json, finished_at)
|
|
210
|
+
VALUES (?, ?, ?, datetime('now'))`,
|
|
211
|
+
)
|
|
212
|
+
const tx = db.transaction(() => {
|
|
213
|
+
for (const [nodeId, output] of Object.entries(outputs)) {
|
|
214
|
+
stmt.run(runId, nodeId, JSON.stringify(output ?? null))
|
|
215
|
+
}
|
|
216
|
+
})
|
|
217
|
+
tx()
|
|
218
|
+
},
|
|
219
|
+
|
|
220
|
+
async list(runId) {
|
|
221
|
+
const rows = db.prepare(
|
|
222
|
+
"SELECT node_id, output_json FROM node_outputs WHERE run_id = ?",
|
|
223
|
+
).all(runId) as { node_id: string; output_json: string | null }[]
|
|
224
|
+
const out: Record<string, unknown> = {}
|
|
225
|
+
for (const row of rows) out[row.node_id] = row.output_json ? JSON.parse(row.output_json) : null
|
|
226
|
+
return out
|
|
227
|
+
},
|
|
228
|
+
|
|
229
|
+
async listOrdered(runId) {
|
|
230
|
+
const rows = db.prepare(
|
|
231
|
+
"SELECT node_id, output_json, started_at, finished_at FROM node_outputs WHERE run_id = ? ORDER BY started_at ASC",
|
|
232
|
+
).all(runId) as Array<{ node_id: string; output_json: string | null; started_at: string; finished_at: string }>
|
|
233
|
+
return rows.map((row) => ({
|
|
234
|
+
nodeId: row.node_id,
|
|
235
|
+
output: row.output_json ? JSON.parse(row.output_json) : null,
|
|
236
|
+
startedAt: row.started_at,
|
|
237
|
+
finishedAt: row.finished_at,
|
|
238
|
+
}))
|
|
239
|
+
},
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// 2121
|
|
2
|
+
// Backwards-compat shim while aja internals switch to @toist/core as the
|
|
3
|
+
// canonical home for runtime/store interfaces.
|
|
4
|
+
|
|
5
|
+
export type {
|
|
6
|
+
ToistRuntime,
|
|
7
|
+
RunLedger,
|
|
8
|
+
RunRow,
|
|
9
|
+
TaskStore,
|
|
10
|
+
TaskListItem,
|
|
11
|
+
LogStore,
|
|
12
|
+
LogRow,
|
|
13
|
+
NodeOutputStore,
|
|
14
|
+
NodeOutputRow,
|
|
15
|
+
ResourceResolver,
|
|
16
|
+
KindRegistry,
|
|
17
|
+
TaskDescriptor,
|
|
18
|
+
} from "@toist/core"
|
package/src/config.ts
DELETED
|
@@ -1,129 +0,0 @@
|
|
|
1
|
-
// 2121
|
|
2
|
-
// Single source of truth for instance-relative paths. Per
|
|
3
|
-
// context/instance-spec.md §3 + §5:
|
|
4
|
-
//
|
|
5
|
-
// <rootDir>/
|
|
6
|
-
// instance.json — optional metadata (§6)
|
|
7
|
-
// pipelines/ — pipeline YAML files
|
|
8
|
-
// resources/ — resource YAML files
|
|
9
|
-
// data/ — runner-managed runtime state (gitignored):
|
|
10
|
-
// data.db — host's domain data (schema-on-write)
|
|
11
|
-
// runtime.db — platform ledger (runs/logs/tasks/node_outputs)
|
|
12
|
-
// cache.db — TTL-driven cache
|
|
13
|
-
// .lock — proper-lockfile holder (single-leader guard)
|
|
14
|
-
//
|
|
15
|
-
// rootDir resolution order:
|
|
16
|
-
// 1. process.env.PLATFORM_ROOT_DIR — explicit override (used by hosts
|
|
17
|
-
// that bootstrap @toist/aja from
|
|
18
|
-
// a non-cwd-aligned location)
|
|
19
|
-
// 2. join(process.cwd(), "..") — fallback matching the historic
|
|
20
|
-
// "server cwd is <root>/server"
|
|
21
|
-
// convention
|
|
22
|
-
//
|
|
23
|
-
// Note: the legacy `<rootDir>/.platform/` directory is platform-internal
|
|
24
|
-
// dev-mode coordination (pm2 port handoff in runtime.json, CLI tooling output
|
|
25
|
-
// in generated/). It is NOT part of the instance contract and is not exposed
|
|
26
|
-
// from this module — start.ts and ecosystem.config.cjs handle it on their own.
|
|
27
|
-
|
|
28
|
-
import { join } from "node:path"
|
|
29
|
-
|
|
30
|
-
let rootDirOverride: string | null = null
|
|
31
|
-
let uiDisabled = false
|
|
32
|
-
let watchDisabled = false
|
|
33
|
-
let corsOriginsConfig: string | string[] | null = null
|
|
34
|
-
|
|
35
|
-
/** Set the rootDir explicitly. Used by `startRunner({rootDir, ...})` to
|
|
36
|
-
* configure paths before the db modules' static imports trigger their
|
|
37
|
-
* module-load init. Once set, the override wins over PLATFORM_ROOT_DIR
|
|
38
|
-
* and the cwd-parent fallback. Calling twice is an error — rootDir is
|
|
39
|
-
* once-set per process. */
|
|
40
|
-
export function setRootDir(dir: string): void {
|
|
41
|
-
if (rootDirOverride !== null && rootDirOverride !== dir) {
|
|
42
|
-
throw new Error(
|
|
43
|
-
`[config] setRootDir(${JSON.stringify(dir)}) called after rootDir ` +
|
|
44
|
-
`was already set to ${JSON.stringify(rootDirOverride)}. ` +
|
|
45
|
-
`rootDir is once-set per process.`,
|
|
46
|
-
)
|
|
47
|
-
}
|
|
48
|
-
rootDirOverride = dir
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
/** Set the disableUi flag. Used by `startRunner({disableUi: true, ...})` to
|
|
52
|
-
* opt out of the bundled static-UI mount before server.ts's outer app
|
|
53
|
-
* construction reads the flag. Default: UI is enabled when the dist/
|
|
54
|
-
* directory in @toist/ui exists. */
|
|
55
|
-
export function setDisableUi(v: boolean): void {
|
|
56
|
-
uiDisabled = v
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
export function isUiDisabled(): boolean {
|
|
60
|
-
return uiDisabled
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
/** Set the disableWatch flag. Used by `startRunner({disableWatch: true, ...})`
|
|
64
|
-
* to opt out of the filesystem watcher on pipelineDir/resourceDir. Default:
|
|
65
|
-
* watcher is enabled. Production deployments may set true to avoid
|
|
66
|
-
* inotify/fsevent overhead and accidental hot-reloads. */
|
|
67
|
-
export function setDisableWatch(v: boolean): void {
|
|
68
|
-
watchDisabled = v
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
export function isWatchDisabled(): boolean {
|
|
72
|
-
return watchDisabled
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
/** Set the CORS allow-origin configuration. `string` for a single origin,
|
|
76
|
-
* `string[]` for an allowlist, `null` for default (permissive `*`).
|
|
77
|
-
* Per Hono's cors() middleware semantics. */
|
|
78
|
-
export function setCorsOrigins(origins: string | string[] | null | undefined): void {
|
|
79
|
-
corsOriginsConfig = origins ?? null
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
export function getCorsOrigins(): string | string[] | null {
|
|
83
|
-
return corsOriginsConfig
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
export function resolveRootDir(): string {
|
|
87
|
-
if (rootDirOverride !== null) return rootDirOverride
|
|
88
|
-
const override = process.env.PLATFORM_ROOT_DIR
|
|
89
|
-
if (override && override.length > 0) return override
|
|
90
|
-
return join(process.cwd(), "..")
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
export function dataDir(): string {
|
|
94
|
-
const override = process.env.PLATFORM_DATA_DIR
|
|
95
|
-
if (override && override.length > 0) return override
|
|
96
|
-
return join(resolveRootDir(), "data")
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
export function dataDbPath(): string {
|
|
100
|
-
return join(dataDir(), "data.db")
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
export function runtimeDbPath(): string {
|
|
104
|
-
return join(dataDir(), "runtime.db")
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
export function cacheDbPath(): string {
|
|
108
|
-
return join(dataDir(), "cache.db")
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
export function lockfilePath(): string {
|
|
112
|
-
return join(dataDir(), ".lock")
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
export function instanceFilePath(): string {
|
|
116
|
-
return join(resolveRootDir(), "instance.json")
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
export function pipelinesDir(): string {
|
|
120
|
-
const override = process.env.PIPELINES_DIR
|
|
121
|
-
if (override && override.length > 0) return override
|
|
122
|
-
return join(resolveRootDir(), "pipelines")
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
export function resourcesDir(): string {
|
|
126
|
-
const override = process.env.RESOURCES_DIR
|
|
127
|
-
if (override && override.length > 0) return override
|
|
128
|
-
return join(resolveRootDir(), "resources")
|
|
129
|
-
}
|