@toren-run/core 0.1.0

package/dist/store.d.ts

@@ -0,0 +1,50 @@
+import type pg from "pg";
+import type { NewEvent, RecordedEvent, StreamId } from "./events.js";
+export type AppendResult = {
+    ok: true;
+    lastSeq: number;
+} | {
+    ok: false;
+    conflict: true;
+    actualSeq: number;
+};
+export interface CreateRun {
+    runId: string;
+    agent: string;
+    input?: unknown;
+    codeHash?: string;
+    mode?: "task" | "session";
+    process?: string;
+}
+export declare class PgStateStore {
+    private pool;
+    private schema;
+    constructor(pool: pg.Pool, schema: string);
+    createRun(req: CreateRun): Promise<void>;
+    getRun(runId: string): Promise<{
+        runId: string;
+        agent: string;
+        status: string;
+        mode: string;
+        process: string;
+        input: unknown;
+        output: unknown;
+        error: unknown;
+    } | null>;
+    updateRun(runId: string, patch: {
+        status?: string;
+        output?: unknown;
+        error?: unknown;
+    }): Promise<void>;
+    listRuns(limit?: number): Promise<{
+        runId: string;
+        agent: string;
+        status: string;
+        mode: string;
+        process: string;
+        createdAt: Date;
+    }[]>;
+    listNonTerminalRuns(): Promise<string[]>;
+    append(runId: string, streamId: StreamId, expectedSeq: number, events: NewEvent[]): Promise<AppendResult>;
+    read(runId: string, streamId: StreamId, fromSeq?: number): Promise<RecordedEvent[]>;
+}

package/dist/store.js

@@ -0,0 +1,65 @@
+import { tx } from "./db.js";
+export class PgStateStore {
+    pool;
+    schema;
+    constructor(pool, schema) {
+        this.pool = pool;
+        this.schema = schema;
+    }
+    async createRun(req) {
+        await this.pool.query(`INSERT INTO ${this.schema}.runs (run_id, agent, input, code_hash, mode, process) VALUES ($1,$2,$3,$4,$5,$6)`, [req.runId, req.agent, JSON.stringify(req.input ?? null), req.codeHash ?? null, req.mode ?? "task", req.process ?? "main"]);
+    }
+    async getRun(runId) {
+        const r = await this.pool.query(`SELECT run_id, agent, status, mode, process, input, output, error FROM ${this.schema}.runs WHERE run_id = $1`, [runId]);
+        const row = r.rows[0];
+        return row ? { runId: row.run_id, agent: row.agent, status: row.status, mode: row.mode, process: row.process, input: row.input, output: row.output, error: row.error } : null;
+    }
+    async updateRun(runId, patch) {
+        await this.pool.query(`UPDATE ${this.schema}.runs
+       SET status = COALESCE($2, status),
+           output = COALESCE($3, output),
+           error  = COALESCE($4, error),
+           updated_at = now()
+       WHERE run_id = $1`, [
+            runId,
+            patch.status ?? null,
+            patch.output !== undefined ? JSON.stringify(patch.output) : null,
+            patch.error !== undefined ? JSON.stringify(patch.error) : null,
+        ]);
+    }
+    async listRuns(limit = 50) {
+        const r = await this.pool.query(`SELECT run_id, agent, status, mode, process, created_at FROM ${this.schema}.runs ORDER BY created_at DESC LIMIT $1`, [limit]);
+        return r.rows.map((row) => ({ runId: row.run_id, agent: row.agent, status: row.status, mode: row.mode, process: row.process, createdAt: row.created_at }));
+    }
+    async listNonTerminalRuns() {
+        const r = await this.pool.query(`SELECT run_id FROM ${this.schema}.runs WHERE status NOT IN ('completed','failed','cancelled') ORDER BY created_at`);
+        return r.rows.map((row) => row.run_id);
+    }
+    async append(runId, streamId, expectedSeq, events) {
+        if (events.length === 0)
+            throw new Error("append requires at least one event");
+        return tx(this.pool, async (c) => {
+            await c.query(`INSERT INTO ${this.schema}.streams (run_id, stream_id) VALUES ($1,$2) ON CONFLICT DO NOTHING`, [runId, streamId]);
+            const cas = await c.query(`UPDATE ${this.schema}.streams SET head_seq = head_seq + $3
+         WHERE run_id = $1 AND stream_id = $2 AND head_seq = $4 RETURNING head_seq`, [runId, streamId, events.length, expectedSeq]);
+            if (cas.rowCount === 0) {
+                const cur = await c.query(`SELECT head_seq FROM ${this.schema}.streams WHERE run_id = $1 AND stream_id = $2`, [runId, streamId]);
+                // COMMIT of a no-op tx is harmless; nothing was written.
+                return { ok: false, conflict: true, actualSeq: Number(cur.rows[0].head_seq) };
+            }
+            let seq = expectedSeq;
+            for (const e of events) {
+                seq += 1;
+                await c.query(`INSERT INTO ${this.schema}.events (run_id, stream_id, seq, type, payload) VALUES ($1,$2,$3,$4,$5)`, [runId, streamId, seq, e.type, JSON.stringify(e.payload)]);
+            }
+            return { ok: true, lastSeq: seq };
+        });
+    }
+    async read(runId, streamId, fromSeq = 0) {
+        const r = await this.pool.query(`SELECT seq, type, payload, recorded_at FROM ${this.schema}.events
+       WHERE run_id = $1 AND stream_id = $2 AND seq > $3 ORDER BY seq`, [runId, streamId, fromSeq]);
+        return r.rows.map((row) => ({
+            seq: Number(row.seq), type: row.type, payload: row.payload, recordedAt: row.recorded_at,
+        }));
+    }
+}

package/dist/tools.d.ts

@@ -0,0 +1,98 @@
+import type { z } from "zod";
+import type { ToolSpec } from "./model.js";
+export type ToolEffects = "external" | "sandbox" | "none";
+/**
+ * Background processes: spawn one of the agent's named processes as a durable
+ * run and inspect it. Wired by the runtime (like the sandbox); a watcher
+ * recorded at spawn time wakes the parent session when the child settles.
+ */
+export interface ProcessesCtx {
+    /** Process names this agent serves (the workflows map's keys). */
+    names: string[];
+    defaultProcess?: string;
+    start(req: {
+        process: string;
+        input: string;
+        parentRunId: string;
+        parentTaskId: string;
+        toolUseId: string;
+    }): Promise<{
+        runId: string;
+        started: boolean;
+    }>;
+    status(runId: string): Promise<null | {
+        runId: string;
+        process: string;
+        status: string;
+        output?: string;
+        error?: string;
+        waves: {
+            name: string;
+            tasks: number;
+            settled: number;
+            done: boolean;
+        }[];
+    }>;
+}
+export interface ToolCtx {
+    runId: string;
+    taskId: string;
+    /** The model's tool-use block id — stable across replays; key derived side effects on it. */
+    toolUseId: string;
+    /** Env values declared in agent.yaml `env:` — never raw process.env. */
+    env: Record<string, string>;
+    /** Attached-file access for the read_file builtin; wired by the runtime. */
+    files?: {
+        get(id: string): Promise<{
+            id: string;
+            name: string;
+            pages: string[];
+        } | null>;
+    };
+    /** Durable per-run workspace execution for the bash builtin; wired by the runtime. */
+    sandbox?: SandboxExec;
+    /** Background named-process runs for the run_process/check_run builtins; wired by the runtime. */
+    processes?: ProcessesCtx;
+}
+/** One run's sandbox: commands and file operations against a persistent workspace. */
+export interface SandboxExec {
+    exec(command: string, opts?: {
+        timeoutMs?: number;
+    }): Promise<{
+        stdout: string;
+        stderr: string;
+        exitCode: number;
+    }>;
+    readFile(path: string): Promise<string>;
+    writeFile(path: string, content: string): Promise<void>;
+    /** Cheap park: preserve the workspace, stop paying for compute. Reconnected on next use. */
+    pause?(): Promise<void>;
+    /** Terminal cleanup: destroy the workspace and forget it. */
+    dispose?(): Promise<void>;
+}
+/** Constructs per-run sandbox handles; implemented by the CLI runtime (docker locally). */
+export interface SandboxProvider {
+    forRun(runId: string): SandboxExec;
+}
+export interface ToolDef<S extends z.ZodTypeAny = z.ZodTypeAny> {
+    name: string;
+    description: string;
+    input: S;
+    effects: ToolEffects;
+    idempotency: "keyed" | "none";
+    approval: "never" | "always" | ((args: z.infer<S>) => boolean);
+    handler: (args: z.infer<S>, ctx: ToolCtx) => Promise<string>;
+}
+/** Type-erased tool shape — what registries, the loop, and AgentSpec consume. */
+export interface ToolDefAny {
+    name: string;
+    description: string;
+    input: z.ZodTypeAny;
+    effects: ToolEffects;
+    idempotency: "keyed" | "none";
+    approval: "never" | "always" | ((args: any) => boolean);
+    handler: (args: any, ctx: ToolCtx) => Promise<string>;
+}
+export declare function defineTool<S extends z.ZodTypeAny>(def: ToolDef<S>): ToolDefAny;
+export declare function toolSpecs(tools: ToolDefAny[]): ToolSpec[];
+export declare function needsApproval(t: ToolDefAny, args: unknown): boolean;

package/dist/tools.js

@@ -0,0 +1,14 @@
+import { zodToJsonSchema } from "zod-to-json-schema";
+export function defineTool(def) {
+    return def;
+}
+export function toolSpecs(tools) {
+    return tools.map((t) => ({
+        name: t.name,
+        description: t.description,
+        inputSchema: zodToJsonSchema(t.input, { target: "jsonSchema7" }),
+    }));
+}
+export function needsApproval(t, args) {
+    return t.approval === "always" || (typeof t.approval === "function" && t.approval(args));
+}

package/dist/tracing.d.ts

@@ -0,0 +1,6 @@
+import { type Span } from "@opentelemetry/api";
+/**
+ * Every model call, tool invocation, task, and tick emits a
+ * standard span. No-op unless the host registers an OTel SDK/exporter.
+ */
+export declare function withSpan<T>(name: string, attributes: Record<string, string | number>, fn: (span: Span) => Promise<T>): Promise<T>;

package/dist/tracing.js

@@ -0,0 +1,21 @@
+import { SpanStatusCode, trace } from "@opentelemetry/api";
+// Resolved per call: hosts may register their OTel SDK after toren is imported.
+const tracer = () => trace.getTracer("toren");
+/**
+ * Every model call, tool invocation, task, and tick emits a
+ * standard span. No-op unless the host registers an OTel SDK/exporter.
+ */
+export async function withSpan(name, attributes, fn) {
+    return tracer().startActiveSpan(name, { attributes }, async (span) => {
+        try {
+            return await fn(span);
+        }
+        catch (e) {
+            span.setStatus({ code: SpanStatusCode.ERROR, message: e instanceof Error ? e.message : String(e) });
+            throw e;
+        }
+        finally {
+            span.end();
+        }
+    });
+}

package/dist/worker.d.ts

@@ -0,0 +1,34 @@
+import { type TickDeps } from "./orchestrator.js";
+export interface WorkerOpts {
+    concurrency?: number;
+    visibilitySeconds?: number;
+    pollMs?: number;
+}
+/**
+ * Local worker runtime: in-process pollers over the orchestrator and task
+ * queues (local binding). One instance serves one agent or a whole
+ * fleet — construct with a single TickDeps or a Record keyed by agent name;
+ * messages carry the agent and are routed to that agent's deps (store,
+ * leases, and specs are per-agent; the queue tables are shared).
+ */
+export declare class LocalWorkerRuntime {
+    private stopped;
+    private loops;
+    private inFlight;
+    private readonly opts;
+    private readonly byAgent;
+    /** Single-agent construction routes every message here, agent label or not. */
+    private readonly sole;
+    /** Any entry — used for the shared queue (polling, acking, depth). */
+    private readonly shared;
+    constructor(deps: TickDeps | Record<string, TickDeps>, opts?: WorkerOpts);
+    private depsFor;
+    /** Fleet workers claim only their own agents' messages; sole workers claim everything (legacy). */
+    private get agentScope();
+    start(): void;
+    stop(): Promise<void>;
+    /** Test helper: run until both queues stay empty and nothing is in flight. */
+    drain(timeoutMs?: number): Promise<void>;
+    private pollLoop;
+    private handle;
+}

package/dist/worker.js

@@ -0,0 +1,176 @@
+import { randomUUID } from "node:crypto";
+import { InvalidationStormError, runTaskLoop, TaskLeaseLostError } from "./loop.js";
+import { findTaskSpec, tick } from "./orchestrator.js";
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/**
+ * Local worker runtime: in-process pollers over the orchestrator and task
+ * queues (local binding). One instance serves one agent or a whole
+ * fleet — construct with a single TickDeps or a Record keyed by agent name;
+ * messages carry the agent and are routed to that agent's deps (store,
+ * leases, and specs are per-agent; the queue tables are shared).
+ */
+export class LocalWorkerRuntime {
+    stopped = false;
+    loops = [];
+    inFlight = 0;
+    opts;
+    byAgent;
+    /** Single-agent construction routes every message here, agent label or not. */
+    sole;
+    /** Any entry — used for the shared queue (polling, acking, depth). */
+    shared;
+    constructor(deps, opts = {}) {
+        this.opts = { concurrency: opts.concurrency ?? 2, visibilitySeconds: opts.visibilitySeconds ?? 60, pollMs: opts.pollMs ?? 20 };
+        if ("store" in deps) {
+            // Legacy single-deps form: unscoped — claims and serves everything.
+            this.sole = deps;
+            this.byAgent = new Map();
+            this.shared = this.sole;
+        }
+        else {
+            // Fleet form — ALWAYS scoped to its agents, even a fleet of one, so
+            // separate fleets can share a queue table without stealing hints.
+            const entries = Object.entries(deps);
+            if (entries.length === 0)
+                throw new Error("worker needs at least one agent's deps");
+            this.byAgent = new Map(entries);
+            this.sole = null;
+            this.shared = entries[0][1];
+        }
+    }
+    depsFor(agent) {
+        if (this.sole)
+            return this.sole;
+        if (agent && this.byAgent.has(agent))
+            return this.byAgent.get(agent);
+        // Unlabeled (pre-fleet) messages: a fleet of one can safely own them.
+        if (!agent && this.byAgent.size === 1)
+            return this.shared;
+        return null; // unknown agent, or unlabeled on a multi-crew fleet — stale hint
+    }
+    /** Fleet workers claim only their own agents' messages; sole workers claim everything (legacy). */
+    get agentScope() {
+        return this.sole ? undefined : [...this.byAgent.keys()];
+    }
+    start() {
+        for (let i = 0; i < this.opts.concurrency; i++)
+            this.loops.push(this.pollLoop(`worker-${i}-${randomUUID()}`));
+    }
+    async stop() {
+        this.stopped = true;
+        await Promise.all(this.loops);
+        this.loops = [];
+    }
+    /** Test helper: run until both queues stay empty and nothing is in flight. */
+    async drain(timeoutMs = 10_000) {
+        const deadline = Date.now() + timeoutMs;
+        let quiet = 0;
+        while (Date.now() < deadline) {
+            const busy = this.inFlight > 0 || (await this.shared.queue.depth({ agents: this.agentScope })) > 0;
+            quiet = busy ? 0 : quiet + 1;
+            if (quiet >= 5)
+                return;
+            await sleep(this.opts.pollMs);
+        }
+        throw new Error(`drain timed out after ${timeoutMs}ms`);
+    }
+    async pollLoop(owner) {
+        let errorStreak = 0;
+        while (!this.stopped) {
+            let d;
+            try {
+                d =
+                    (await this.shared.queue.receive("orchestrator", { max: 1, visibilitySeconds: this.opts.visibilitySeconds, agents: this.agentScope }))[0] ??
+                        (await this.shared.queue.receive("tasks-short", { max: 1, visibilitySeconds: this.opts.visibilitySeconds, agents: this.agentScope }))[0];
+                errorStreak = 0;
+            }
+            catch {
+                // Transient infrastructure error (DB restart, network blip). Back off
+                // and keep polling — an uncaught throw here would kill this loop for
+                // good and leave a worker that looks healthy but processes nothing.
+                errorStreak = Math.min(errorStreak + 1, 6);
+                await sleep(Math.min(this.opts.pollMs * 2 ** errorStreak, 30_000));
+                continue;
+            }
+            if (!d) {
+                await sleep(this.opts.pollMs);
+                continue;
+            }
+            this.inFlight += 1;
+            try {
+                await this.handle(d, owner);
+            }
+            finally {
+                this.inFlight -= 1;
+            }
+        }
+    }
+    async handle(d, owner) {
+        const msg = d.message;
+        const deps = this.depsFor(msg.agent);
+        if (!deps) {
+            // A hint for an agent this worker doesn't serve. Messages are hints,
+            // never truth — the owning worker's guardians re-derive the work.
+            await this.shared.queue.ack(d);
+            return;
+        }
+        try {
+            if (msg.kind === "tick") {
+                await tick(deps, msg.runId);
+                await this.shared.queue.ack(d);
+                return;
+            }
+            // task message
+            const taskId = msg.taskId;
+            const streamId = `task:${taskId}`;
+            const lease = await deps.leases.acquire(msg.runId, streamId, owner, this.opts.visibilitySeconds);
+            if (!lease) {
+                await this.shared.queue.ack(d); // someone else owns it; message was a hint
+                return;
+            }
+            try {
+                const spec = await findTaskSpec(deps.store, msg.runId, taskId);
+                if (!spec) {
+                    await this.shared.queue.ack(d); // stale hint for an invalidated plan
+                    return;
+                }
+                const agent = deps.agents[spec.agentRef];
+                if (!agent)
+                    throw new Error(`no agent registered for ref ${spec.agentRef}`);
+                const run = await deps.store.getRun(msg.runId);
+                await runTaskLoop({
+                    store: deps.store, provider: deps.provider,
+                    runId: msg.runId, taskId, agent, input: spec.input, files: deps.files,
+                    sandbox: deps.sandbox?.forRun(msg.runId),
+                    processes: deps.processes,
+                    sessionMode: run?.mode === "session",
+                });
+                await this.shared.queue.ack(d);
+                // Nudge the orchestrator to absorb the terminal/parked state.
+                await this.shared.queue.send("orchestrator", { kind: "tick", runId: msg.runId, agent: msg.agent, dedupeKey: `settle-${msg.runId}-${taskId}` });
+            }
+            finally {
+                await deps.leases.release(lease);
+            }
+        }
+        catch (e) {
+            try {
+                if (e instanceof TaskLeaseLostError) {
+                    await this.shared.queue.ack(d);
+                    return;
+                }
+                if (e instanceof InvalidationStormError) {
+                    // Mixed worker versions are fighting over this stream. Back off well past
+                    // a deploy's drain window so the surviving version picks it up.
+                    await this.shared.queue.nack(d, { delaySeconds: 20 });
+                    return;
+                }
+                await this.shared.queue.nack(d, { delaySeconds: 0.2 });
+            }
+            catch {
+                // Queue unreachable too: do nothing. The visibility timeout redelivers
+                // the message; hints are never truth, so losing an ack costs a retry.
+            }
+        }
+    }
+}

package/dist/workflow.d.ts

@@ -0,0 +1,91 @@
+import { type RecordedEvent } from "./events.js";
+import type { PgStateStore } from "./store.js";
+import type { QueueMessage, QueueName } from "./queue.js";
+export interface TaskSpec {
+    agentRef: string;
+    input: string;
+}
+export interface TaskOutcome {
+    taskId: string;
+    status: "completed" | "failed";
+    output?: string;
+    error?: string;
+}
+export interface WaveResult {
+    name: string;
+    results: TaskOutcome[];
+}
+export interface WaveOpts {
+    onTaskFailure?: "fail" | "collect";
+}
+export interface WorkflowCtx {
+    input: string;
+    task(agentRef: string, input: string): TaskSpec;
+    wave(name: string, tasks: TaskSpec[], opts?: WaveOpts): Promise<WaveResult>;
+    now(): Promise<number>;
+    random(): Promise<number>;
+    sleep(ms: number): Promise<void>;
+}
+export type WorkflowFn = (ctx: WorkflowCtx) => Promise<string>;
+export declare class WorkflowBlocked extends Error {
+    reason: string;
+    constructor(reason: string);
+}
+export declare class WaveFailedError extends Error {
+}
+export declare class RunLeaseLostError extends Error {
+}
+export interface WaveState {
+    seq: number;
+    waveId: string;
+    index: number;
+    name: string;
+    tasks: {
+        taskId: string;
+        agentRef: string;
+        input: string;
+    }[];
+    planDigest: string;
+    dispatched: boolean;
+    settledTasks: Map<string, TaskOutcome>;
+    settled: boolean;
+}
+export interface TimerState {
+    seq: number;
+    timerId: string;
+    fireAt: number;
+    fired: boolean;
+}
+export interface RunStreamState {
+    sideEffects: RecordedEvent[];
+    waves: WaveState[];
+    timers: TimerState[];
+    started: boolean;
+    terminal?: {
+        status: "completed" | "failed" | "cancelled";
+        output?: string;
+        error?: string;
+    };
+}
+export declare function foldRunStream(eff: RecordedEvent[]): RunStreamState;
+export interface PendingDispatch {
+    queue: QueueName;
+    msg: QueueMessage;
+    delaySeconds?: number;
+}
+export interface RunSession {
+    store: PgStateStore;
+    runId: string;
+    input: string;
+    head: number;
+    folded: RunStreamState;
+    seCursor: number;
+    waveCursor: number;
+    timerCursor: number;
+    invalidated: boolean;
+    pendingDispatch: PendingDispatch[];
+    /** Tasks parked on an unresolved approval — never re-nudged by ticks. */
+    parkedTasks: Set<string>;
+}
+export declare function makeSession(store: PgStateStore, runId: string, input: string, raw: RecordedEvent[], eff: RecordedEvent[], parkedTasks?: Set<string>): RunSession;
+export declare function createWorkflowCtx(s: RunSession): WorkflowCtx;