@toren-run/core 0.1.0

package/dist/orchestrator.js

@@ -0,0 +1,146 @@
+import { randomUUID } from "node:crypto";
+import { effectiveEvents } from "./fold.js";
+import { ev } from "./events.js";
+import { withSpan } from "./tracing.js";
+import { createWorkflowCtx, foldRunStream, makeSession, RunLeaseLostError, WaveFailedError, WorkflowBlocked, } from "./workflow.js";
+const SESSION_WORKFLOW = async (ctx) => {
+    const w = await ctx.wave("session", [ctx.task("main", ctx.input)]);
+    return w.results[0]?.output ?? "";
+};
+export async function startRun(deps, req) {
+    const runId = req.runId ?? randomUUID();
+    const process = req.process ?? "main";
+    if (req.mode !== "session" && !deps.workflows[process]) {
+        throw new Error(`no process "${process}" for ${req.agent} (has: ${Object.keys(deps.workflows).join(", ")})`);
+    }
+    await deps.store.createRun({ runId, agent: req.agent, input: req.input, mode: req.mode, process });
+    const r = await deps.store.append(runId, "run", 0, [ev("RunCreated", { agent: req.agent, input: req.input, process })]);
+    if (!r.ok)
+        throw new Error("fresh run stream was not empty");
+    await deps.queue.send("orchestrator", { kind: "tick", runId, agent: req.agent, dedupeKey: `start-${runId}` });
+    return runId;
+}
+export async function tick(deps, runId) {
+    return withSpan("toren.run.tick", { "toren.run_id": runId }, () => tickImpl(deps, runId));
+}
+async function tickImpl(deps, runId) {
+    const lease = await deps.leases.acquire(runId, "run", `orch-${randomUUID()}`, 60);
+    if (!lease)
+        return "leased";
+    try {
+        const run = await deps.store.getRun(runId);
+        if (!run)
+            return "terminal";
+        if (run.status === "completed" || run.status === "failed" || run.status === "cancelled")
+            return "terminal";
+        // Sessions always converse with the crew's root agent through the
+        // implicit single-task workflow. A crew's custom batch workflow (which
+        // may parse structured input, fan out waves, etc.) never sees a chat.
+        const workflow = run.mode === "session" ? SESSION_WORKFLOW : deps.workflows[run.process ?? "main"];
+        if (!workflow)
+            throw new Error(`no process "${run.process ?? "main"}" for agent ${run.agent} (has: ${Object.keys(deps.workflows).join(", ")})`);
+        let raw = await deps.store.read(runId, "run");
+        let head = raw.at(-1)?.seq ?? 0;
+        let eff = effectiveEvents(raw);
+        let folded = foldRunStream(eff);
+        if (folded.terminal) {
+            await deps.store.updateRun(runId, { status: folded.terminal.status, output: folded.terminal.output, error: folded.terminal.error });
+            return "terminal";
+        }
+        const appendRun = async (events) => {
+            const r = await deps.store.append(runId, "run", head, events);
+            if (!r.ok)
+                throw new RunLeaseLostError("run stream advanced concurrently");
+            head = r.lastSeq;
+        };
+        if (!folded.started)
+            await appendRun([ev("RunStarted", {})]);
+        // Absorb task terminal states into the run stream;
+        // track parked tasks so waves don't re-nudge work waiting on a human.
+        const parkedTasks = new Set();
+        for (const w of folded.waves) {
+            if (w.settled)
+                continue;
+            for (const t of w.tasks) {
+                if (w.settledTasks.has(t.taskId))
+                    continue;
+                const taskEff = effectiveEvents(await deps.store.read(runId, `task:${t.taskId}`));
+                const completed = taskEff.find((e) => e.type === "TaskCompleted");
+                const failed = taskEff.find((e) => e.type === "TaskFailed" && !e.payload.willRetry);
+                if (completed) {
+                    await appendRun([ev("WaveTaskSettled", { waveId: w.waveId, taskId: t.taskId, status: "completed", output: completed.payload.result })]);
+                }
+                else if (failed) {
+                    await appendRun([ev("WaveTaskSettled", { waveId: w.waveId, taskId: t.taskId, status: "failed", error: failed.payload.error })]);
+                }
+                else {
+                    const resolved = new Set(taskEff.filter((e) => e.type === "ApprovalResolved").map((e) => String(e.payload.stepId)));
+                    const parkedOnApproval = taskEff.some((e) => e.type === "ApprovalRequested" && !resolved.has(String(e.payload.stepId)));
+                    // Sessions park at the turn boundary: an InputRequested with no
+                    // later UserMessage means the agent is waiting for a human.
+                    let lastInputSeq = 0, lastUserSeq = 0;
+                    for (const e of taskEff) {
+                        if (e.type === "InputRequested")
+                            lastInputSeq = e.seq;
+                        else if (e.type === "UserMessage")
+                            lastUserSeq = e.seq;
+                    }
+                    const parkedOnInput = lastInputSeq > 0 && lastUserSeq < lastInputSeq;
+                    if (parkedOnApproval || parkedOnInput)
+                        parkedTasks.add(t.taskId);
+                }
+            }
+        }
+        // Re-read so the workflow sees absorbed settlements.
+        raw = await deps.store.read(runId, "run");
+        eff = effectiveEvents(raw);
+        const session = makeSession(deps.store, runId, String(run.input ?? ""), raw, eff, parkedTasks);
+        const ctx = createWorkflowCtx(session);
+        const flush = async () => {
+            for (const d of session.pendingDispatch) {
+                await deps.queue.send(d.queue, { ...d.msg, agent: run.agent }, { delaySeconds: d.delaySeconds });
+            }
+            session.pendingDispatch = [];
+        };
+        try {
+            const output = await workflow(ctx);
+            const r = await deps.store.append(runId, "run", session.head, [ev("RunCompleted", { output })]);
+            if (!r.ok)
+                throw new RunLeaseLostError("run stream advanced concurrently");
+            await deps.store.updateRun(runId, { status: "completed", output });
+            await flush();
+            return "completed";
+        }
+        catch (e) {
+            if (e instanceof WorkflowBlocked) {
+                await flush();
+                await deps.store.updateRun(runId, { status: "running" });
+                return "blocked";
+            }
+            if (e instanceof RunLeaseLostError)
+                throw e;
+            const error = e instanceof WaveFailedError ? e.message : `workflow error: ${e instanceof Error ? e.message : String(e)}`;
+            const r = await deps.store.append(runId, "run", session.head, [ev("RunFailed", { error })]);
+            if (r.ok)
+                await deps.store.updateRun(runId, { status: "failed", error });
+            await flush();
+            return "failed";
+        }
+    }
+    finally {
+        await deps.leases.release(lease);
+    }
+}
+/** Locates a planned task's spec by scanning the run stream (workers use this). */
+export async function findTaskSpec(store, runId, taskId) {
+    const eff = effectiveEvents(await store.read(runId, "run"));
+    for (const e of eff) {
+        if (e.type !== "WavePlanned")
+            continue;
+        const tasks = e.payload.tasks;
+        const hit = tasks.find((t) => t.taskId === taskId);
+        if (hit)
+            return { agentRef: hit.agentRef, input: hit.input };
+    }
+    return null;
+}

package/dist/providers/echo.d.ts

@@ -0,0 +1,9 @@
+import type { ModelProvider, ModelRequest, ModelResponse } from "../model.js";
+/**
+ * Deterministic offline provider for demos, scaffolds, and tests: replies
+ * `echo(<first user text>)`. Lets `toren init` output run with zero API keys.
+ */
+export declare class EchoProvider implements ModelProvider {
+    calls: Map<string, number>;
+    complete(req: ModelRequest): Promise<ModelResponse>;
+}

package/dist/providers/echo.js

@@ -0,0 +1,17 @@
+/**
+ * Deterministic offline provider for demos, scaffolds, and tests: replies
+ * `echo(<first user text>)`. Lets `toren init` output run with zero API keys.
+ */
+export class EchoProvider {
+    calls = new Map();
+    async complete(req) {
+        const first = req.messages[0]?.content.find((b) => b.type === "text");
+        const input = first && first.type === "text" ? first.text : "";
+        this.calls.set(input, (this.calls.get(input) ?? 0) + 1);
+        return {
+            content: [{ type: "text", text: `echo(${input})` }],
+            stopReason: "endTurn",
+            usage: { inputTokens: 1, outputTokens: 1 },
+        };
+    }
+}

package/dist/providers/mock.d.ts

@@ -0,0 +1,12 @@
+import type { ModelProvider, ModelRequest, ModelResponse } from "../model.js";
+type Scripted = Omit<ModelResponse, "usage"> & {
+    usage?: ModelResponse["usage"];
+};
+export declare class MockProvider implements ModelProvider {
+    private script;
+    calls: number;
+    private i;
+    constructor(script: Scripted[]);
+    complete(_req: ModelRequest): Promise<ModelResponse>;
+}
+export {};

package/dist/providers/mock.js

@@ -0,0 +1,15 @@
+export class MockProvider {
+    script;
+    calls = 0;
+    i = 0;
+    constructor(script) {
+        this.script = script;
+    }
+    async complete(_req) {
+        this.calls += 1;
+        const next = this.script[this.i++];
+        if (!next)
+            throw new Error(`MockProvider script exhausted after ${this.script.length} responses`);
+        return { usage: { inputTokens: 10, outputTokens: 10 }, ...next };
+    }
+}

package/dist/queue.d.ts

@@ -0,0 +1,68 @@
+import type pg from "pg";
+export type QueueName = "orchestrator" | "tasks-short" | "tasks-long";
+export interface QueueMessage {
+    kind: "tick" | "task";
+    runId: string;
+    taskId?: string;
+    dedupeKey: string;
+    /** Which agent's run this belongs to — workers serving several agents route by it. */
+    agent?: string;
+}
+export interface Delivery {
+    message: QueueMessage;
+    receipt: number | string;
+    attempt: number;
+}
+/**
+ * The queue seam: at-least-once delivery with visibility
+ * timeouts and a dead-letter policy. Messages are hints, never truth.
+ * Local: Postgres SKIP LOCKED. AWS: SQS (@toren-run/adapters-aws).
+ */
+export interface QueueAdapter {
+    send(queue: QueueName, msg: QueueMessage, opts?: {
+        delaySeconds?: number;
+        maxAttempts?: number;
+    }): Promise<void>;
+    /**
+     * opts.agents scopes claiming to those agents' messages (unlabeled legacy
+     * messages included) — separate worker fleets share one queue table without
+     * stealing each other's hints. Adapters that can't filter server-side (SQS)
+     * may deliver everything; workers ack-and-skip foreign hints regardless.
+     */
+    receive(queue: QueueName, opts: {
+        max: number;
+        visibilitySeconds: number;
+        agents?: string[];
+    }): Promise<Delivery[]>;
+    extend(d: Delivery, visibilitySeconds: number): Promise<void>;
+    ack(d: Delivery): Promise<void>;
+    nack(d: Delivery, opts?: {
+        delaySeconds?: number;
+    }): Promise<void>;
+    /** Messages in flight or ready (delayed excluded) — used by drain/tests. Scopable like receive. */
+    depth(opts?: {
+        agents?: string[];
+    }): Promise<number>;
+}
+export declare class PgQueue implements QueueAdapter {
+    private pool;
+    constructor(pool: pg.Pool);
+    send(queue: QueueName, msg: QueueMessage, opts?: {
+        delaySeconds?: number;
+        maxAttempts?: number;
+    }): Promise<void>;
+    receive(queue: QueueName, opts: {
+        max: number;
+        visibilitySeconds: number;
+        agents?: string[];
+    }): Promise<Delivery[]>;
+    /** Messages that are in flight or ready for delivery (delayed ones excluded). */
+    depth(opts?: {
+        agents?: string[];
+    }): Promise<number>;
+    extend(d: Delivery, visibilitySeconds: number): Promise<void>;
+    ack(d: Delivery): Promise<void>;
+    nack(d: Delivery, opts?: {
+        delaySeconds?: number;
+    }): Promise<void>;
+}

package/dist/queue.js

@@ -0,0 +1,54 @@
+export class PgQueue {
+    pool;
+    constructor(pool) {
+        this.pool = pool;
+    }
+    async send(queue, msg, opts) {
+        await this.pool.query(`INSERT INTO toren_control.queue_messages (queue, payload, dedupe_key, visible_at, max_attempts)
+       VALUES ($1, $2, $3, now() + make_interval(secs => $4), $5)`, [queue, JSON.stringify(msg), msg.dedupeKey, opts?.delaySeconds ?? 0, opts?.maxAttempts ?? 5]);
+    }
+    async receive(queue, opts) {
+        // Move exhausted messages to the DLQ first, then claim visible ones.
+        await this.pool.query(`WITH exhausted AS (
+         DELETE FROM toren_control.queue_messages
+         WHERE queue = $1 AND attempts >= max_attempts
+           AND (locked_until IS NULL OR locked_until <= now())
+         RETURNING id, queue, payload, attempts
+       )
+       INSERT INTO toren_control.dead_letters (id, queue, payload, attempts)
+       SELECT id, queue, payload, attempts FROM exhausted ON CONFLICT DO NOTHING`, [queue]);
+        const r = await this.pool.query(`UPDATE toren_control.queue_messages m
+       SET locked_until = now() + make_interval(secs => $3), attempts = attempts + 1
+       WHERE m.id IN (
+         SELECT id FROM toren_control.queue_messages
+         WHERE queue = $1 AND visible_at <= now()
+           AND (locked_until IS NULL OR locked_until <= now())
+           AND attempts < max_attempts
+           AND ($4::text[] IS NULL OR payload->>'agent' IS NULL OR payload->>'agent' = ANY($4))
+         ORDER BY id
+         LIMIT $2
+         FOR UPDATE SKIP LOCKED
+       )
+       RETURNING id, payload, attempts`, [queue, opts.max, opts.visibilitySeconds, opts.agents ?? null]);
+        return r.rows.map((row) => ({ message: row.payload, receipt: Number(row.id), attempt: Number(row.attempts) }));
+    }
+    /** Messages that are in flight or ready for delivery (delayed ones excluded). */
+    async depth(opts) {
+        const r = await this.pool.query(`SELECT count(*)::int AS n FROM toren_control.queue_messages
+       WHERE ((locked_until IS NOT NULL AND locked_until > now())
+          OR (visible_at <= now() AND (locked_until IS NULL OR locked_until <= now()) AND attempts < max_attempts))
+         AND ($1::text[] IS NULL OR payload->>'agent' IS NULL OR payload->>'agent' = ANY($1))`, [opts?.agents ?? null]);
+        return r.rows[0].n;
+    }
+    async extend(d, visibilitySeconds) {
+        await this.pool.query(`UPDATE toren_control.queue_messages SET locked_until = now() + make_interval(secs => $2) WHERE id = $1`, [d.receipt, visibilitySeconds]);
+    }
+    async ack(d) {
+        await this.pool.query(`DELETE FROM toren_control.queue_messages WHERE id = $1`, [d.receipt]);
+    }
+    async nack(d, opts) {
+        await this.pool.query(`UPDATE toren_control.queue_messages
+       SET locked_until = NULL, visible_at = now() + make_interval(secs => $2)
+       WHERE id = $1`, [d.receipt, opts?.delaySeconds ?? 0]);
+    }
+}

package/dist/schedules.d.ts

@@ -0,0 +1,61 @@
+import type pg from "pg";
+import { type TickDeps } from "./orchestrator.js";
+/**
+ * Cron schedules with exactly-once firing. Schedules live in Postgres — no
+ * process ever holds a timer, so nothing is lost when processes die. Firing
+ * is two-phase, mirroring the hints-not-truth discipline:
+ *
+ *   Phase A (one transaction): claim due schedules (FOR UPDATE SKIP LOCKED),
+ *   advance next_fire_at, and insert a fire record keyed (schedule, moment)
+ *   carrying a pre-assigned run id. Commit. The fire record is now truth.
+ *
+ *   Phase B (idempotent): for every unsettled fire record, ensure its run
+ *   exists (create is a no-op if a racing worker won), nudge the queue, and
+ *   mark the fire settled.
+ *
+ * Any crash between any two writes is healed by the next sweep: no missed
+ * fires, no duplicate runs — proven by the schedule kill-matrix test.
+ */
+export interface ScheduleRecord {
+    id: string;
+    agent: string;
+    name: string;
+    cron: string;
+    tz: string;
+    input: string;
+    process: string;
+    enabled: boolean;
+    nextFireAt: Date;
+    lastFiredAt: Date | null;
+    createdAt: Date;
+}
+/** Next occurrence strictly after `after`. Throws on an invalid expression or timezone. */
+export declare function nextFire(cron: string, tz: string, after?: Date): Date;
+export declare function createSchedule(pool: pg.Pool, req: {
+    agent: string;
+    name: string;
+    cron: string;
+    input: string;
+    process?: string;
+    tz?: string;
+}): Promise<ScheduleRecord>;
+export declare function listSchedules(pool: pg.Pool, agents?: string[]): Promise<ScheduleRecord[]>;
+/** Pausing keeps the row; resuming recomputes next_fire_at from now (no surprise catch-up burst). */
+export declare function setScheduleEnabled(pool: pg.Pool, id: string, enabled: boolean): Promise<boolean>;
+export declare function deleteSchedule(pool: pg.Pool, id: string): Promise<boolean>;
+export interface FireRecord {
+    scheduleId: string;
+    scheduledFor: Date;
+    runId: string;
+    agent: string;
+    process: string;
+    firedAt: Date;
+    settled: boolean;
+}
+export declare function listFires(pool: pg.Pool, scheduleId: string, limit?: number): Promise<FireRecord[]>;
+/**
+ * One sweep pass: fire everything due for the served agents. Safe to run
+ * concurrently from any number of workers. Returns the number of runs whose
+ * fires were settled this pass.
+ */
+export declare function sweepSchedules(pool: pg.Pool, byAgent: Record<string, TickDeps>): Promise<number>;

package/dist/schedules.js

@@ -0,0 +1,128 @@
+import { randomUUID } from "node:crypto";
+import { Cron } from "croner";
+import { startRun } from "./orchestrator.js";
+const row = (r) => ({
+    id: String(r.id),
+    agent: String(r.agent),
+    name: String(r.name),
+    cron: String(r.cron),
+    tz: String(r.tz),
+    input: String(r.input),
+    process: String(r.process ?? "main"),
+    enabled: Boolean(r.enabled),
+    nextFireAt: r.next_fire_at,
+    lastFiredAt: r.last_fired_at ?? null,
+    createdAt: r.created_at,
+});
+/** Next occurrence strictly after `after`. Throws on an invalid expression or timezone. */
+export function nextFire(cron, tz, after = new Date()) {
+    const next = new Cron(cron, { timezone: tz }).nextRun(after);
+    if (!next)
+        throw new Error(`cron "${cron}" has no future occurrences`);
+    return next;
+}
+export async function createSchedule(pool, req) {
+    const tz = req.tz ?? "UTC";
+    const first = nextFire(req.cron, tz); // validates expression + tz
+    const res = await pool.query(`INSERT INTO toren_control.schedules (id, agent, name, cron, tz, input, process, next_fire_at)
+     VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *`, [randomUUID(), req.agent, req.name, req.cron, tz, req.input, req.process ?? "main", first]);
+    return row(res.rows[0]);
+}
+export async function listSchedules(pool, agents) {
+    const res = await pool.query(`SELECT * FROM toren_control.schedules
+     WHERE ($1::text[] IS NULL OR agent = ANY($1))
+     ORDER BY created_at DESC`, [agents ?? null]);
+    return res.rows.map(row);
+}
+/** Pausing keeps the row; resuming recomputes next_fire_at from now (no surprise catch-up burst). */
+export async function setScheduleEnabled(pool, id, enabled) {
+    if (!enabled) {
+        const res = await pool.query(`UPDATE toren_control.schedules SET enabled = false WHERE id = $1`, [id]);
+        return (res.rowCount ?? 0) > 0;
+    }
+    const cur = await pool.query(`SELECT cron, tz FROM toren_control.schedules WHERE id = $1`, [id]);
+    if (!cur.rows[0])
+        return false;
+    const next = nextFire(String(cur.rows[0].cron), String(cur.rows[0].tz));
+    const res = await pool.query(`UPDATE toren_control.schedules SET enabled = true, next_fire_at = $2 WHERE id = $1`, [id, next]);
+    return (res.rowCount ?? 0) > 0;
+}
+export async function deleteSchedule(pool, id) {
+    const res = await pool.query(`DELETE FROM toren_control.schedules WHERE id = $1`, [id]);
+    return (res.rowCount ?? 0) > 0;
+}
+export async function listFires(pool, scheduleId, limit = 20) {
+    const res = await pool.query(`SELECT * FROM toren_control.schedule_fires WHERE schedule_id = $1 ORDER BY scheduled_for DESC LIMIT $2`, [scheduleId, limit]);
+    return res.rows.map((r) => ({
+        scheduleId: String(r.schedule_id),
+        scheduledFor: r.scheduled_for,
+        runId: String(r.run_id),
+        agent: String(r.agent),
+        process: String(r.process ?? "main"),
+        firedAt: r.fired_at,
+        settled: Boolean(r.settled),
+    }));
+}
+/**
+ * One sweep pass: fire everything due for the served agents. Safe to run
+ * concurrently from any number of workers. Returns the number of runs whose
+ * fires were settled this pass.
+ */
+export async function sweepSchedules(pool, byAgent) {
+    const agents = Object.keys(byAgent);
+    if (agents.length === 0)
+        return 0;
+    // Phase A — claim + advance + record intent, atomically per schedule.
+    const client = await pool.connect();
+    try {
+        await client.query("BEGIN");
+        const due = await client.query(`SELECT * FROM toren_control.schedules
+       WHERE enabled AND next_fire_at <= now() AND agent = ANY($1)
+       FOR UPDATE SKIP LOCKED`, [agents]);
+        for (const s of due.rows) {
+            // Missed occurrences while workers were down collapse into this one
+            // catch-up fire; the fire record keeps the originally scheduled time
+            // so lateness stays visible.
+            await client.query(`INSERT INTO toren_control.schedule_fires (schedule_id, scheduled_for, run_id, agent, input, process)
+         VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING`, [s.id, s.next_fire_at, randomUUID(), s.agent, s.input, s.process ?? "main"]);
+            await client.query(`UPDATE toren_control.schedules SET next_fire_at = $2, last_fired_at = now() WHERE id = $1`, [s.id, nextFire(String(s.cron), String(s.tz))]);
+        }
+        await client.query("COMMIT");
+    }
+    catch (e) {
+        await client.query("ROLLBACK");
+        throw e;
+    }
+    finally {
+        client.release();
+    }
+    // Phase B — fulfill every unsettled fire, idempotently.
+    const open = await pool.query(`SELECT * FROM toren_control.schedule_fires WHERE NOT settled AND agent = ANY($1)`, [agents]);
+    let settled = 0;
+    for (const f of open.rows) {
+        const deps = byAgent[String(f.agent)];
+        if (!deps)
+            continue;
+        const runId = String(f.run_id);
+        const existing = await deps.store.getRun(runId);
+        if (!existing) {
+            try {
+                await startRun(deps, { agent: String(f.agent), input: String(f.input), runId, process: String(f.process ?? "main") });
+            }
+            catch {
+                // A racing worker created it between our check and insert — fine,
+                // the run exists; settle below. Anything else retries next sweep.
+                if (!(await deps.store.getRun(runId)))
+                    continue;
+            }
+        }
+        else {
+            // Run exists but the fire may have crashed before the queue nudge —
+            // re-send; messages are hints, duplicates no-op.
+            await deps.queue.send("orchestrator", { kind: "tick", runId, agent: String(f.agent), dedupeKey: `start-${runId}` });
+        }
+        await pool.query(`UPDATE toren_control.schedule_fires SET settled = true WHERE schedule_id = $1 AND scheduled_for = $2`, [f.schedule_id, f.scheduled_for]);
+        settled += 1;
+    }
+    return settled;
+}

package/dist/spawn.d.ts

@@ -0,0 +1,14 @@
+import type pg from "pg";
+import { type TickDeps } from "./orchestrator.js";
+import type { ProcessesCtx } from "./tools.js";
+/** Deterministic uuid (v4 shape) from a key — same key, same run. */
+export declare function deterministicRunId(key: string): string;
+export declare function makeProcessesFacet(pool: pg.Pool, agentName: string, deps: TickDeps, opts?: {
+    defaultProcess?: string;
+}): ProcessesCtx;
+/**
+ * One sweep pass: wake every session whose watched child has settled. Safe to
+ * run concurrently from any number of workers (the wake path CAS-es on the
+ * session stream via sendSessionMessage). Returns the number of wakes delivered.
+ */
+export declare function sweepWatchers(pool: pg.Pool, byAgent: Record<string, TickDeps>): Promise<number>;

package/dist/spawn.js

@@ -0,0 +1,113 @@
+import { createHash } from "node:crypto";
+import { effectiveEvents } from "./fold.js";
+import { startRun } from "./orchestrator.js";
+import { sendSessionMessage, SessionBusyError } from "./conversations.js";
+import { foldRunStream } from "./workflow.js";
+/**
+ * The spawn arc: a conversation triggers a named process as a background run,
+ * and the runtime messages the user when it settles.
+ *
+ * Spawning is effectively-once: the child runId derives deterministically from
+ * (parent run, task, toolUseId), so the crash-window re-run of the keyed tool
+ * finds the child already exists. The watcher row is written BEFORE the child
+ * run so no settlement can slip between the two; an orphaned watcher (crash
+ * before the child was created, run abandoned) settles after a grace period.
+ *
+ * The wake reuses the session machinery wholesale: sweepWatchers appends a
+ * normal UserMessage (channel "watcher") via sendSessionMessage, so strict
+ * turn-taking holds — a mid-turn session throws SessionBusyError and the wake
+ * lands on a later sweep — and every channel delivers the agent's reply like
+ * any other turn.
+ */
+const ORPHAN_GRACE_MS = 60 * 60 * 1000;
+/** Deterministic uuid (v4 shape) from a key — same key, same run. */
+export function deterministicRunId(key) {
+    const h = createHash("sha256").update(key).digest();
+    h[6] = (h[6] & 0x0f) | 0x40;
+    h[8] = (h[8] & 0x3f) | 0x80;
+    const hex = h.subarray(0, 16).toString("hex");
+    return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`;
+}
+export function makeProcessesFacet(pool, agentName, deps, opts = {}) {
+    return {
+        get names() { return Object.keys(deps.workflows); },
+        defaultProcess: opts.defaultProcess,
+        async start(req) {
+            if (!deps.workflows[req.process]) {
+                throw new Error(`no process "${req.process}" for ${agentName} (has: ${Object.keys(deps.workflows).join(", ")})`);
+            }
+            const runId = deterministicRunId(`spawn:${req.parentRunId}:${req.parentTaskId}:${req.toolUseId}`);
+            await pool.query(`INSERT INTO toren_control.run_watchers (child_run_id, parent_run_id, agent, process)
+         VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING`, [runId, req.parentRunId, agentName, req.process]);
+            if (await deps.store.getRun(runId))
+                return { runId, started: false };
+            await startRun(deps, { agent: agentName, input: req.input, process: req.process, runId });
+            return { runId, started: true };
+        },
+        async status(runId) {
+            const run = await deps.store.getRun(runId);
+            if (!run)
+                return null;
+            const folded = foldRunStream(effectiveEvents(await deps.store.read(runId, "run")));
+            const cap = (v) => (v == null ? undefined : String(v).slice(0, 4000));
+            return {
+                runId, process: run.process, status: run.status,
+                ...(run.output != null ? { output: cap(run.output) } : {}),
+                ...(run.error != null ? { error: cap(run.error) } : {}),
+                waves: folded.waves.map((w) => ({ name: w.name, tasks: w.tasks.length, settled: w.settledTasks.size, done: w.settled })),
+            };
+        },
+    };
+}
+const TERMINAL = new Set(["completed", "failed", "cancelled"]);
+function wakeText(child) {
+    const trim = (v) => { const s = String(v ?? ""); return s.length > 1500 ? `${s.slice(0, 1500)} …[truncated]` : s; };
+    return child.status === "completed"
+        ? `[background run] process "${child.process}" (run ${child.runId}) finished:\n${trim(child.output)}`
+        : `[background run] process "${child.process}" (run ${child.runId}) ${child.status.toUpperCase()}: ${trim(child.error)}`;
+}
+/**
+ * One sweep pass: wake every session whose watched child has settled. Safe to
+ * run concurrently from any number of workers (the wake path CAS-es on the
+ * session stream via sendSessionMessage). Returns the number of wakes delivered.
+ */
+export async function sweepWatchers(pool, byAgent) {
+    const agents = Object.keys(byAgent);
+    if (agents.length === 0)
+        return 0;
+    const settle = (childRunId) => pool.query(`UPDATE toren_control.run_watchers SET settled = true WHERE child_run_id = $1`, [childRunId]);
+    const { rows } = await pool.query(`SELECT * FROM toren_control.run_watchers WHERE NOT settled AND agent = ANY($1)`, [agents]);
+    let woken = 0;
+    for (const w of rows) {
+        const deps = byAgent[String(w.agent)];
+        if (!deps)
+            continue;
+        const childId = String(w.child_run_id);
+        const child = await deps.store.getRun(childId);
+        if (!child) {
+            // Crash before the child run was created and the parent never replayed.
+            if (Date.now() - w.created_at.getTime() > ORPHAN_GRACE_MS)
+                await settle(childId);
+            continue;
+        }
+        if (!TERMINAL.has(child.status))
+            continue;
+        const parent = await deps.store.getRun(String(w.parent_run_id));
+        if (!parent || parent.mode !== "session" || TERMINAL.has(parent.status)) {
+            // Nothing to wake: check_run remains the pull path for batch parents.
+            await settle(childId);
+            continue;
+        }
+        try {
+            await sendSessionMessage(deps, String(w.parent_run_id), { text: wakeText(child), channel: "watcher" });
+            await settle(childId);
+            woken += 1;
+        }
+        catch (e) {
+            if (e instanceof SessionBusyError)
+                continue; // mid-turn — the next sweep delivers
+            continue; // transient (DB flap, racing close) — retry next sweep
+        }
+    }
+    return woken;
+}