@toren-run/core 0.1.1 → 0.1.3

package/dist/index.d.ts

@@ -25,3 +25,5 @@ export * from "./worker.js";
 export * from "./guardians.js";
 export * from "./conversations.js";
 export * from "./spawn.js";
+export * from "./usage.js";
+export * from "./tail.js";

package/dist/index.js

@@ -25,3 +25,5 @@ export * from "./worker.js";
 export * from "./guardians.js";
 export * from "./conversations.js";
 export * from "./spawn.js";
+export * from "./usage.js";
+export * from "./tail.js";

package/dist/loop.d.ts

@@ -22,6 +22,10 @@ export interface AgentSpec {
     tools: ToolDefAny[];
     maxTokens: number;
     maxSteps: number;
+    /** Passed to reasoning models (agent.yaml `reasoning_effort:`); absent leaves request digests untouched. */
+    reasoningEffort?: string;
+    /** Opt-in poison-pill (agent.yaml `limits.maxAttemptsPerTask`): terminally fail a task after this many attempts instead of retrying forever. */
+    maxTaskAttempts?: number;
     outputSchema?: z.ZodTypeAny;
     /** Declared env values (from agent.yaml `env:`) passed to tool handlers as ctx.env. */
     env?: Record<string, string>;

package/dist/loop.js

@@ -93,6 +93,11 @@ async function runTaskLoopImpl(args) {
         invalidated = true;
     }
     const attempt = raw.filter((e) => e.type === "TaskStarted").length + 1;
+    if (agent.maxTaskAttempts && attempt > agent.maxTaskAttempts) {
+        const error = `gave up after ${agent.maxTaskAttempts} attempts (limits.maxAttemptsPerTask); the run's recorded error holds the last failure reason`;
+        await append([ev("TaskFailed", { error, willRetry: false })]);
+        return { status: "failed", error };
+    }
     await append([ev("TaskStarted", { attempt })]);
     const messages = [{ role: "user", content: [{ type: "text", text: args.input }] }];
     const specs = toolSpecs(agent.tools);
@@ -129,14 +134,24 @@ async function runTaskLoopImpl(args) {
             else {
                 // Crash window: call was issued but the response never landed. Re-issue (at-least-once).
                 ptr += 1;
-                response = await withSpan("toren.llm", { "gen_ai.request.model": request.model }, () => provider.complete(request));
+                response = await withSpan("toren.llm", { "gen_ai.request.model": request.model }, async (span) => {
+                    const r = await provider.complete(request);
+                    if (r.usage)
+                        span.setAttributes({ "gen_ai.usage.input_tokens": r.usage.inputTokens, "gen_ai.usage.output_tokens": r.usage.outputTokens });
+                    return r;
+                });
                 await append([ev("LlmCallCompleted", { stepId: next.payload.stepId, response, usage: response.usage })]);
             }
         }
         else {
             const stepId = `s${head + 1}`;
             await append([ev("LlmCallStarted", { stepId, requestDigest: digest, model: request.model })]);
-            response = await withSpan("toren.llm", { "gen_ai.request.model": request.model }, () => provider.complete(request));
+            response = await withSpan("toren.llm", { "gen_ai.request.model": request.model }, async (span) => {
+                const r = await provider.complete(request);
+                if (r.usage)
+                    span.setAttributes({ "gen_ai.usage.input_tokens": r.usage.inputTokens, "gen_ai.usage.output_tokens": r.usage.outputTokens });
+                return r;
+            });
             await append([ev("LlmCallCompleted", { stepId, response, usage: response.usage })]);
         }
         if (response.usage) {
@@ -337,7 +352,7 @@ async function runTaskLoopImpl(args) {
             return { status: "failed", error };
         }
         await maybeCompact();
-        const request = { model: agent.model, system, messages: [...messages], tools: specs, maxTokens: agent.maxTokens };
+        const request = { model: agent.model, system, messages: [...messages], tools: specs, maxTokens: agent.maxTokens, ...(agent.reasoningEffort ? { reasoningEffort: agent.reasoningEffort } : {}) };
         const response = await recordedLlmCall(request);
         messages.push({ role: "assistant", content: response.content });
         if (response.stopReason === "toolUse") {

package/dist/model.d.ts

@@ -27,6 +27,8 @@ export interface ModelRequest {
     messages: ChatMessage[];
     tools: ToolSpec[];
     maxTokens: number;
+    /** OpenAI reasoning models: "none" | "low" | "medium" | "high". Absent keeps old request digests byte-identical. */
+    reasoningEffort?: string;
 }
 export type StopReason = "endTurn" | "toolUse" | "maxTokens" | "refusal";
 export interface ModelResponse {

package/dist/orchestrator.d.ts

@@ -27,6 +27,13 @@ export declare function startRun(deps: TickDeps, req: {
     runId?: string;
     mode?: "task" | "session";
 }): Promise<string>;
+/**
+ * Retire a run from outside: appends RunCancelled and marks the row, so every
+ * queued hint for it becomes a no-op and retries stop. The escape hatch for a
+ * run stuck retrying a permanently broken dependency. Returns false for an
+ * unknown run; true if the run is (now) cancelled.
+ */
+export declare function cancelRun(deps: TickDeps, runId: string, reason?: string): Promise<boolean>;
 export declare function tick(deps: TickDeps, runId: string): Promise<TickResult>;
 /** Locates a planned task's spec by scanning the run stream (workers use this). */
 export declare function findTaskSpec(store: PgStateStore, runId: string, taskId: string): Promise<{

package/dist/orchestrator.js

@@ -20,6 +20,31 @@ export async function startRun(deps, req) {
     await deps.queue.send("orchestrator", { kind: "tick", runId, agent: req.agent, dedupeKey: `start-${runId}` });
     return runId;
 }
+/**
+ * Retire a run from outside: appends RunCancelled and marks the row, so every
+ * queued hint for it becomes a no-op and retries stop. The escape hatch for a
+ * run stuck retrying a permanently broken dependency. Returns false for an
+ * unknown run; true if the run is (now) cancelled.
+ */
+export async function cancelRun(deps, runId, reason = "cancelled by operator") {
+    for (let i = 0; i < 5; i++) {
+        const run = await deps.store.getRun(runId);
+        if (!run)
+            return false;
+        if (run.status === "cancelled")
+            return true;
+        if (run.status === "completed" || run.status === "failed")
+            return false;
+        const head = (await deps.store.read(runId, "run")).at(-1)?.seq ?? 0;
+        const r = await deps.store.append(runId, "run", head, [ev("RunCancelled", { reason })]);
+        if (r.ok) {
+            await deps.store.updateRun(runId, { status: "cancelled", error: reason });
+            return true;
+        }
+        // A worker advanced the stream between read and append; look again.
+    }
+    throw new Error(`could not cancel ${runId}: the run stream kept advancing`);
+}
 export async function tick(deps, runId) {
     return withSpan("toren.run.tick", { "toren.run_id": runId }, () => tickImpl(deps, runId));
 }
@@ -107,14 +132,15 @@ async function tickImpl(deps, runId) {
             const r = await deps.store.append(runId, "run", session.head, [ev("RunCompleted", { output })]);
             if (!r.ok)
                 throw new RunLeaseLostError("run stream advanced concurrently");
-            await deps.store.updateRun(runId, { status: "completed", output });
+            await deps.store.updateRun(runId, { status: "completed", output, error: null });
             await flush();
             return "completed";
         }
         catch (e) {
             if (e instanceof WorkflowBlocked) {
                 await flush();
-                await deps.store.updateRun(runId, { status: "running" });
+                // Parking cleanly is progress: a transient error recorded by a worker retry is stale now.
+                await deps.store.updateRun(runId, { status: "running", error: null });
                 return "blocked";
             }
             if (e instanceof RunLeaseLostError)

package/dist/tail.d.ts

@@ -0,0 +1,16 @@
+import type { PgStateStore } from "./store.js";
+import type { RecordedEvent, StreamId } from "./events.js";
+/**
+ * Incremental follow of a run's event streams, for `toren jobs tail` and the
+ * SSE endpoint. The cursor is per-stream last-seen seq; callers poll and get
+ * only what landed since. Streams are append-only, so this is exact, not
+ * best-effort.
+ */
+export type TailCursor = Partial<Record<string, number>>;
+export interface TailedEvent extends RecordedEvent {
+    streamId: StreamId;
+}
+export declare function followRun(store: PgStateStore, runId: string, cursor: TailCursor): Promise<{
+    events: TailedEvent[];
+    done: boolean;
+}>;

package/dist/tail.js

@@ -0,0 +1,27 @@
+const TERMINAL = new Set(["completed", "failed", "cancelled"]);
+export async function followRun(store, runId, cursor) {
+    const events = [];
+    const pull = async (streamId) => {
+        const after = cursor[streamId] ?? 0;
+        for (const e of await store.read(runId, streamId)) {
+            if (e.seq <= after)
+                continue;
+            events.push({ ...e, streamId });
+            cursor[streamId] = e.seq;
+        }
+    };
+    await pull("run");
+    // Task streams come from every wave ever planned; the run stream is truth.
+    const taskIds = new Set();
+    for (const e of await store.read(runId, "run")) {
+        if (e.type !== "WavePlanned")
+            continue;
+        for (const t of e.payload.tasks)
+            taskIds.add(t.taskId);
+    }
+    for (const taskId of taskIds)
+        await pull(`task:${taskId}`);
+    events.sort((a, b) => a.recordedAt.getTime() - b.recordedAt.getTime());
+    const run = await store.getRun(runId);
+    return { events, done: !run || TERMINAL.has(run.status) };
+}

package/dist/usage.d.ts

@@ -0,0 +1,29 @@
+import type { PgStateStore } from "./store.js";
+/**
+ * Cost roll-up for one run, straight from the event log. Every model call's
+ * usage rides in LlmCallCompleted; the raw streams (not the effective view)
+ * are the tally, because invalidated calls were still paid for. The number
+ * the product exists for: completed calls recorded before a later attempt
+ * replayed from the log instead of the provider, so their cost was paid once.
+ */
+export interface RunUsage {
+    models: Record<string, {
+        calls: number;
+        inputTokens: number;
+        outputTokens: number;
+    }>;
+    totalCalls: number;
+    /** Sum of TaskStarted across all task streams; > tasks means resumes happened. */
+    taskAttempts: number;
+    /** Completed calls that later attempts replayed from the log instead of re-buying. */
+    replayedCalls: number;
+    /** Present only when every used model has a known price. USD. */
+    estCostUsd?: number;
+    replaySavingsUsd?: number;
+}
+/** USD per million tokens. Approximate list prices; override or extend via TOREN_MODEL_PRICES (JSON, same shape). */
+export declare const MODEL_PRICES: Record<string, {
+    in: number;
+    out: number;
+}>;
+export declare function runUsage(store: PgStateStore, runId: string): Promise<RunUsage>;

package/dist/usage.js

@@ -0,0 +1,73 @@
+/** USD per million tokens. Approximate list prices; override or extend via TOREN_MODEL_PRICES (JSON, same shape). */
+export const MODEL_PRICES = {
+    "openai/gpt-4o": { in: 2.5, out: 10 },
+    "openai/gpt-4o-mini": { in: 0.15, out: 0.6 },
+    "mock/echo": { in: 0, out: 0 },
+    "mock/slow": { in: 0, out: 0 },
+    "mock/m": { in: 0, out: 0 },
+};
+function prices() {
+    const env = process.env.TOREN_MODEL_PRICES;
+    if (!env)
+        return MODEL_PRICES;
+    try {
+        return { ...MODEL_PRICES, ...JSON.parse(env) };
+    }
+    catch {
+        return MODEL_PRICES;
+    }
+}
+export async function runUsage(store, runId) {
+    const runEvents = await store.read(runId, "run");
+    const taskIds = new Set();
+    for (const e of runEvents) {
+        if (e.type !== "WavePlanned")
+            continue;
+        for (const t of e.payload.tasks)
+            taskIds.add(t.taskId);
+    }
+    const table = prices();
+    const models = {};
+    let totalCalls = 0, taskAttempts = 0, replayedCalls = 0;
+    let cost = 0, savings = 0, priceable = true;
+    for (const taskId of taskIds) {
+        const events = await store.read(runId, `task:${taskId}`);
+        const stepModel = new Map();
+        let completedSoFar = 0, costSoFar = 0;
+        for (const e of events) {
+            if (e.type === "TaskStarted") {
+                taskAttempts += 1;
+                if (Number(e.payload.attempt ?? 1) > 1) {
+                    replayedCalls += completedSoFar;
+                    savings += costSoFar;
+                }
+            }
+            else if (e.type === "LlmCallStarted") {
+                stepModel.set(String(e.payload.stepId), String(e.payload.model ?? ""));
+            }
+            else if (e.type === "LlmCallCompleted") {
+                const model = stepModel.get(String(e.payload.stepId)) ?? "";
+                const usage = (e.payload.usage ?? {});
+                const m = (models[model] ??= { calls: 0, inputTokens: 0, outputTokens: 0 });
+                m.calls += 1;
+                m.inputTokens += usage.inputTokens ?? 0;
+                m.outputTokens += usage.outputTokens ?? 0;
+                totalCalls += 1;
+                completedSoFar += 1;
+                const p = table[model];
+                if (p) {
+                    const c = ((usage.inputTokens ?? 0) * p.in + (usage.outputTokens ?? 0) * p.out) / 1_000_000;
+                    cost += c;
+                    costSoFar += c;
+                }
+                else {
+                    priceable = false;
+                }
+            }
+        }
+    }
+    return {
+        models, totalCalls, taskAttempts, replayedCalls,
+        ...(priceable && totalCalls > 0 ? { estCostUsd: cost, replaySavingsUsd: savings } : {}),
+    };
+}

package/dist/worker.d.ts

@@ -1,4 +1,6 @@
 import { type TickDeps } from "./orchestrator.js";
+/** Exponential backoff for task retries: 0.2s doubling per attempt, capped at 60s. */
+export declare const retryDelaySeconds: (attempt: number) => number;
 export interface WorkerOpts {
     concurrency?: number;
     visibilitySeconds?: number;

package/dist/worker.js

@@ -2,6 +2,8 @@ import { randomUUID } from "node:crypto";
 import { InvalidationStormError, runTaskLoop, TaskLeaseLostError } from "./loop.js";
 import { findTaskSpec, tick } from "./orchestrator.js";
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Exponential backoff for task retries: 0.2s doubling per attempt, capped at 60s. */
+export const retryDelaySeconds = (attempt) => Math.min(60, 0.2 * 2 ** Math.max(0, attempt - 1));
 /**
  * Local worker runtime: in-process pollers over the orchestrator and task
  * queues (local binding). One instance serves one agent or a whole
@@ -138,6 +140,11 @@ export class LocalWorkerRuntime {
                 if (!agent)
                     throw new Error(`no agent registered for ref ${spec.agentRef}`);
                 const run = await deps.store.getRun(msg.runId);
+                if (!run || run.status === "cancelled" || run.status === "completed" || run.status === "failed") {
+                    // A hint for a retired run: messages are never truth.
+                    await this.shared.queue.ack(d);
+                    return;
+                }
                 await runTaskLoop({
                     store: deps.store, provider: deps.provider,
                     runId: msg.runId, taskId, agent, input: spec.input, files: deps.files,
@@ -165,7 +172,18 @@ export class LocalWorkerRuntime {
                     await this.shared.queue.nack(d, { delaySeconds: 20 });
                     return;
                 }
-                await this.shared.queue.nack(d, { delaySeconds: 0.2 });
+                // An outside dependency failed (a provider, a tool, the network). The
+                // retry is durability doing its job; the reason must not vanish into
+                // it. Record it on the run so jobs/console answer "why is it stuck"
+                // while the retries continue; a later success clears it.
+                try {
+                    const reason = e instanceof Error ? e.message : String(e);
+                    await deps.store.updateRun(msg.runId, {
+                        error: `${reason}${msg.taskId ? ` (task ${msg.taskId}, attempt ${d.attempt})` : ""}`,
+                    });
+                }
+                catch { /* the store may be the thing that failed */ }
+                await this.shared.queue.nack(d, { delaySeconds: retryDelaySeconds(d.attempt) });
             }
             catch {
                 // Queue unreachable too: do nothing. The visibility timeout redelivers

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toren-run/core",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "type": "module",
   "exports": {
     ".": {