@sebastiantuyu/agest 0.3.2 → 0.3.3-next.2

package/dist/adapters/langchain.d.ts

@@ -1,6 +1,6 @@
 import type { AgentExecutor } from "../types";
 type Runnable = {
-    invoke: (input: any) => Promise<any>;
+    invoke: (input: any, options?: any) => Promise<any>;
 };
 type LangGraphGraph = Runnable & {
     lg_is_pregel: true;

package/dist/adapters/langchain.js

@@ -1,3 +1,5 @@
+import { computeCost } from "../pricing";
+import { createTracingHandle } from "./tracing";
 /**
  * Adapter for LangChain runnables and agents.
  *
@@ -19,23 +21,43 @@ export function langchain(runnable) {
 function langGraphAdapter(graph) {
     const staticTools = extractGraphTools(graph);
     return async (input) => {
+        const baseline = performance.now();
+        const tracing = await createTracingHandle(baseline);
         let result;
         try {
             const { HumanMessage } = await import("@langchain/core/messages");
-            result = await graph.invoke({ messages: [new HumanMessage(input)] });
+            result = await graph.invoke({ messages: [new HumanMessage(input)] }, { callbacks: tracing.callbacks });
         }
         catch (err) {
-            return { text: "", executionError: err.message, metadata: { tools: staticTools } };
+            const { events } = tracing.drain();
+            return {
+                text: "",
+                executionError: err.message,
+                metadata: { tools: staticTools, events: events.length ? events : undefined },
+            };
         }
         const messages = result.messages;
         const last = messages[messages.length - 1];
         const text = typeof last?.content === "string"
             ? last.content
             : JSON.stringify(last?.content ?? result);
-        const model = last?.response_metadata?.model_name;
+        const drained = tracing.drain();
+        const model = last?.response_metadata?.model_name ??
+            drained.modelName;
+        const { tokens, cost } = summarizeRun({
+            events: drained.events,
+            fallbackTokens: extractTokensFromMessage(last),
+            model,
+        });
         return {
             text,
-            metadata: { model, tools: staticTools, tokens: extractTokensFromMessage(last) },
+            metadata: {
+                model,
+                tools: staticTools,
+                tokens,
+                cost,
+                events: drained.events.length ? drained.events : undefined,
+            },
         };
     };
 }
@@ -48,33 +70,69 @@ function reactAgentAdapter(agent) {
         ?.map((t) => t.name ?? t.getName?.())
         .filter(Boolean);
     return async (input) => {
+        const baseline = performance.now();
+        const tracing = await createTracingHandle(baseline);
         let result;
         try {
-            result = await agent.invoke({ messages: [{ role: "human", content: input }] });
+            result = await agent.invoke({ messages: [{ role: "human", content: input }] }, { callbacks: tracing.callbacks });
         }
         catch (err) {
-            return { text: "", executionError: err.message, metadata: { model, systemPrompt, tools } };
+            const { events } = tracing.drain();
+            return {
+                text: "",
+                executionError: err.message,
+                metadata: {
+                    model,
+                    systemPrompt,
+                    tools,
+                    events: events.length ? events : undefined,
+                },
+            };
         }
         const messages = result.messages;
         const last = messages[messages.length - 1];
         const text = typeof last?.content === "string"
             ? last.content
             : JSON.stringify(last?.content ?? result);
+        const drained = tracing.drain();
+        const { tokens, cost } = summarizeRun({
+            events: drained.events,
+            fallbackTokens: extractTokensFromMessage(last),
+            model,
+        });
         return {
             text,
-            metadata: { model, systemPrompt, tools, tokens: extractTokensFromMessage(last) },
+            metadata: {
+                model,
+                systemPrompt,
+                tools,
+                tokens,
+                cost,
+                events: drained.events.length ? drained.events : undefined,
+            },
         };
     };
 }
 function chainAdapter(chain) {
     const { model, systemPrompt } = extractChainMeta(chain);
     return async (input) => {
+        const baseline = performance.now();
+        const tracing = await createTracingHandle(baseline);
         let result;
         try {
-            result = await chain.invoke({ input });
+            result = await chain.invoke({ input }, { callbacks: tracing.callbacks });
         }
         catch (err) {
-            return { text: "", executionError: err.message, metadata: { model, systemPrompt } };
+            const { events } = tracing.drain();
+            return {
+                text: "",
+                executionError: err.message,
+                metadata: {
+                    model,
+                    systemPrompt,
+                    events: events.length ? events : undefined,
+                },
+            };
         }
         const text = typeof result === "string"
             ? result
@@ -83,12 +141,21 @@ function chainAdapter(chain) {
                 : typeof result.content === "string"
                     ? result.content
                     : JSON.stringify(result);
+        const drained = tracing.drain();
+        const effectiveModel = model ?? drained.modelName ?? result.metadata?.model;
+        const { tokens, cost } = summarizeRun({
+            events: drained.events,
+            fallbackTokens: extractTokens(result),
+            model: effectiveModel,
+        });
         return {
             text,
             metadata: {
-                model: model ?? result.metadata?.model,
+                model: effectiveModel,
                 systemPrompt,
-                tokens: extractTokens(result),
+                tokens,
+                cost,
+                events: drained.events.length ? drained.events : undefined,
             },
         };
     };
@@ -153,3 +220,49 @@ function extractTokensFromMessage(msg) {
         output: usage.output_tokens ?? usage.completion_tokens ?? 0,
     };
 }
+function summarizeRun(input) {
+    const modelEvents = input.events.filter((e) => e.kind === "model");
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let providerCost = 0;
+    let hasProviderCost = false;
+    let hasTableCost = false;
+    let tableCost = 0;
+    let hasTokens = false;
+    for (const e of modelEvents) {
+        if (e.tokens) {
+            hasTokens = true;
+            inputTokens += e.tokens.input;
+            outputTokens += e.tokens.output;
+        }
+        if (e.cost?.source === "provider" && e.cost.totalUsd != null) {
+            hasProviderCost = true;
+            providerCost += e.cost.totalUsd;
+        }
+        else if (e.cost?.source === "table" && e.cost.totalUsd != null) {
+            hasTableCost = true;
+            tableCost += e.cost.totalUsd;
+        }
+    }
+    let tokens = hasTokens ? { input: inputTokens, output: outputTokens } : undefined;
+    if (!tokens && input.fallbackTokens)
+        tokens = input.fallbackTokens;
+    // Pick cost: provider > table > recompute from fallback tokens
+    let cost;
+    if (hasProviderCost) {
+        cost = { totalUsd: providerCost, source: "provider" };
+    }
+    else if (hasTableCost) {
+        cost = { totalUsd: tableCost, source: "table" };
+    }
+    else if (tokens && input.model) {
+        const computed = computeCost({
+            model: input.model,
+            inputTokens: tokens.input,
+            outputTokens: tokens.output,
+        });
+        if (computed.source !== "unavailable")
+            cost = computed;
+    }
+    return { tokens, cost };
+}

package/dist/adapters/remote.js

@@ -24,12 +24,13 @@
  */
 export function remote(endpoint, options = {}) {
     const { headers = {}, method = "POST", body: extraBody, buildRequest = defaultBuildRequest, parseResponse, metadata: staticMetadata, } = options;
-    return async (input) => {
+    return async (input, execOptions) => {
         let res;
         try {
             const fetchOptions = {
                 method,
                 headers: { "Content-Type": "application/json", ...headers },
+                signal: execOptions?.signal,
             };
             if (method !== "GET") {
                 const built = buildRequest(input);

package/dist/adapters/tracing.d.ts

@@ -0,0 +1,19 @@
+import type { TimelineEvent } from "../types";
+export interface TracingHandle {
+    /** Pass this into `runnable.invoke(..., { callbacks: [handler.callbacks] })` */
+    callbacks: any[];
+    drain(): {
+        events: TimelineEvent[];
+        modelName?: string;
+    };
+}
+/**
+ * Creates a LangChain callback handler that records every LLM and tool
+ * invocation as a `TimelineEvent`. Returns a handle whose `drain()` method
+ * yields the captured events with `startMs` / `endMs` relative to the
+ * provided baseline.
+ *
+ * Designed to fail open: any unexpected callback shape is ignored rather
+ * than throwing — the underlying agent run must not be broken by tracing.
+ */
+export declare function createTracingHandle(baselineMs: number): Promise<TracingHandle>;

package/dist/adapters/tracing.js

@@ -0,0 +1,200 @@
+import { computeCost } from "../pricing";
+/**
+ * Creates a LangChain callback handler that records every LLM and tool
+ * invocation as a `TimelineEvent`. Returns a handle whose `drain()` method
+ * yields the captured events with `startMs` / `endMs` relative to the
+ * provided baseline.
+ *
+ * Designed to fail open: any unexpected callback shape is ignored rather
+ * than throwing — the underlying agent run must not be broken by tracing.
+ */
+export async function createTracingHandle(baselineMs) {
+    // Import lazily so the adapter still works when @langchain/core is not present.
+    // BaseCallbackHandler is the runtime contract LangChain checks for.
+    let BaseCallbackHandler;
+    try {
+        ({ BaseCallbackHandler } = await import("@langchain/core/callbacks/base"));
+    }
+    catch {
+        return { callbacks: [], drain: () => ({ events: [] }) };
+    }
+    const events = [];
+    const openLLMs = new Map();
+    const openTools = new Map();
+    let lastModelName;
+    class AgestTracer extends BaseCallbackHandler {
+        name = "AgestTracer";
+        awaitHandlers = true;
+        handleLLMStart(llm, _prompts, runId, _parentRunId, extraParams) {
+            openLLMs.set(runId, {
+                startMs: now() - baselineMs,
+                name: extractModelName(llm, extraParams),
+            });
+        }
+        handleChatModelStart(llm, _messages, runId, _parentRunId, extraParams) {
+            openLLMs.set(runId, {
+                startMs: now() - baselineMs,
+                name: extractModelName(llm, extraParams),
+            });
+        }
+        handleLLMEnd(output, runId) {
+            const open = openLLMs.get(runId);
+            if (!open)
+                return;
+            openLLMs.delete(runId);
+            const endMs = now() - baselineMs;
+            const tokens = extractTokensFromLLMOutput(output);
+            const providerCost = extractProviderCost(output);
+            const name = open.name ?? extractModelNameFromOutput(output) ?? "model";
+            if (name && name !== "model")
+                lastModelName = name;
+            const cost = computeCost({
+                model: name,
+                inputTokens: tokens?.input,
+                outputTokens: tokens?.output,
+                providerCost,
+            });
+            events.push({
+                kind: "model",
+                name,
+                startMs: open.startMs,
+                endMs,
+                durationMs: Math.max(0, endMs - open.startMs),
+                tokens,
+                cost: stripCostIfEmpty(cost),
+            });
+        }
+        handleLLMError(err, runId) {
+            const open = openLLMs.get(runId);
+            if (!open)
+                return;
+            openLLMs.delete(runId);
+            const endMs = now() - baselineMs;
+            events.push({
+                kind: "model",
+                name: open.name ?? "model",
+                startMs: open.startMs,
+                endMs,
+                durationMs: Math.max(0, endMs - open.startMs),
+                error: err?.message ?? String(err),
+            });
+        }
+        handleToolStart(tool, _input, runId) {
+            openTools.set(runId, {
+                startMs: now() - baselineMs,
+                name: extractToolName(tool) ?? "tool",
+            });
+        }
+        handleToolEnd(_output, runId) {
+            const open = openTools.get(runId);
+            if (!open)
+                return;
+            openTools.delete(runId);
+            const endMs = now() - baselineMs;
+            events.push({
+                kind: "tool",
+                name: open.name,
+                startMs: open.startMs,
+                endMs,
+                durationMs: Math.max(0, endMs - open.startMs),
+            });
+        }
+        handleToolError(err, runId) {
+            const open = openTools.get(runId);
+            if (!open)
+                return;
+            openTools.delete(runId);
+            const endMs = now() - baselineMs;
+            events.push({
+                kind: "tool",
+                name: open.name,
+                startMs: open.startMs,
+                endMs,
+                durationMs: Math.max(0, endMs - open.startMs),
+                error: err?.message ?? String(err),
+            });
+        }
+    }
+    const handler = new AgestTracer();
+    return {
+        callbacks: [handler],
+        drain: () => {
+            const ordered = [...events].sort((a, b) => a.startMs - b.startMs);
+            events.length = 0;
+            return { events: ordered, modelName: lastModelName };
+        },
+    };
+}
+function now() {
+    return performance.now();
+}
+function extractModelName(llm, extraParams) {
+    const invocation = (extraParams?.invocation_params ?? {});
+    if (typeof invocation.model === "string")
+        return invocation.model;
+    if (typeof invocation.model_name === "string")
+        return invocation.model_name;
+    const kwargs = llm?.kwargs;
+    if (kwargs) {
+        if (typeof kwargs.model === "string")
+            return kwargs.model;
+        if (typeof kwargs.model_name === "string")
+            return kwargs.model_name;
+        if (typeof kwargs.modelName === "string")
+            return kwargs.modelName;
+    }
+    const id = llm?.id;
+    if (Array.isArray(id) && id.length > 0 && typeof id[id.length - 1] === "string") {
+        return id[id.length - 1];
+    }
+    return undefined;
+}
+function extractModelNameFromOutput(output) {
+    const gen = output?.generations?.[0]?.[0];
+    return (gen?.message?.response_metadata?.model_name ??
+        gen?.message?.response_metadata?.model ??
+        output?.llmOutput?.modelName ??
+        output?.llmOutput?.model);
+}
+function extractTokensFromLLMOutput(output) {
+    const usage = output?.llmOutput?.tokenUsage ??
+        output?.llmOutput?.usage ??
+        output?.generations?.[0]?.[0]?.message?.usage_metadata ??
+        output?.generations?.[0]?.[0]?.message?.response_metadata?.usage;
+    if (!usage)
+        return undefined;
+    const input = usage.input_tokens ?? usage.prompt_tokens ?? usage.promptTokens ?? 0;
+    const out = usage.output_tokens ?? usage.completion_tokens ?? usage.completionTokens ?? 0;
+    if (!input && !out)
+        return undefined;
+    return { input, output: out };
+}
+function extractProviderCost(output) {
+    const candidates = [
+        output?.llmOutput?.usage?.cost,
+        output?.llmOutput?.cost,
+        output?.generations?.[0]?.[0]?.message?.usage_metadata?.total_cost,
+        output?.generations?.[0]?.[0]?.message?.response_metadata?.usage?.cost,
+        output?.generations?.[0]?.[0]?.message?.response_metadata?.cost,
+    ];
+    for (const c of candidates) {
+        if (typeof c === "number" && Number.isFinite(c))
+            return c;
+    }
+    return undefined;
+}
+function extractToolName(tool) {
+    if (!tool)
+        return undefined;
+    if (typeof tool.name === "string")
+        return tool.name;
+    if (Array.isArray(tool.id) && tool.id.length > 0) {
+        return String(tool.id[tool.id.length - 1]);
+    }
+    return undefined;
+}
+function stripCostIfEmpty(cost) {
+    if (cost.source === "unavailable" && cost.totalUsd == null)
+        return undefined;
+    return cost;
+}

package/dist/cli.js

@@ -2,11 +2,35 @@
 import { spawn } from "child_process";
 import { main as stats } from "./stats.js";
 import { main as preview } from "./preview.js";
+import { DEFAULT_PATTERN, discoverTestFiles } from "./discover.js";
 const command = process.argv[2];
+function parseRunArgs(args) {
+    const targets = [];
+    let pattern;
+    for (let i = 0; i < args.length; i++) {
+        const a = args[i];
+        if (a === "--pattern" || a === "-p") {
+            pattern = args[++i];
+            if (pattern === undefined) {
+                console.error("  Error: --pattern requires a value");
+                process.exit(1);
+            }
+        }
+        else if (a.startsWith("--pattern=")) {
+            pattern = a.slice("--pattern=".length);
+        }
+        else {
+            targets.push(a);
+        }
+    }
+    return { pattern, targets };
+}
 async function run() {
-    const files = process.argv.slice(3);
+    const { pattern, targets } = parseRunArgs(process.argv.slice(3));
+    const files = await discoverTestFiles(targets, { pattern });
     if (files.length === 0) {
-        console.error("  Usage: agest run <file...>");
+        const effective = pattern ?? DEFAULT_PATTERN;
+        console.error(`  No test files found (pattern: ${effective})`);
         process.exit(1);
     }
     for (const file of files) {
@@ -29,7 +53,10 @@ if (!command || !commands[command]) {
   Usage: agest <command>
 
   Commands:
-    run        Run test file(s)    agest run tests/*.test.ts
+    run        Run test file(s), directories, or glob patterns
+               agest run tests/                       # walks for ${DEFAULT_PATTERN}
+               agest run src/agest --pattern "**/*.test.ts"
+               agest run "tests/**/*.agest.ts" path/to/file.agest.ts
     stats      Show aggregated test statistics
     preview    Generate an HTML report preview
 `);

package/dist/config.d.ts

@@ -15,6 +15,15 @@ export interface AgestConfig {
     turns?: number;
     runs?: number;
     judge?: JudgeConfig;
+    /**
+     * Per-model pricing override (USD per 1M tokens). Merged on top of the
+     * built-in `src/pricing/models.json` table. Provide entries for any model
+     * you use that isn't already in the table, or to override a default.
+     */
+    pricing?: Record<string, {
+        input: number;
+        output: number;
+    }>;
 }
 export declare function defineConfig(config: AgestConfig): AgestConfig;
 export declare function loadConfig(): Promise<AgestConfig>;

package/dist/context.js

@@ -3,6 +3,7 @@ import { executeScene } from "./runner";
 import { formatReport, writeReport, writeDiffEntry } from "./reporter";
 import { logger, c } from "./logger";
 import { loadConfig } from "./config";
+import { setPricingOverrides } from "./pricing";
 import { PromisePool } from "@supercharge/promise-pool";
 export class SceneBuilder {
     _prompt;
@@ -77,6 +78,7 @@ export class AgentContext {
     }
     async execute() {
         const config = await loadConfig();
+        setPricingOverrides(config.pricing);
         const parallelism = Math.max(1, config.parallelism ?? 1);
         const definitions = this._scenes.map((s) => s.toDefinition());
         const orderedResults = new Array(definitions.length);
@@ -170,14 +172,25 @@ export class AgentContext {
         const successRate = results.length > 0
             ? Number((results.filter((r) => r.passed).length / results.length).toFixed(2))
             : 0;
-        const tokensAvailable = results.some((r) => r.response.metadata?.tokens != null);
+        const sceneTokens = results
+            .map((r) => r.tokens ?? r.response.metadata?.tokens)
+            .filter((t) => t != null);
         let averageInputTokensPerCase;
         let averageOutputTokensPerCase;
-        if (tokensAvailable) {
-            const withTokens = results.filter((r) => r.response.metadata?.tokens != null);
-            averageInputTokensPerCase = Math.round(withTokens.reduce((sum, r) => sum + (r.response.metadata.tokens.input ?? 0), 0) / withTokens.length);
-            averageOutputTokensPerCase = Math.round(withTokens.reduce((sum, r) => sum + (r.response.metadata.tokens.output ?? 0), 0) / withTokens.length);
+        let totalInputTokens;
+        let totalOutputTokens;
+        if (sceneTokens.length > 0) {
+            totalInputTokens = sceneTokens.reduce((s, t) => s + (t.input ?? 0), 0);
+            totalOutputTokens = sceneTokens.reduce((s, t) => s + (t.output ?? 0), 0);
+            averageInputTokensPerCase = Math.round(totalInputTokens / sceneTokens.length);
+            averageOutputTokensPerCase = Math.round(totalOutputTokens / sceneTokens.length);
         }
+        const sceneCosts = results
+            .map((r) => r.costUsd)
+            .filter((c) => typeof c === "number");
+        const totalCostUsd = sceneCosts.length > 0
+            ? sceneCosts.reduce((s, c) => s + c, 0)
+            : undefined;
         const firstMeta = results.find((r) => r.response.metadata)?.response
             .metadata;
         const dimensions = {};
@@ -208,6 +221,9 @@ export class AgentContext {
             totalCases: results.length,
             averageInputTokensPerCase,
             averageOutputTokensPerCase,
+            totalInputTokens,
+            totalOutputTokens,
+            totalCostUsd,
             results,
         };
         if (report.systemPromptHash && firstMeta?.systemPrompt) {

package/dist/discover.d.ts

@@ -0,0 +1,16 @@
+export declare const DEFAULT_PATTERN = "**/*.agest.ts";
+export interface DiscoverOptions {
+    pattern?: string;
+    cwd?: string;
+}
+/**
+ * Resolve a mix of file paths, directories, and glob patterns into a
+ * deduplicated, sorted list of absolute file paths.
+ *
+ * Rules per target:
+ *   - directory: search recursively for `pattern` (default `**\/*.agest.ts`)
+ *   - glob (contains *, ?, [], {}): expand it
+ *   - file: use as-is
+ *   - anything else: try as glob (zero matches is fine)
+ */
+export declare function discoverTestFiles(targets: string[], options?: DiscoverOptions): Promise<string[]>;

package/dist/discover.js

@@ -0,0 +1,62 @@
+import { promises as fs } from "node:fs";
+import { isAbsolute, resolve } from "node:path";
+export const DEFAULT_PATTERN = "**/*.agest.ts";
+const GLOB_CHARS = /[*?[\]{}]/;
+function hasGlobChars(value) {
+    return GLOB_CHARS.test(value);
+}
+async function statSafe(path) {
+    try {
+        const stat = await fs.stat(path);
+        return { isFile: stat.isFile(), isDir: stat.isDirectory() };
+    }
+    catch {
+        return { isFile: false, isDir: false };
+    }
+}
+async function expandGlob(pattern, cwd) {
+    const out = [];
+    // fs.promises.glob is available in Node >= 22 (the package's required engine).
+    for await (const match of fs.glob(pattern, { cwd })) {
+        out.push(isAbsolute(match) ? match : resolve(cwd, match));
+    }
+    return out;
+}
+/**
+ * Resolve a mix of file paths, directories, and glob patterns into a
+ * deduplicated, sorted list of absolute file paths.
+ *
+ * Rules per target:
+ *   - directory: search recursively for `pattern` (default `**\/*.agest.ts`)
+ *   - glob (contains *, ?, [], {}): expand it
+ *   - file: use as-is
+ *   - anything else: try as glob (zero matches is fine)
+ */
+export async function discoverTestFiles(targets, options = {}) {
+    const cwd = options.cwd ?? process.cwd();
+    const pattern = options.pattern ?? DEFAULT_PATTERN;
+    const work = targets.length === 0 ? ["."] : targets;
+    const found = new Set();
+    for (const target of work) {
+        if (hasGlobChars(target)) {
+            for (const f of await expandGlob(target, cwd))
+                found.add(f);
+            continue;
+        }
+        const stat = await statSafe(isAbsolute(target) ? target : resolve(cwd, target));
+        if (stat.isDir) {
+            const trimmed = target.replace(/\/+$/, "");
+            const dirPattern = `${trimmed}/${pattern}`;
+            for (const f of await expandGlob(dirPattern, cwd))
+                found.add(f);
+            continue;
+        }
+        if (stat.isFile) {
+            found.add(isAbsolute(target) ? target : resolve(cwd, target));
+            continue;
+        }
+        for (const f of await expandGlob(target, cwd))
+            found.add(f);
+    }
+    return [...found].sort();
+}

package/dist/index.d.ts

@@ -7,7 +7,7 @@ export type { AgestConfig, JudgeConfig, JudgeExecutor } from "./config";
 export type { LogLevel } from "./logger";
 export type { AgentExpectation, AgentMatchers } from "./assertions";
 export type { JudgeCriteria } from "./judge";
-export type { AgentExecutor, AgentResponse, AgentReport, SceneResult, RunResult, JudgeVerdict, JudgeResult, HookFn, } from "./types";
+export type { AgentExecutor, ExecutorOptions, AgentResponse, AgentReport, SceneResult, RunResult, JudgeVerdict, JudgeResult, HookFn, } from "./types";
 export interface AgentOptions {
     name?: string;
 }

package/dist/pricing/models.json

@@ -0,0 +1,21 @@
+{
+  "claude-opus-4-7": { "input": 15, "output": 75 },
+  "claude-opus-4-6": { "input": 15, "output": 75 },
+  "claude-opus-4-5": { "input": 15, "output": 75 },
+  "claude-sonnet-4-6": { "input": 3, "output": 15 },
+  "claude-sonnet-4-5": { "input": 3, "output": 15 },
+  "claude-haiku-4-5": { "input": 1, "output": 5 },
+  "claude-3-5-sonnet-20241022": { "input": 3, "output": 15 },
+  "claude-3-5-haiku-20241022": { "input": 0.8, "output": 4 },
+  "claude-3-opus-20240229": { "input": 15, "output": 75 },
+  "gpt-4o": { "input": 2.5, "output": 10 },
+  "gpt-4o-mini": { "input": 0.15, "output": 0.6 },
+  "gpt-4.1": { "input": 2, "output": 8 },
+  "gpt-4.1-mini": { "input": 0.4, "output": 1.6 },
+  "gpt-4.1-nano": { "input": 0.1, "output": 0.4 },
+  "gpt-5": { "input": 1.25, "output": 10 },
+  "gpt-5-mini": { "input": 0.25, "output": 2 },
+  "o1": { "input": 15, "output": 60 },
+  "o1-mini": { "input": 1.1, "output": 4.4 },
+  "o3-mini": { "input": 1.1, "output": 4.4 }
+}

package/dist/pricing.d.ts

@@ -0,0 +1,23 @@
+export interface ModelPrice {
+    /** USD per 1M input tokens */
+    input: number;
+    /** USD per 1M output tokens */
+    output: number;
+}
+export type CostSource = "provider" | "table" | "unavailable";
+export interface CostBreakdown {
+    inputUsd?: number;
+    outputUsd?: number;
+    totalUsd?: number;
+    source: CostSource;
+}
+export declare function setPricingOverrides(table?: Record<string, ModelPrice>): void;
+export declare function lookupPrice(model: string | undefined): ModelPrice | undefined;
+export interface ComputeCostInput {
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    /** USD cost the provider already reported (takes precedence) */
+    providerCost?: number;
+}
+export declare function computeCost(input: ComputeCostInput): CostBreakdown;

package/dist/pricing.js

@@ -0,0 +1,42 @@
+import { readFileSync } from "fs";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+const here = dirname(fileURLToPath(import.meta.url));
+const builtIn = JSON.parse(readFileSync(join(here, "pricing", "models.json"), "utf-8"));
+let overrides = {};
+export function setPricingOverrides(table) {
+    overrides = table ?? {};
+}
+export function lookupPrice(model) {
+    if (!model)
+        return undefined;
+    if (overrides[model])
+        return overrides[model];
+    if (builtIn[model])
+        return builtIn[model];
+    // Loose suffix/prefix match — pick the longest matching key
+    const lowered = model.toLowerCase();
+    const keys = Object.keys({ ...builtIn, ...overrides })
+        .filter((k) => lowered.includes(k.toLowerCase()) || k.toLowerCase().includes(lowered))
+        .sort((a, b) => b.length - a.length);
+    if (keys.length > 0) {
+        return overrides[keys[0]] ?? builtIn[keys[0]];
+    }
+    return undefined;
+}
+export function computeCost(input) {
+    if (input.providerCost != null && Number.isFinite(input.providerCost)) {
+        return { totalUsd: input.providerCost, source: "provider" };
+    }
+    const price = lookupPrice(input.model);
+    if (!price)
+        return { source: "unavailable" };
+    const inputUsd = ((input.inputTokens ?? 0) / 1_000_000) * price.input;
+    const outputUsd = ((input.outputTokens ?? 0) / 1_000_000) * price.output;
+    return {
+        inputUsd,
+        outputUsd,
+        totalUsd: inputUsd + outputUsd,
+        source: "table",
+    };
+}

package/dist/reporter.js

@@ -78,8 +78,75 @@ export function formatReport(report) {
     if (report.averageOutputTokensPerCase != null) {
         lines.push(`    average_output_tokens_per_case: ${report.averageOutputTokensPerCase}`);
     }
+    if (report.totalInputTokens != null) {
+        lines.push(`    total_input_tokens: ${report.totalInputTokens}`);
+    }
+    if (report.totalOutputTokens != null) {
+        lines.push(`    total_output_tokens: ${report.totalOutputTokens}`);
+    }
+    if (report.totalCostUsd != null) {
+        lines.push(`    total_cost_usd: ${formatUsd(report.totalCostUsd)}`);
+    }
+    const observedScenes = report.results.filter((r) => r.tokens || r.costUsd != null || (r.events && r.events.length));
+    if (observedScenes.length > 0) {
+        lines.push(`    scenes:`);
+        for (const r of observedScenes) {
+            lines.push(...renderSceneObservability(r));
+        }
+    }
     return lines.join("\n");
 }
+function renderSceneObservability(r) {
+    const out = [];
+    const promptLabel = r.prompt.length > 80 ? r.prompt.slice(0, 77) + "..." : r.prompt;
+    out.push(`        - prompt: "${escapeYaml(promptLabel)}"`);
+    out.push(`          duration_ms: ${Math.round(r.duration)}`);
+    if (r.tokens) {
+        out.push(`          tokens: { input: ${r.tokens.input}, output: ${r.tokens.output} }`);
+    }
+    if (r.costUsd != null) {
+        const source = r.costSource ?? "table";
+        out.push(`          cost_usd: ${formatUsd(r.costUsd)}`);
+        out.push(`          cost_source: ${source}`);
+    }
+    if (r.events && r.events.length) {
+        out.push(`          timeline:`);
+        for (const e of r.events) {
+            out.push(...renderTimelineEvent(e));
+        }
+    }
+    return out;
+}
+function renderTimelineEvent(e) {
+    const out = [];
+    out.push(`              - kind: ${e.kind}`);
+    out.push(`                name: "${escapeYaml(e.name)}"`);
+    out.push(`                start_ms: ${Math.round(e.startMs)}`);
+    out.push(`                duration_ms: ${Math.round(e.durationMs)}`);
+    if (e.tokens) {
+        out.push(`                tokens: { input: ${e.tokens.input}, output: ${e.tokens.output} }`);
+    }
+    if (e.cost?.totalUsd != null) {
+        out.push(`                cost_usd: ${formatUsd(e.cost.totalUsd)}`);
+        out.push(`                cost_source: ${e.cost.source}`);
+    }
+    if (e.runIndex != null) {
+        out.push(`                run_index: ${e.runIndex}`);
+    }
+    if (e.error) {
+        out.push(`                error: "${escapeYaml(e.error)}"`);
+    }
+    return out;
+}
+function formatUsd(n) {
+    if (n === 0)
+        return "0";
+    // Up to 6 decimal places, but trim trailing zeros for compactness
+    return Number(n.toFixed(6)).toString();
+}
+function escapeYaml(s) {
+    return s.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
+}
 export async function writeReport(content, timestamp, name, dimensions) {
     const reportsDir = join(process.cwd(), ".reports");
     await mkdir(reportsDir, { recursive: true });

package/dist/runner.js

@@ -37,13 +37,20 @@ async function executeSingleRun(executor, scene, timeoutMs, turns, judgeConfig)
         const start = performance.now();
         const input = scene.prompt;
         for (let t = 0; t < turns; t++) {
-            let timer;
-            response = await Promise.race([
-                executor(input).finally(() => clearTimeout(timer)),
-                new Promise((_, reject) => {
-                    timer = setTimeout(() => reject(new Error(`Scene timed out after ${timeoutMs}ms`)), timeoutMs);
-                }),
-            ]);
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), timeoutMs);
+            try {
+                response = await executor(input, { signal: controller.signal });
+            }
+            catch (err) {
+                if (err.name === "AbortError" || controller.signal.aborted) {
+                    throw new Error(`Scene timed out after ${timeoutMs}ms`);
+                }
+                throw err;
+            }
+            finally {
+                clearTimeout(timer);
+            }
             if (response.executionError)
                 break;
         }
@@ -109,6 +116,9 @@ export async function executeScene(executor, scene, globalTimeout, judgeConfig,
     // Single run — original fast path
     if (numRuns <= 1) {
         const run = await executeSingleRun(executor, scene, timeoutMs, turns, judgeConfig);
+        const tokens = run.response.metadata?.tokens;
+        const cost = run.response.metadata?.cost;
+        const events = run.response.metadata?.events;
         return {
             prompt: scene.prompt,
             response: run.response,
@@ -117,6 +127,10 @@ export async function executeScene(executor, scene, globalTimeout, judgeConfig,
             error: run.error,
             judgement: run.judgement,
             suite: scene.suite,
+            tokens: tokens ? { input: tokens.input, output: tokens.output } : undefined,
+            costUsd: cost?.totalUsd,
+            costSource: cost?.source,
+            events: events && events.length ? events : undefined,
         };
     }
     // Multiple runs — execute N times and aggregate
@@ -136,6 +150,37 @@ export async function executeScene(executor, scene, globalTimeout, judgeConfig,
     const error = overallPassed
         ? undefined
         : failedRuns[0]?.error ?? "Majority of runs failed";
+    // Aggregate tokens, cost, events across runs
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let hasTokens = false;
+    let costTotal = 0;
+    let hasCost = false;
+    let costSource;
+    const allEvents = [];
+    runs.forEach((r, runIndex) => {
+        const meta = r.response.metadata;
+        if (meta?.tokens) {
+            hasTokens = true;
+            inputTokens += meta.tokens.input;
+            outputTokens += meta.tokens.output;
+        }
+        if (meta?.cost?.totalUsd != null) {
+            hasCost = true;
+            costTotal += meta.cost.totalUsd;
+            // Promote weakest source: provider > table > unavailable
+            if (costSource !== "table")
+                costSource = meta.cost.source;
+            if (meta.cost.source === "table" && costSource !== "table") {
+                costSource = "table";
+            }
+        }
+        if (meta?.events?.length) {
+            for (const e of meta.events) {
+                allEvents.push({ ...e, runIndex });
+            }
+        }
+    });
     return {
         prompt: scene.prompt,
         response: lastRun.response,
@@ -147,5 +192,9 @@ export async function executeScene(executor, scene, globalTimeout, judgeConfig,
         runs,
         passRate,
         statisticalSignificance,
+        tokens: hasTokens ? { input: inputTokens, output: outputTokens } : undefined,
+        costUsd: hasCost ? costTotal : undefined,
+        costSource,
+        events: allEvents.length ? allEvents : undefined,
     };
 }

package/dist/types.d.ts

@@ -1,4 +1,31 @@
-export type AgentExecutor = (input: string) => Promise<AgentResponse>;
+export interface ExecutorOptions {
+    signal?: AbortSignal;
+}
+export type AgentExecutor = (input: string, options?: ExecutorOptions) => Promise<AgentResponse>;
+export type CostSource = "provider" | "table" | "unavailable";
+export interface CostBreakdown {
+    inputUsd?: number;
+    outputUsd?: number;
+    totalUsd?: number;
+    source: CostSource;
+}
+export type TimelineEventKind = "model" | "tool";
+export interface TimelineEvent {
+    kind: TimelineEventKind;
+    name: string;
+    /** ms relative to the scene start */
+    startMs: number;
+    endMs: number;
+    durationMs: number;
+    tokens?: {
+        input: number;
+        output: number;
+    };
+    cost?: CostBreakdown;
+    /** Index of the run this event belongs to (only set when aggregating across multi-run scenes) */
+    runIndex?: number;
+    error?: string;
+}
 export interface AgentResponse {
     text: string;
     refusal?: boolean;
@@ -11,6 +38,8 @@ export interface AgentResponse {
         };
         tools?: string[];
         systemPrompt?: string;
+        events?: TimelineEvent[];
+        cost?: CostBreakdown;
         [key: string]: unknown;
     };
 }
@@ -50,6 +79,16 @@ export interface SceneResult {
     runs?: RunResult[];
     passRate?: number;
     statisticalSignificance?: number;
+    /** Aggregate tokens across all runs of this scene */
+    tokens?: {
+        input: number;
+        output: number;
+    };
+    /** Aggregate USD cost across all runs of this scene */
+    costUsd?: number;
+    costSource?: CostSource;
+    /** Ordered timeline events from every run of the scene */
+    events?: TimelineEvent[];
 }
 export interface AgentReport {
     name?: string;
@@ -66,5 +105,8 @@ export interface AgentReport {
     totalCases: number;
     averageInputTokensPerCase?: number;
     averageOutputTokensPerCase?: number;
+    totalInputTokens?: number;
+    totalOutputTokens?: number;
+    totalCostUsd?: number;
     results: SceneResult[];
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sebastiantuyu/agest",
-  "version": "0.3.2",
+  "version": "0.3.3-next.2",
   "description": "A testing library for agents",
   "repository": {
     "type": "git",
@@ -26,7 +26,7 @@
     }
   },
   "scripts": {
-    "build": "tsc -p tsconfig.build.json",
+    "build": "tsc -p tsconfig.build.json && mkdir -p dist/pricing && cp src/pricing/models.json dist/pricing/models.json",
     "test": "vitest run",
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
@@ -37,25 +37,26 @@
     "site:preview": "npx serve site -p 3000",
     "release:patch": "npm version patch && git push && git push --tags",
     "release:minor": "npm version minor && git push && git push --tags",
-    "release:major": "npm version major && git push && git push --tags"
+    "release:major": "npm version major && git push && git push --tags",
+    "release:next": "npm version prerelease --preid=next && git push && git push --tags"
   },
   "engines": {
     "node": ">=22.0.0"
   },
   "devDependencies": {
-    "@langchain/core": "^1.1.39",
-    "@langchain/langgraph": "^1.2.8",
-    "@langchain/openai": "^1.4.4",
-    "@types/node": "^22.0.0",
-    "@vitest/coverage-v8": "^3",
-    "dotenv": "^17.4.1",
-    "langchain": "^1.3.1",
-    "tsx": "^4.21.0",
-    "typescript": "^5.4.0",
-    "vitest": "^3",
-    "zod": "^4.3.6"
+    "@langchain/core": "1.1.39",
+    "@langchain/langgraph": "1.2.8",
+    "@langchain/openai": "1.4.4",
+    "@types/node": "22.19.17",
+    "@vitest/coverage-v8": "3.2.4",
+    "dotenv": "17.4.1",
+    "langchain": "1.3.1",
+    "tsx": "4.21.0",
+    "typescript": "5.9.3",
+    "vitest": "3.2.4",
+    "zod": "4.3.6"
   },
   "dependencies": {
-    "@supercharge/promise-pool": "^3.3.0"
+    "@supercharge/promise-pool": "3.3.0"
   }
 }