@sebastiantuyu/agest 0.3.3-next.5 → 0.3.3-next.7

package/dist/adapters/tracing.js

@@ -48,6 +48,7 @@ export async function createTracingHandle(baselineMs) {
             const endMs = now() - baselineMs;
             const tokens = extractTokensFromLLMOutput(output);
             const providerCost = extractProviderCost(output);
+            const cachedInputTokens = extractCachedTokens(output);
             const name = open.name ?? extractModelNameFromOutput(output) ?? "model";
             if (name && name !== "model")
                 lastModelName = name;
@@ -64,6 +65,7 @@ export async function createTracingHandle(baselineMs) {
                 endMs,
                 durationMs: Math.max(0, endMs - open.startMs),
                 tokens,
+                cachedInputTokens,
                 cost: stripCostIfEmpty(cost),
             });
         }
@@ -82,10 +84,10 @@ export async function createTracingHandle(baselineMs) {
                 error: err?.message ?? String(err),
             });
         }
-        handleToolStart(tool, _input, runId) {
+        handleToolStart(tool, _input, runId, _parentRunId, _tags, _metadata, runName) {
             openTools.set(runId, {
                 startMs: now() - baselineMs,
-                name: extractToolName(tool) ?? "tool",
+                name: extractToolName(tool, runName) ?? "tool",
             });
         }
         handleToolEnd(_output, runId) {
@@ -252,30 +254,82 @@ function extractTokensFromLLMOutput(output) {
         return undefined;
     return { input, output: out };
 }
+/** Collect the usage-bearing objects LangChain/OpenRouter may attach to an LLM result. */
+function usageObjects(output) {
+    const msg = output?.generations?.[0]?.[0]?.message;
+    return [
+        output?.llmOutput?.usage,
+        output?.llmOutput?.tokenUsage,
+        output?.llmOutput?.estimatedTokenUsage,
+        output?.llmOutput,
+        msg?.usage_metadata,
+        msg?.response_metadata?.usage,
+        msg?.response_metadata?.tokenUsage,
+        msg?.response_metadata?.estimatedTokenUsage,
+        msg?.response_metadata,
+        msg?.additional_kwargs?.usage,
+    ].filter((u) => u && typeof u === "object");
+}
+/**
+ * OpenRouter (with `usage: { include: true }`) reports real USD cost. LangChain
+ * surfaces it inconsistently across versions, so scan the known usage objects
+ * for a numeric `cost` / `total_cost`.
+ */
 function extractProviderCost(output) {
-    const candidates = [
-        output?.llmOutput?.usage?.cost,
-        output?.llmOutput?.cost,
-        output?.generations?.[0]?.[0]?.message?.usage_metadata?.total_cost,
-        output?.generations?.[0]?.[0]?.message?.response_metadata?.usage?.cost,
-        output?.generations?.[0]?.[0]?.message?.response_metadata?.cost,
-    ];
-    for (const c of candidates) {
+    for (const u of usageObjects(output)) {
+        const c = (typeof u.cost === "number" ? u.cost : undefined) ??
+            (typeof u.total_cost === "number" ? u.total_cost : undefined) ??
+            (typeof u.cost_usd === "number" ? u.cost_usd : undefined) ??
+            (typeof u.cost_details?.upstream_inference_cost === "number"
+                ? u.cost_details.upstream_inference_cost
+                : undefined);
         if (typeof c === "number" && Number.isFinite(c))
             return c;
     }
     return undefined;
 }
-function extractToolName(tool) {
-    if (!tool)
-        return undefined;
-    if (typeof tool.name === "string")
-        return tool.name;
-    if (Array.isArray(tool.id) && tool.id.length > 0) {
-        return String(tool.id[tool.id.length - 1]);
+/**
+ * Cached (prompt-cache hit) input tokens, when the provider reports them.
+ * Charged at a fraction of the normal input rate, so surfacing them lets the
+ * report explain why provider cost is below the flat-table estimate.
+ */
+function extractCachedTokens(output) {
+    for (const u of usageObjects(output)) {
+        const cached = u.input_token_details?.cache_read ??
+            u.prompt_tokens_details?.cached_tokens ??
+            u.cache_read_input_tokens ??
+            u.cached_tokens;
+        if (typeof cached === "number" && cached > 0)
+            return cached;
     }
     return undefined;
 }
+const TOOL_CLASS_NAMES = new Set([
+    "DynamicStructuredTool",
+    "DynamicTool",
+    "StructuredTool",
+    "Tool",
+]);
+function extractToolName(tool, runName) {
+    // `runName` is the actual tool name LangChain assigns the run (e.g.
+    // "search_recipes"); prefer it over the serialized class name.
+    if (runName && !TOOL_CLASS_NAMES.has(runName))
+        return runName;
+    if (tool) {
+        if (typeof tool.name === "string" && !TOOL_CLASS_NAMES.has(tool.name))
+            return tool.name;
+        if (typeof tool.kwargs?.name === "string")
+            return tool.kwargs.name;
+        if (Array.isArray(tool.id) && tool.id.length > 0) {
+            const last = String(tool.id[tool.id.length - 1]);
+            if (!TOOL_CLASS_NAMES.has(last))
+                return last;
+        }
+        if (typeof tool.name === "string")
+            return tool.name;
+    }
+    return runName;
+}
 function stripCostIfEmpty(cost) {
     if (cost.source === "unavailable" && cost.totalUsd == null)
         return undefined;

package/dist/context.js

@@ -4,6 +4,7 @@ import { formatReport, writeReport, writeDiffEntry } from "./reporter";
 import { logger, c } from "./logger";
 import { loadConfig } from "./config";
 import { setPricingOverrides } from "./pricing";
+import { renderTerminalWaterfall } from "./waterfall";
 import { PromisePool } from "@supercharge/promise-pool";
 export class SceneBuilder {
     _prompt;
@@ -129,6 +130,18 @@ export class AgentContext {
                 const sigColor = sig >= 0.95 ? c.green : sig >= 0.80 ? c.yellow : c.red;
                 logger.info(`${indent}       ${c.dim("significance:")} ${sigColor(`${(sig * 100).toFixed(1)}%`)} ${c.dim(`(pass rate: ${((result.passRate ?? 0) * 100).toFixed(1)}%)`)}`);
             }
+            if (result.events && result.events.length > 0) {
+                const costLabel = result.costUsd != null
+                    ? ` ${c.dim("·")} ${c.green(`$${Number(result.costUsd.toFixed(4))}`)}`
+                    : "";
+                const tokLabel = result.tokens
+                    ? ` ${c.dim(`(${result.tokens.input}→${result.tokens.output} tok)`)}`
+                    : "";
+                logger.info(`${indent}       ${c.dim("waterfall:")}${tokLabel}${costLabel}`);
+                for (const line of renderTerminalWaterfall(result.events, { indent: `${indent}       ` })) {
+                    logger.info(line);
+                }
+            }
             logger.debug(`${indent}       response: ${result.response.text?.slice(0, 120)}`);
         };
         if (hasSuites) {

package/dist/preview.js

@@ -189,6 +189,91 @@ function renderFailedCases(cases) {
           </ul>
         </details>`;
 }
+const WF_MODEL = "#38bdf8";
+const WF_TOOL = "#facc15";
+const WF_ERROR = "#f87171";
+function fmtUsdHtml(n) {
+    if (n === 0)
+        return "$0";
+    return "$" + Number(n.toFixed(4)).toString();
+}
+/**
+ * Chrome-DevTools-style waterfall for a report's per-scene timelines. Bars are
+ * absolutely positioned within a track by start_ms / duration_ms. Returns "" if
+ * the report carries no timeline data (older reports / non-traced executors).
+ */
+function renderWaterfallHtml(report) {
+    const scenes = (report.scenes ?? []).filter((s) => s.timeline && s.timeline.length > 0);
+    if (scenes.length === 0)
+        return "";
+    const sceneBlocks = scenes
+        .map((scene) => {
+        const events = scene.timeline;
+        const t0 = Math.min(...events.map((e) => e.startMs));
+        const tEnd = Math.max(...events.map((e) => e.startMs + e.durationMs));
+        const span = Math.max(1, tEnd - t0);
+        const rows = events
+            .map((e) => {
+            const left = ((e.startMs - t0) / span) * 100;
+            const width = Math.max(0.6, (e.durationMs / span) * 100);
+            const bg = e.error ? WF_ERROR : e.kind === "model" ? WF_MODEL : WF_TOOL;
+            const icon = e.kind === "model" ? "◆" : "▸";
+            const tip = [
+                `${e.kind}: ${e.name}`,
+                `start ${Math.round(e.startMs)}ms · ${Math.round(e.durationMs)}ms`,
+                e.tokens ? `${e.tokens.input}→${e.tokens.output} tok` : "",
+                e.cachedInputTokens ? `${e.cachedInputTokens} cached` : "",
+                e.costUsd != null ? fmtUsdHtml(e.costUsd) : "",
+                e.error ? `error: ${e.error}` : "",
+            ]
+                .filter(Boolean)
+                .join(" · ");
+            const cost = e.costUsd != null ? fmtUsdHtml(e.costUsd) : "";
+            return `
+            <div class="flex items-center gap-2 text-[11px] leading-5">
+              <span class="w-44 shrink-0 truncate ${e.error ? "text-red-400" : "text-zinc-400"}" title="${escHtml(e.name)}">
+                <span style="color:${bg}">${icon}</span> ${escHtml(e.name)}
+              </span>
+              <div class="relative flex-1 h-3 bg-zinc-800/40 rounded">
+                <div class="absolute top-0 h-3 rounded" style="left:${left.toFixed(2)}%;width:${width.toFixed(2)}%;background:${bg}" title="${escHtml(tip)}"></div>
+              </div>
+              <span class="w-16 shrink-0 text-right text-zinc-500">${Math.round(e.durationMs)}ms</span>
+              <span class="w-16 shrink-0 text-right text-zinc-500">${cost}</span>
+            </div>`;
+        })
+            .join("\n");
+        const meta = [
+            scene.tokens ? `${scene.tokens.input}→${scene.tokens.output} tok` : "",
+            scene.costUsd != null ? fmtUsdHtml(scene.costUsd) : "",
+            scene.costSource ? scene.costSource : "",
+            scene.durationMs != null ? `${Math.round(scene.durationMs)}ms` : "",
+        ]
+            .filter(Boolean)
+            .join(" · ");
+        return `
+        <div>
+          <div class="flex items-center justify-between mb-1.5">
+            <span class="text-xs text-zinc-300 truncate" title="${escHtml(scene.prompt)}">${escHtml(scene.prompt)}</span>
+            <span class="text-[11px] text-zinc-500 shrink-0 ml-3">${escHtml(meta)}</span>
+          </div>
+          <div class="space-y-1">${rows}</div>
+        </div>`;
+    })
+        .join("\n");
+    return `
+    <details class="mt-2" open>
+      <summary class="text-xs text-sky-400 cursor-pointer hover:text-sky-300 select-none">
+        waterfall &middot; ${scenes.length} scene${scenes.length !== 1 ? "s" : ""}
+      </summary>
+      <div class="mt-3 mb-2 pl-3 border-l border-zinc-800 space-y-5">
+        <div class="flex gap-4 text-[10px] text-zinc-500">
+          <span><span style="color:${WF_MODEL}">◆</span> model</span>
+          <span><span style="color:${WF_TOOL}">▸</span> tool</span>
+        </div>
+        ${sceneBlocks}
+      </div>
+    </details>`;
+}
 function renderRunRow(entry, idx) {
     const { report, delta, diffLines } = entry;
     const pct = report.successRate * 100;
@@ -233,6 +318,7 @@ function renderRunRow(entry, idx) {
           </div>
           <div class="ml-10 mt-0.5 flex gap-3 flex-wrap">${dimTags}</div>
           ${diffHtml}
+          <div class="ml-10">${renderWaterfallHtml(report)}</div>
         </div>`;
 }
 // ---------------------------------------------------------------------------
@@ -990,6 +1076,12 @@ function renderSingleRun(report) {
           <p class="text-zinc-300">${Math.round(report.averageOutputTokensPerCase)}</p>
         </div>`
         : ""}
+        ${report.totalCostUsd != null
+        ? `<div>
+          <span class="text-zinc-500">Total Cost</span>
+          <p class="text-zinc-300">${fmtUsdHtml(report.totalCostUsd)}${report.totalInputTokens != null ? ` <span class="text-zinc-600">· ${report.totalInputTokens}→${report.totalOutputTokens} tok</span>` : ""}</p>
+        </div>`
+        : ""}
         ${report.tools && report.tools.length > 0
         ? `<div>
           <span class="text-zinc-500">Tools</span>
@@ -997,6 +1089,7 @@ function renderSingleRun(report) {
         </div>`
         : ""}
       </div>
+      ${renderWaterfallHtml(report)}
       ${failedSection}
     </div>`;
 }

package/dist/reporter.js

@@ -126,6 +126,9 @@ function renderTimelineEvent(e) {
     if (e.tokens) {
         out.push(`                tokens: { input: ${e.tokens.input}, output: ${e.tokens.output} }`);
     }
+    if (e.cachedInputTokens != null && e.cachedInputTokens > 0) {
+        out.push(`                cached_input_tokens: ${e.cachedInputTokens}`);
+    }
     if (e.cost?.totalUsd != null) {
         out.push(`                cost_usd: ${formatUsd(e.cost.totalUsd)}`);
         out.push(`                cost_source: ${e.cost.source}`);

package/dist/reports.d.ts

@@ -9,6 +9,32 @@ export interface ParsedSuiteResult {
         response?: string;
     }>;
 }
+export interface ParsedTimelineEvent {
+    kind: "model" | "tool";
+    name: string;
+    startMs: number;
+    durationMs: number;
+    tokens?: {
+        input: number;
+        output: number;
+    };
+    cachedInputTokens?: number;
+    costUsd?: number;
+    costSource?: string;
+    runIndex?: number;
+    error?: string;
+}
+export interface ParsedScene {
+    prompt: string;
+    durationMs?: number;
+    tokens?: {
+        input: number;
+        output: number;
+    };
+    costUsd?: number;
+    costSource?: string;
+    timeline?: ParsedTimelineEvent[];
+}
 export interface ParsedReport {
     name?: string;
     systemPromptHash?: string;
@@ -28,6 +54,10 @@ export interface ParsedReport {
     timestamp: string;
     averageInputTokensPerCase?: number;
     averageOutputTokensPerCase?: number;
+    totalInputTokens?: number;
+    totalOutputTokens?: number;
+    totalCostUsd?: number;
+    scenes?: ParsedScene[];
     suites?: ParsedSuiteResult[];
     source: string;
 }
@@ -44,6 +74,13 @@ export declare function parseFailedCases(content: string): Array<{
 }>;
 export declare function parseDimensions(content: string): Record<string, string> | undefined;
 export declare function parseSuites(content: string): ParsedSuiteResult[] | undefined;
+/**
+ * Parse the `scenes:` block (per-scene tokens/cost + timeline waterfall) from a
+ * report. The emitted format is fixed (see reporter.ts `renderSceneObservability`),
+ * so this hand-parses by indentation: scenes start at 8 spaces, scene fields at
+ * 10, timeline events at 14, event fields at 16.
+ */
+export declare function parseScenes(content: string): ParsedScene[] | undefined;
 export declare function parseReport(content: string, source: string): ParsedReport;
 export declare function findReports(dir: string, depth?: number): Promise<string[]>;
 export declare function loadDiffEntry(hash: string): Promise<DiffEntry | null>;

package/dist/reports.js

@@ -124,6 +124,124 @@ export function parseSuites(content) {
         suites.push(current);
     return suites.length > 0 ? suites : undefined;
 }
+function parseTokens(raw) {
+    const m = raw.match(/input:\s*(\d+),\s*output:\s*(\d+)/);
+    if (!m)
+        return undefined;
+    return { input: parseInt(m[1], 10), output: parseInt(m[2], 10) };
+}
+/**
+ * Parse the `scenes:` block (per-scene tokens/cost + timeline waterfall) from a
+ * report. The emitted format is fixed (see reporter.ts `renderSceneObservability`),
+ * so this hand-parses by indentation: scenes start at 8 spaces, scene fields at
+ * 10, timeline events at 14, event fields at 16.
+ */
+export function parseScenes(content) {
+    const lines = content.split("\n");
+    const startIdx = lines.findIndex((l) => l === "    scenes:");
+    if (startIdx === -1)
+        return undefined;
+    const scenes = [];
+    let scene;
+    let inTimeline = false;
+    let event;
+    const pushEvent = () => {
+        if (event && scene) {
+            (scene.timeline ??= []).push(event);
+            event = undefined;
+        }
+    };
+    const pushScene = () => {
+        pushEvent();
+        if (scene)
+            scenes.push(scene);
+        scene = undefined;
+        inTimeline = false;
+    };
+    for (let i = startIdx + 1; i < lines.length; i++) {
+        const line = lines[i];
+        if (line.trim() === "")
+            continue;
+        const indent = line.length - line.trimStart().length;
+        // A new top-level agent field (<= 4 spaces, not part of scenes) ends the block.
+        if (indent <= 4)
+            break;
+        const sceneStart = line.match(/^        - prompt: "(.*)"$/);
+        if (sceneStart) {
+            pushScene();
+            scene = { prompt: sceneStart[1].replace(/\\"/g, '"').replace(/\\n/g, "\n") };
+            continue;
+        }
+        if (!scene)
+            continue;
+        const eventStart = line.match(/^              - kind: (model|tool)$/);
+        if (eventStart) {
+            pushEvent();
+            event = { kind: eventStart[1], name: "", startMs: 0, durationMs: 0 };
+            inTimeline = true;
+            continue;
+        }
+        if (line.match(/^          timeline:$/)) {
+            inTimeline = true;
+            continue;
+        }
+        const trimmed = line.trim();
+        const target = inTimeline && event ? "event" : "scene";
+        const kv = trimmed.match(/^([a-z_]+):\s*(.*)$/);
+        if (!kv)
+            continue;
+        const [, key, value] = kv;
+        if (target === "event" && event) {
+            switch (key) {
+                case "name":
+                    event.name = value.replace(/^"|"$/g, "").replace(/\\"/g, '"');
+                    break;
+                case "start_ms":
+                    event.startMs = parseFloat(value);
+                    break;
+                case "duration_ms":
+                    event.durationMs = parseFloat(value);
+                    break;
+                case "tokens":
+                    event.tokens = parseTokens(value);
+                    break;
+                case "cached_input_tokens":
+                    event.cachedInputTokens = parseInt(value, 10);
+                    break;
+                case "cost_usd":
+                    event.costUsd = parseFloat(value);
+                    break;
+                case "cost_source":
+                    event.costSource = value;
+                    break;
+                case "run_index":
+                    event.runIndex = parseInt(value, 10);
+                    break;
+                case "error":
+                    event.error = value.replace(/^"|"$/g, "").replace(/\\"/g, '"');
+                    break;
+            }
+        }
+        else if (scene) {
+            switch (key) {
+                case "duration_ms":
+                    scene.durationMs = parseFloat(value);
+                    break;
+                case "tokens":
+                    scene.tokens = parseTokens(value);
+                    break;
+                case "cost_usd":
+                    scene.costUsd = parseFloat(value);
+                    break;
+                case "cost_source":
+                    scene.costSource = value;
+                    break;
+            }
+        }
+    }
+    pushScene();
+    return scenes.length > 0 ? scenes : undefined;
+}
 export function parseReport(content, source) {
     const num = (key, fallback = 0) => parseFloat(extractField(content, key) ?? String(fallback));
     const avgIn = extractField(content, "average_input_tokens_per_case");
@@ -158,9 +276,17 @@ export function parseReport(content, source) {
         timestamp: extractField(content, "timestamp") ?? "",
         averageInputTokensPerCase: avgIn != null ? parseFloat(avgIn) : undefined,
         averageOutputTokensPerCase: avgOut != null ? parseFloat(avgOut) : undefined,
+        totalInputTokens: optNum("total_input_tokens"),
+        totalOutputTokens: optNum("total_output_tokens"),
+        totalCostUsd: optNum("total_cost_usd"),
+        scenes: parseScenes(content),
         suites: parseSuites(content),
         source,
     };
+    function optNum(key) {
+        const v = extractField(content, key);
+        return v != null ? parseFloat(v) : undefined;
+    }
 }
 export async function findReports(dir, depth = 0) {
     if (depth > 6)

package/dist/types.d.ts

@@ -21,6 +21,8 @@ export interface TimelineEvent {
         input: number;
         output: number;
     };
+    /** Prompt-cache-hit input tokens (subset of tokens.input), when reported by the provider */
+    cachedInputTokens?: number;
     cost?: CostBreakdown;
     /** Index of the run this event belongs to (only set when aggregating across multi-run scenes) */
     runIndex?: number;

package/dist/waterfall.d.ts

@@ -0,0 +1,11 @@
+import type { TimelineEvent } from "./types";
+/**
+ * Render a Chrome-DevTools-style waterfall of timeline events as colored
+ * terminal lines. Bars are positioned by `startMs` and sized by `durationMs`
+ * relative to the full span of the scene. Returns one string per event row
+ * (already indented), or `[]` when there's nothing to draw.
+ */
+export declare function renderTerminalWaterfall(events: TimelineEvent[], opts?: {
+    width?: number;
+    indent?: string;
+}): string[];

package/dist/waterfall.js

@@ -0,0 +1,46 @@
+import { c } from "./logger";
+const BLOCK = "█";
+const THIN = "▏";
+function truncate(s, n) {
+    return s.length > n ? s.slice(0, n - 1) + "…" : s;
+}
+function fmtUsd(n) {
+    if (n === 0)
+        return "$0";
+    return "$" + Number(n.toFixed(4)).toString();
+}
+/**
+ * Render a Chrome-DevTools-style waterfall of timeline events as colored
+ * terminal lines. Bars are positioned by `startMs` and sized by `durationMs`
+ * relative to the full span of the scene. Returns one string per event row
+ * (already indented), or `[]` when there's nothing to draw.
+ */
+export function renderTerminalWaterfall(events, opts = {}) {
+    if (!events || events.length === 0)
+        return [];
+    const width = opts.width ?? 28;
+    const indent = opts.indent ?? "";
+    const t0 = Math.min(...events.map((e) => e.startMs));
+    const tEnd = Math.max(...events.map((e) => e.endMs));
+    const span = Math.max(1, tEnd - t0);
+    const nameWidth = 16;
+    return events.map((e) => {
+        const lead = Math.min(width - 1, Math.round(((e.startMs - t0) / span) * width));
+        const barLen = Math.max(1, Math.round((e.durationMs / span) * width));
+        const fill = e.durationMs === 0 ? THIN : BLOCK.repeat(Math.min(barLen, width - lead));
+        const cells = Array(width).fill(" ");
+        for (let i = 0; i < fill.length && lead + i < width; i++) {
+            cells[lead + i] = fill[i];
+        }
+        let bar = cells.join("");
+        const color = e.error ? c.red : e.kind === "model" ? c.cyan : c.yellow;
+        bar = color(bar);
+        const kindLabel = (e.kind === "model" ? "model" : "tool ").padEnd(5);
+        const nameLabel = truncate(e.name, nameWidth).padEnd(nameWidth);
+        const dur = `${Math.round(e.durationMs)}ms`.padStart(7);
+        const cost = e.cost?.totalUsd != null ? `  ${fmtUsd(e.cost.totalUsd)}` : "";
+        const cached = e.cachedInputTokens ? `  ${c.dim(`(${e.cachedInputTokens} cached)`)}` : "";
+        const err = e.error ? `  ${c.red("✗ " + truncate(e.error, 40))}` : "";
+        return `${indent}${c.dim(kindLabel)} ${nameLabel} ${bar} ${c.dim(dur)}${c.dim(cost)}${cached}${err}`;
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sebastiantuyu/agest",
-  "version": "0.3.3-next.5",
+  "version": "0.3.3-next.7",
   "description": "A testing library for agents",
   "repository": {
     "type": "git",