@sebastiantuyu/agest 0.3.3-next.5 → 0.3.3-next.6

package/dist/context.js

@@ -4,6 +4,7 @@ import { formatReport, writeReport, writeDiffEntry } from "./reporter";
 import { logger, c } from "./logger";
 import { loadConfig } from "./config";
 import { setPricingOverrides } from "./pricing";
+import { renderTerminalWaterfall } from "./waterfall";
 import { PromisePool } from "@supercharge/promise-pool";
 export class SceneBuilder {
     _prompt;
@@ -129,6 +130,18 @@ export class AgentContext {
                 const sigColor = sig >= 0.95 ? c.green : sig >= 0.80 ? c.yellow : c.red;
                 logger.info(`${indent}       ${c.dim("significance:")} ${sigColor(`${(sig * 100).toFixed(1)}%`)} ${c.dim(`(pass rate: ${((result.passRate ?? 0) * 100).toFixed(1)}%)`)}`);
             }
+            if (result.events && result.events.length > 0) {
+                const costLabel = result.costUsd != null
+                    ? ` ${c.dim("·")} ${c.green(`$${Number(result.costUsd.toFixed(4))}`)}`
+                    : "";
+                const tokLabel = result.tokens
+                    ? ` ${c.dim(`(${result.tokens.input}→${result.tokens.output} tok)`)}`
+                    : "";
+                logger.info(`${indent}       ${c.dim("waterfall:")}${tokLabel}${costLabel}`);
+                for (const line of renderTerminalWaterfall(result.events, { indent: `${indent}       ` })) {
+                    logger.info(line);
+                }
+            }
             logger.debug(`${indent}       response: ${result.response.text?.slice(0, 120)}`);
         };
         if (hasSuites) {

package/dist/preview.js

@@ -189,6 +189,90 @@ function renderFailedCases(cases) {
           </ul>
         </details>`;
 }
+const WF_MODEL = "#38bdf8";
+const WF_TOOL = "#facc15";
+const WF_ERROR = "#f87171";
+function fmtUsdHtml(n) {
+    if (n === 0)
+        return "$0";
+    return "$" + Number(n.toFixed(4)).toString();
+}
+/**
+ * Chrome-DevTools-style waterfall for a report's per-scene timelines. Bars are
+ * absolutely positioned within a track by start_ms / duration_ms. Returns "" if
+ * the report carries no timeline data (older reports / non-traced executors).
+ */
+function renderWaterfallHtml(report) {
+    const scenes = (report.scenes ?? []).filter((s) => s.timeline && s.timeline.length > 0);
+    if (scenes.length === 0)
+        return "";
+    const sceneBlocks = scenes
+        .map((scene) => {
+        const events = scene.timeline;
+        const t0 = Math.min(...events.map((e) => e.startMs));
+        const tEnd = Math.max(...events.map((e) => e.startMs + e.durationMs));
+        const span = Math.max(1, tEnd - t0);
+        const rows = events
+            .map((e) => {
+            const left = ((e.startMs - t0) / span) * 100;
+            const width = Math.max(0.6, (e.durationMs / span) * 100);
+            const bg = e.error ? WF_ERROR : e.kind === "model" ? WF_MODEL : WF_TOOL;
+            const icon = e.kind === "model" ? "◆" : "▸";
+            const tip = [
+                `${e.kind}: ${e.name}`,
+                `start ${Math.round(e.startMs)}ms · ${Math.round(e.durationMs)}ms`,
+                e.tokens ? `${e.tokens.input}→${e.tokens.output} tok` : "",
+                e.costUsd != null ? fmtUsdHtml(e.costUsd) : "",
+                e.error ? `error: ${e.error}` : "",
+            ]
+                .filter(Boolean)
+                .join(" · ");
+            const cost = e.costUsd != null ? fmtUsdHtml(e.costUsd) : "";
+            return `
+            <div class="flex items-center gap-2 text-[11px] leading-5">
+              <span class="w-44 shrink-0 truncate ${e.error ? "text-red-400" : "text-zinc-400"}" title="${escHtml(e.name)}">
+                <span style="color:${bg}">${icon}</span> ${escHtml(e.name)}
+              </span>
+              <div class="relative flex-1 h-3 bg-zinc-800/40 rounded">
+                <div class="absolute top-0 h-3 rounded" style="left:${left.toFixed(2)}%;width:${width.toFixed(2)}%;background:${bg}" title="${escHtml(tip)}"></div>
+              </div>
+              <span class="w-16 shrink-0 text-right text-zinc-500">${Math.round(e.durationMs)}ms</span>
+              <span class="w-16 shrink-0 text-right text-zinc-500">${cost}</span>
+            </div>`;
+        })
+            .join("\n");
+        const meta = [
+            scene.tokens ? `${scene.tokens.input}→${scene.tokens.output} tok` : "",
+            scene.costUsd != null ? fmtUsdHtml(scene.costUsd) : "",
+            scene.costSource ? scene.costSource : "",
+            scene.durationMs != null ? `${Math.round(scene.durationMs)}ms` : "",
+        ]
+            .filter(Boolean)
+            .join(" · ");
+        return `
+        <div>
+          <div class="flex items-center justify-between mb-1.5">
+            <span class="text-xs text-zinc-300 truncate" title="${escHtml(scene.prompt)}">${escHtml(scene.prompt)}</span>
+            <span class="text-[11px] text-zinc-500 shrink-0 ml-3">${escHtml(meta)}</span>
+          </div>
+          <div class="space-y-1">${rows}</div>
+        </div>`;
+    })
+        .join("\n");
+    return `
+    <details class="mt-2" open>
+      <summary class="text-xs text-sky-400 cursor-pointer hover:text-sky-300 select-none">
+        waterfall &middot; ${scenes.length} scene${scenes.length !== 1 ? "s" : ""}
+      </summary>
+      <div class="mt-3 mb-2 pl-3 border-l border-zinc-800 space-y-5">
+        <div class="flex gap-4 text-[10px] text-zinc-500">
+          <span><span style="color:${WF_MODEL}">◆</span> model</span>
+          <span><span style="color:${WF_TOOL}">▸</span> tool</span>
+        </div>
+        ${sceneBlocks}
+      </div>
+    </details>`;
+}
 function renderRunRow(entry, idx) {
     const { report, delta, diffLines } = entry;
     const pct = report.successRate * 100;
@@ -233,6 +317,7 @@ function renderRunRow(entry, idx) {
           </div>
           <div class="ml-10 mt-0.5 flex gap-3 flex-wrap">${dimTags}</div>
           ${diffHtml}
+          <div class="ml-10">${renderWaterfallHtml(report)}</div>
         </div>`;
 }
 // ---------------------------------------------------------------------------
@@ -990,6 +1075,12 @@ function renderSingleRun(report) {
           <p class="text-zinc-300">${Math.round(report.averageOutputTokensPerCase)}</p>
         </div>`
         : ""}
+        ${report.totalCostUsd != null
+        ? `<div>
+          <span class="text-zinc-500">Total Cost</span>
+          <p class="text-zinc-300">${fmtUsdHtml(report.totalCostUsd)}${report.totalInputTokens != null ? ` <span class="text-zinc-600">· ${report.totalInputTokens}→${report.totalOutputTokens} tok</span>` : ""}</p>
+        </div>`
+        : ""}
         ${report.tools && report.tools.length > 0
         ? `<div>
           <span class="text-zinc-500">Tools</span>
@@ -997,6 +1088,7 @@ function renderSingleRun(report) {
         </div>`
         : ""}
       </div>
+      ${renderWaterfallHtml(report)}
       ${failedSection}
     </div>`;
 }

package/dist/reports.d.ts

@@ -9,6 +9,31 @@ export interface ParsedSuiteResult {
         response?: string;
     }>;
 }
+export interface ParsedTimelineEvent {
+    kind: "model" | "tool";
+    name: string;
+    startMs: number;
+    durationMs: number;
+    tokens?: {
+        input: number;
+        output: number;
+    };
+    costUsd?: number;
+    costSource?: string;
+    runIndex?: number;
+    error?: string;
+}
+export interface ParsedScene {
+    prompt: string;
+    durationMs?: number;
+    tokens?: {
+        input: number;
+        output: number;
+    };
+    costUsd?: number;
+    costSource?: string;
+    timeline?: ParsedTimelineEvent[];
+}
 export interface ParsedReport {
     name?: string;
     systemPromptHash?: string;
@@ -28,6 +53,10 @@ export interface ParsedReport {
     timestamp: string;
     averageInputTokensPerCase?: number;
     averageOutputTokensPerCase?: number;
+    totalInputTokens?: number;
+    totalOutputTokens?: number;
+    totalCostUsd?: number;
+    scenes?: ParsedScene[];
     suites?: ParsedSuiteResult[];
     source: string;
 }
@@ -44,6 +73,13 @@ export declare function parseFailedCases(content: string): Array<{
 }>;
 export declare function parseDimensions(content: string): Record<string, string> | undefined;
 export declare function parseSuites(content: string): ParsedSuiteResult[] | undefined;
+/**
+ * Parse the `scenes:` block (per-scene tokens/cost + timeline waterfall) from a
+ * report. The emitted format is fixed (see reporter.ts `renderSceneObservability`),
+ * so this hand-parses by indentation: scenes start at 8 spaces, scene fields at
+ * 10, timeline events at 14, event fields at 16.
+ */
+export declare function parseScenes(content: string): ParsedScene[] | undefined;
 export declare function parseReport(content: string, source: string): ParsedReport;
 export declare function findReports(dir: string, depth?: number): Promise<string[]>;
 export declare function loadDiffEntry(hash: string): Promise<DiffEntry | null>;

package/dist/reports.js

@@ -124,6 +124,121 @@ export function parseSuites(content) {
         suites.push(current);
     return suites.length > 0 ? suites : undefined;
 }
+function parseTokens(raw) {
+    const m = raw.match(/input:\s*(\d+),\s*output:\s*(\d+)/);
+    if (!m)
+        return undefined;
+    return { input: parseInt(m[1], 10), output: parseInt(m[2], 10) };
+}
+/**
+ * Parse the `scenes:` block (per-scene tokens/cost + timeline waterfall) from a
+ * report. The emitted format is fixed (see reporter.ts `renderSceneObservability`),
+ * so this hand-parses by indentation: scenes start at 8 spaces, scene fields at
+ * 10, timeline events at 14, event fields at 16.
+ */
+export function parseScenes(content) {
+    const lines = content.split("\n");
+    const startIdx = lines.findIndex((l) => l === "    scenes:");
+    if (startIdx === -1)
+        return undefined;
+    const scenes = [];
+    let scene;
+    let inTimeline = false;
+    let event;
+    const pushEvent = () => {
+        if (event && scene) {
+            (scene.timeline ??= []).push(event);
+            event = undefined;
+        }
+    };
+    const pushScene = () => {
+        pushEvent();
+        if (scene)
+            scenes.push(scene);
+        scene = undefined;
+        inTimeline = false;
+    };
+    for (let i = startIdx + 1; i < lines.length; i++) {
+        const line = lines[i];
+        if (line.trim() === "")
+            continue;
+        const indent = line.length - line.trimStart().length;
+        // A new top-level agent field (<= 4 spaces, not part of scenes) ends the block.
+        if (indent <= 4)
+            break;
+        const sceneStart = line.match(/^        - prompt: "(.*)"$/);
+        if (sceneStart) {
+            pushScene();
+            scene = { prompt: sceneStart[1].replace(/\\"/g, '"').replace(/\\n/g, "\n") };
+            continue;
+        }
+        if (!scene)
+            continue;
+        const eventStart = line.match(/^              - kind: (model|tool)$/);
+        if (eventStart) {
+            pushEvent();
+            event = { kind: eventStart[1], name: "", startMs: 0, durationMs: 0 };
+            inTimeline = true;
+            continue;
+        }
+        if (line.match(/^          timeline:$/)) {
+            inTimeline = true;
+            continue;
+        }
+        const trimmed = line.trim();
+        const target = inTimeline && event ? "event" : "scene";
+        const kv = trimmed.match(/^([a-z_]+):\s*(.*)$/);
+        if (!kv)
+            continue;
+        const [, key, value] = kv;
+        if (target === "event" && event) {
+            switch (key) {
+                case "name":
+                    event.name = value.replace(/^"|"$/g, "").replace(/\\"/g, '"');
+                    break;
+                case "start_ms":
+                    event.startMs = parseFloat(value);
+                    break;
+                case "duration_ms":
+                    event.durationMs = parseFloat(value);
+                    break;
+                case "tokens":
+                    event.tokens = parseTokens(value);
+                    break;
+                case "cost_usd":
+                    event.costUsd = parseFloat(value);
+                    break;
+                case "cost_source":
+                    event.costSource = value;
+                    break;
+                case "run_index":
+                    event.runIndex = parseInt(value, 10);
+                    break;
+                case "error":
+                    event.error = value.replace(/^"|"$/g, "").replace(/\\"/g, '"');
+                    break;
+            }
+        }
+        else if (scene) {
+            switch (key) {
+                case "duration_ms":
+                    scene.durationMs = parseFloat(value);
+                    break;
+                case "tokens":
+                    scene.tokens = parseTokens(value);
+                    break;
+                case "cost_usd":
+                    scene.costUsd = parseFloat(value);
+                    break;
+                case "cost_source":
+                    scene.costSource = value;
+                    break;
+            }
+        }
+    }
+    pushScene();
+    return scenes.length > 0 ? scenes : undefined;
+}
 export function parseReport(content, source) {
     const num = (key, fallback = 0) => parseFloat(extractField(content, key) ?? String(fallback));
     const avgIn = extractField(content, "average_input_tokens_per_case");
@@ -158,9 +273,17 @@ export function parseReport(content, source) {
         timestamp: extractField(content, "timestamp") ?? "",
         averageInputTokensPerCase: avgIn != null ? parseFloat(avgIn) : undefined,
         averageOutputTokensPerCase: avgOut != null ? parseFloat(avgOut) : undefined,
+        totalInputTokens: optNum("total_input_tokens"),
+        totalOutputTokens: optNum("total_output_tokens"),
+        totalCostUsd: optNum("total_cost_usd"),
+        scenes: parseScenes(content),
         suites: parseSuites(content),
         source,
     };
+    function optNum(key) {
+        const v = extractField(content, key);
+        return v != null ? parseFloat(v) : undefined;
+    }
 }
 export async function findReports(dir, depth = 0) {
     if (depth > 6)

package/dist/waterfall.d.ts

@@ -0,0 +1,11 @@
+import type { TimelineEvent } from "./types";
+/**
+ * Render a Chrome-DevTools-style waterfall of timeline events as colored
+ * terminal lines. Bars are positioned by `startMs` and sized by `durationMs`
+ * relative to the full span of the scene. Returns one string per event row
+ * (already indented), or `[]` when there's nothing to draw.
+ */
+export declare function renderTerminalWaterfall(events: TimelineEvent[], opts?: {
+    width?: number;
+    indent?: string;
+}): string[];

package/dist/waterfall.js

@@ -0,0 +1,45 @@
+import { c } from "./logger";
+const BLOCK = "█";
+const THIN = "▏";
+function truncate(s, n) {
+    return s.length > n ? s.slice(0, n - 1) + "…" : s;
+}
+function fmtUsd(n) {
+    if (n === 0)
+        return "$0";
+    return "$" + Number(n.toFixed(4)).toString();
+}
+/**
+ * Render a Chrome-DevTools-style waterfall of timeline events as colored
+ * terminal lines. Bars are positioned by `startMs` and sized by `durationMs`
+ * relative to the full span of the scene. Returns one string per event row
+ * (already indented), or `[]` when there's nothing to draw.
+ */
+export function renderTerminalWaterfall(events, opts = {}) {
+    if (!events || events.length === 0)
+        return [];
+    const width = opts.width ?? 28;
+    const indent = opts.indent ?? "";
+    const t0 = Math.min(...events.map((e) => e.startMs));
+    const tEnd = Math.max(...events.map((e) => e.endMs));
+    const span = Math.max(1, tEnd - t0);
+    const nameWidth = 16;
+    return events.map((e) => {
+        const lead = Math.min(width - 1, Math.round(((e.startMs - t0) / span) * width));
+        const barLen = Math.max(1, Math.round((e.durationMs / span) * width));
+        const fill = e.durationMs === 0 ? THIN : BLOCK.repeat(Math.min(barLen, width - lead));
+        const cells = Array(width).fill(" ");
+        for (let i = 0; i < fill.length && lead + i < width; i++) {
+            cells[lead + i] = fill[i];
+        }
+        let bar = cells.join("");
+        const color = e.error ? c.red : e.kind === "model" ? c.cyan : c.yellow;
+        bar = color(bar);
+        const kindLabel = (e.kind === "model" ? "model" : "tool ").padEnd(5);
+        const nameLabel = truncate(e.name, nameWidth).padEnd(nameWidth);
+        const dur = `${Math.round(e.durationMs)}ms`.padStart(7);
+        const cost = e.cost?.totalUsd != null ? `  ${fmtUsd(e.cost.totalUsd)}` : "";
+        const err = e.error ? `  ${c.red("✗ " + truncate(e.error, 40))}` : "";
+        return `${indent}${c.dim(kindLabel)} ${nameLabel} ${bar} ${c.dim(dur)}${c.dim(cost)}${err}`;
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sebastiantuyu/agest",
-  "version": "0.3.3-next.5",
+  "version": "0.3.3-next.6",
   "description": "A testing library for agents",
   "repository": {
     "type": "git",