@tangle-network/agent-eval 0.120.2 → 0.120.3

package/dist/storyboard/index.d.ts

@@ -0,0 +1,279 @@
+type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
+type SpanStatus = 'ok' | 'error';
+interface SpanBase {
+    spanId: string;
+    parentSpanId?: string;
+    runId: string;
+    kind: SpanKind;
+    name: string;
+    startedAt: number;
+    endedAt?: number;
+    status?: SpanStatus;
+    error?: string;
+    /** Anything not covered by typed fields. Kept deliberately free-form. */
+    attributes?: Record<string, unknown>;
+}
+interface Message {
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content: string;
+    tokens?: number;
+    /** Multi-modal content descriptors; blobs themselves live in Artifacts. */
+    images?: Array<{
+        artifactId?: string;
+        url?: string;
+        mime?: string;
+    }>;
+}
+interface LlmSpan extends SpanBase {
+    kind: 'llm';
+    model: string;
+    messages: Message[];
+    output?: string;
+    inputTokens?: number;
+    /** All generated tokens, including the reasoning subset when present. */
+    outputTokens?: number;
+    cachedTokens?: number;
+    cacheWriteTokens?: number;
+    /** Reasoning-token subset of `outputTokens`. */
+    reasoningTokens?: number;
+    costUsd?: number;
+    finishReason?: string;
+}
+interface ToolSpan extends SpanBase {
+    kind: 'tool';
+    toolName: string;
+    args: unknown;
+    /** False when the source observed the call but did not capture its arguments. */
+    argsCaptured?: boolean;
+    result?: unknown;
+    latencyMs?: number;
+}
+interface RetrievalSpan extends SpanBase {
+    kind: 'retrieval';
+    query: string;
+    hits: Array<{
+        docId: string;
+        score: number;
+        content?: string;
+    }>;
+}
+interface JudgeSpan extends SpanBase {
+    kind: 'judge';
+    judgeId: string;
+    /** Span this judgment applies to. */
+    targetSpanId: string;
+    dimension: string;
+    /** Numeric score (free-range; interpretation up to the judge). */
+    score: number;
+    rationale?: string;
+    evidence?: string;
+}
+interface SandboxSpan extends SpanBase {
+    kind: 'sandbox';
+    image?: string;
+    command?: string;
+    exitCode?: number;
+    testsTotal?: number;
+    testsPassed?: number;
+    stdoutHash?: string;
+    stderrHash?: string;
+    /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */
+    wallMs?: number;
+}
+interface GenericSpan extends SpanBase {
+    kind: 'agent' | 'custom';
+}
+type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan;
+
+/**
+ * Code-edit extraction + the "watch the agent write code" animation.
+ *
+ * The base storyboard (./index) compresses a run into scenes but keeps scenes
+ * payload-free (title + narration + evidence span ids). To actually SHOW the
+ * code an agent wrote, this module pulls the concrete edit (path, diff,
+ * before/after, language) out of the raw edit-tool spans and renders a
+ * self-contained HTML replay that types each file out, character by character,
+ * like an editor recording.
+ *
+ * Additive on purpose: it reuses the canonical `Span` (trace/schema) and the
+ * `Storyboard` IR (./index) without modifying the reducer/compiler — so it
+ * composes with the existing pipeline and a Remotion consumer can read the same
+ * `CodeEdit[]` to render an MP4.
+ */
+
+/** A concrete code change lifted from a tool span. */
+interface CodeEdit {
+    /** Span the edit came from — back-reference into the trace. */
+    spanId: string;
+    path: string;
+    language?: string;
+    /** Unified diff, when the tool carried one. */
+    diff?: string;
+    /** Full file body before / after, when present — enables true keystroke
+     *  animation rather than only flashing a diff. */
+    before?: string;
+    after?: string;
+    additions: number;
+    deletions: number;
+    startedAt: number;
+}
+/** Extract a CodeEdit from a single span, or undefined if it is not an edit. */
+declare function codeEditFromSpan(span: Span): CodeEdit | undefined;
+/** All code edits in a run, in chronological order. */
+declare function extractCodeEdits(spans: readonly Span[]): CodeEdit[];
+/** Pair each code-edit (`diff`) scene with its concrete edit, matched via the
+ *  scene's evidence span ids. Scenes without a recoverable edit are dropped. */
+declare function codeEditsForStoryboard(storyboard: Storyboard, spans: readonly Span[]): Array<{
+    scene: Scene;
+    edit: CodeEdit;
+}>;
+/** The text the animation types for an edit: prefer the full new file body,
+ *  then the added lines of a diff, then a minimal placeholder so the scene is
+ *  never blank. */
+declare function editAnimationText(edit: CodeEdit): string;
+interface CodeAnimationOptions {
+    /** Page title. Default "Agent writing code". */
+    title?: string;
+    /** Characters typed per animation tick. Higher = faster. Default 4. */
+    charsPerTick?: number;
+    /** Pause between files, ms. Default 900. */
+    pauseBetweenMs?: number;
+}
+/**
+ * Render a self-contained HTML page that animates the agent writing each file,
+ * character by character, with a filename tab, line numbers, and a blinking
+ * cursor — the shareable "watch it build" clip. No build step, no assets, no
+ * deps. The same `CodeEdit[]` feeds a Remotion MP4 renderer in a consumer
+ * package; this is the dependency-free baseline.
+ */
+declare function renderCodeAnimationHtml(source: readonly Span[] | readonly CodeEdit[], opts?: CodeAnimationOptions): string;
+
+/**
+ * Storyboard — compile a raw agent trace into a small set of meaningful scenes,
+ * then render them as a shareable, watchable replay.
+ *
+ *   Span[] → SemanticEvent[] → Storyboard{scenes} → { markdown | html }
+ *
+ * The trace is the source of truth; the storyboard is the IR; markdown/html
+ * (and, in a consumer package, Remotion/MP4) are compiled targets. Everything
+ * here is PURE — same spans in, same bytes out (no clock/random) — so a run's
+ * replay is deterministic and diffable.
+ *
+ * Crucially, every scene carries a modality-typed VISUAL extracted from the
+ * span — a browser/computer screenshot, a code diff, a terminal command +
+ * output, file content, an API request/response, or reasoning prose — so no
+ * matter what the agent did, the replay SHOWS it, not just narrates it. The
+ * reducer is rules-based; an LLM pass can refine titles/summaries later, but
+ * the structure + visuals do not depend on one. Renderers that need a browser
+ * (Remotion, React) live in consumers; this module only emits strings.
+ */
+
+/** What the agent meaningfully did — the compressed vocabulary. */
+type SemanticKind = 'understood_task' | 'user_message' | 'agent_reply' | 'reasoned' | 'ran_command' | 'read_file' | 'edited_code' | 'searched' | 'used_browser' | 'used_computer' | 'called_api' | 'called_tool' | 'evaluated' | 'observed_failure' | 'completed';
+/** The modality-typed payload a scene shows. This is what makes any agent
+ *  action — screen, browser, shell, code, API — actually viewable. */
+type SceneVisual = {
+    type: 'screenshot';
+    src: string;
+    caption?: string;
+} | {
+    type: 'browser';
+    url?: string;
+    action?: string;
+    screenshot?: string;
+} | {
+    type: 'diff';
+    path: string;
+    patch: string;
+} | {
+    type: 'code';
+    path: string;
+    content: string;
+} | {
+    type: 'terminal';
+    command: string;
+    output?: string;
+} | {
+    type: 'api';
+    method?: string;
+    url: string;
+    status?: number;
+    request?: string;
+    response?: string;
+} | {
+    type: 'prose';
+    text: string;
+} | {
+    type: 'none';
+};
+interface SemanticEvent {
+    kind: SemanticKind;
+    /** One-line headline (the ticket/scene title). */
+    title: string;
+    /** Short human summary. */
+    summary: string;
+    /** The modality payload to show for this moment. */
+    visual: SceneVisual;
+    /** Span ids backing this moment — the evidence trail. */
+    evidenceSpanIds: string[];
+    /** 1 (noise) … 5 (pivotal: failures, edits). Drives selection + duration. */
+    importance: 1 | 2 | 3 | 4 | 5;
+    startTs: number;
+    endTs: number;
+}
+interface ReduceOptions {
+    /** Collapse adjacent same-kind moments into one. Default true. */
+    collapseAdjacent?: boolean;
+}
+/**
+ * Reduce a span tree into the meaningful moments a viewer cares about. The
+ * conversation (user asks, agent replies) is lifted from LLM spans first so the
+ * dialogue is first-class; the remaining work spans are classified + have their
+ * modality visual extracted. Adjacent same-kind work moments collapse (the
+ * compression step), carrying the latest visual; failures and conversation
+ * turns never collapse.
+ */
+declare function reduceToSemanticEvents(spans: readonly Span[], opts?: ReduceOptions): SemanticEvent[];
+type SceneType = 'title_card' | 'prompt' | 'reply' | 'reasoning' | 'terminal' | 'file' | 'diff' | 'search' | 'browser' | 'computer' | 'api' | 'tool' | 'eval' | 'error' | 'summary';
+interface Scene {
+    sceneType: SceneType;
+    title: string;
+    narration: string;
+    /** The modality payload the renderer shows. */
+    visual: SceneVisual;
+    durationMs: number;
+    evidenceSpanIds: string[];
+}
+interface Storyboard {
+    title: string;
+    totalMs: number;
+    scenes: Scene[];
+}
+interface CompileOptions {
+    /** Headline for the title card. Default "Agent run". */
+    title?: string;
+    /** Max action scenes between the title + summary cards. Default 16. */
+    maxScenes?: number;
+}
+/**
+ * Select the moments worth showing and lay them out as scenes. Every failure
+ * (importance 5) and edit (4) is always kept; the rest fill the budget by
+ * importance, then re-sort chronological. A title card opens, a summary closes.
+ */
+declare function compileStoryboard(events: readonly SemanticEvent[], opts?: CompileOptions): Storyboard;
+/** Render the storyboard as a Markdown timeline — the human-readable shot list. */
+declare function renderStoryboardMarkdown(storyboard: Storyboard): string;
+interface HtmlRenderOptions {
+    /** Document title + on-page heading. Defaults to the storyboard title. */
+    title?: string;
+}
+/**
+ * Render the storyboard as a self-contained, auto-playing HTML replay — one
+ * shareable file, no build step, no external assets. Each scene animates in
+ * for its duration and SHOWS its modality (screenshot / diff / terminal / API
+ * / browser / prose); controls let a viewer scrub. This is the "free clip" a
+ * run produces; an MP4 is the same storyboard fed to Remotion in a consumer.
+ */
+declare function renderStoryboardHtml(storyboard: Storyboard, opts?: HtmlRenderOptions): string;
+
+export { type CodeAnimationOptions, type CodeEdit, type CompileOptions, type HtmlRenderOptions, type ReduceOptions, type Scene, type SceneType, type SceneVisual, type SemanticEvent, type SemanticKind, type Storyboard, codeEditFromSpan, codeEditsForStoryboard, compileStoryboard, editAnimationText, extractCodeEdits, reduceToSemanticEvents, renderCodeAnimationHtml, renderStoryboardHtml, renderStoryboardMarkdown };