@theoremjs/react 0.0.0-stage → 0.3.0

package/client/trace-story.js

@@ -0,0 +1,416 @@
+import { inlineContent, traceAttributeMeta, traceEventAttributeMeta, traceEventMeta } from '@theoremjs/agents';
+import { isRecord } from '@theoremjs/agents/kernel';
+import { nanosToMs, traceSpans, traceTotals } from "./trace-view.js";
+export function traceTurns(tree) {
+    return tree.map((node) => ({
+        node,
+        totals: traceTotals([node]),
+        split: traceTimeSplit(node),
+        failed: node.span.status.code === 'ERROR',
+    }));
+}
+/** A host's trace: each root is one tool call, not a conversation's turn. */
+export function isCallTrace(turns) {
+    return turns.length > 0 && turns.every((turn) => TOOL_TYPES.has(turn.node.meta.type));
+}
+/** A turn named by what the person asked, a host's call by its tool; a failure by its outcome, else the answer. */
+export function traceTurnRow(turn, isCall) {
+    const story = traceStory(turn.node);
+    const ask = story.find((step) => step.kind === 'ask');
+    const answer = story.findLast((step) => step.kind === 'answer');
+    const title = isCall ? turn.node.meta.subject : ask?.kind === 'ask' ? ask.text : undefined;
+    const outcome = turn.failed ? traceOutcome(turn.node) : undefined;
+    const answered = answer?.kind === 'answer' ? answer.text : undefined;
+    return {
+        actor: turn.failed ? 'error' : isCall ? 'tool' : 'user',
+        ...(title ? { title } : {}),
+        ...(outcome ? { outcome } : answered ? { answered } : {}),
+    };
+}
+export function traceActor(node) {
+    if (node.span.status.code === 'ERROR')
+        return 'error';
+    switch (node.meta.type) {
+        case 'call':
+        case 'response':
+        case 'decision':
+            return 'model';
+        case 'tool':
+            return 'tool';
+        case 'cutout':
+        case 'host':
+            return 'host';
+        default:
+            return 'theorem';
+    }
+}
+/** Events worth a line of their own: something acted, retried, waited or warned. */
+const STORY_EVENTS = new Set([
+    'theorem.guardrail',
+    'theorem.attempt.retry',
+    'theorem.gate',
+    'theorem.tool.warning',
+    'theorem.tool.cancel',
+]);
+/** Event keys whose value reads as a detail of the line, in this order. */
+const EVENT_DETAIL_KEYS = ['action', 'stage', 'kind', 'reason', 'code'];
+function stepOf(node) {
+    const step = node.span.attributes['theorem.step'];
+    return typeof step === 'number' ? step : undefined;
+}
+function messagesOf(node, key) {
+    const value = inlineContent(node.record, node.span.attributes[key]);
+    return Array.isArray(value) ? value.filter(isRecord) : [];
+}
+function partsText(parts) {
+    if (!Array.isArray(parts))
+        return '';
+    return parts
+        .filter((part) => isRecord(part) && part.type === 'text' && typeof part.content === 'string')
+        .map((part) => part.content)
+        .join('\n')
+        .trim();
+}
+/** The last message's text from `role`, read from a span's input or output messages. */
+export function messageText(node, key, role) {
+    const found = messagesOf(node, key).filter((message) => message.role === role);
+    for (let index = found.length - 1; index >= 0; index -= 1) {
+        const text = partsText(found[index]?.parts);
+        if (text)
+            return text;
+    }
+    return '';
+}
+/** A model call that wrote text back (rather than only asking for tools). */
+function wroteText(node) {
+    return messageText(node, 'gen_ai.output.messages', 'assistant') !== '';
+}
+function eventDetail(name, attributes) {
+    return EVENT_DETAIL_KEYS.flatMap((key) => {
+        const value = attributes[key];
+        if (typeof value !== 'string')
+            return [];
+        const meta = traceEventAttributeMeta(name, key);
+        return [meta?.options?.[value]?.label ?? value];
+    });
+}
+function isQuiet(name, attributes) {
+    return name === 'theorem.guardrail' && attributes.action === 'allow';
+}
+function storyEvents(node, turnStart) {
+    return node.span.events.flatMap((event, index) => {
+        if (!STORY_EVENTS.has(event.name) || isQuiet(event.name, event.attributes))
+            return [];
+        const meta = traceEventMeta(event.name);
+        return [
+            {
+                kind: 'event',
+                id: `${node.id}:event:${index}`,
+                node,
+                offsetMs: nanosToMs(event.timeUnixNano) - turnStart,
+                label: meta?.label ?? event.name,
+                detail: eventDetail(event.name, event.attributes),
+                doc: meta?.doc ?? event.name,
+            },
+        ];
+    });
+}
+/**
+ * A turn as it happened: what the user asked, each model call and the tool
+ * calls it asked for (nested under it), the events that changed the course,
+ * and how it ended. HTTP tries stay in the timeline; they are how a call
+ * travelled, not a step of the story.
+ */
+export function traceStory(root) {
+    const start = root.startMs;
+    const spans = traceSpans(root.children)
+        .filter((node) => node.meta.type !== 'http')
+        .sort((a, b) => a.startMs - b.startMs);
+    const toolsByStep = new Map();
+    for (const node of spans) {
+        const step = stepOf(node);
+        if (node.meta.type === 'tool' && step !== undefined)
+            toolsByStep.set(step, (toolsByStep.get(step) ?? 0) + 1);
+    }
+    const calledSteps = new Set();
+    const steps = [];
+    const asked = messageText(root, 'gen_ai.input.messages', 'user');
+    if (asked)
+        steps.push({ kind: 'ask', id: `${root.id}:ask`, node: root, offsetMs: 0, text: asked });
+    steps.push(...storyEvents(root, start).filter((step) => step.offsetMs <= 0));
+    for (const node of spans) {
+        const step = stepOf(node);
+        const isCall = node.meta.type === 'call' || node.meta.type === 'response';
+        if (isCall && step !== undefined)
+            calledSteps.add(step);
+        steps.push({
+            kind: 'span',
+            id: node.id,
+            node,
+            offsetMs: node.startMs - start,
+            nested: node.meta.type === 'tool' && step !== undefined && calledSteps.has(step),
+            requested: isCall && step !== undefined && !wroteText(node) ? (toolsByStep.get(step) ?? 0) : 0,
+        });
+        steps.push(...storyEvents(node, start));
+    }
+    const rootEvents = storyEvents(root, start).filter((step) => step.offsetMs > 0);
+    const ordered = [...steps, ...rootEvents].sort((a, b) => a.offsetMs - b.offsetMs || rank(a) - rank(b));
+    if (root.meta.type === 'turn') {
+        const answer = messageText(root, 'gen_ai.output.messages', 'assistant');
+        ordered.push({
+            kind: 'answer',
+            id: `${root.id}:answer`,
+            node: root,
+            offsetMs: root.durationMs,
+            ...(answer && { text: answer }),
+        });
+    }
+    return ordered;
+}
+/** Keeps the question first and a span ahead of the events it recorded at the same instant. */
+function rank(step) {
+    return step.kind === 'ask' ? 0 : step.kind === 'span' ? 1 : 2;
+}
+/** A tool call's arguments as `[key, value]` pairs of plain values, for a one-line summary. */
+export function toolArguments(node) {
+    const stored = inlineContent(node.record, node.span.attributes['gen_ai.tool.call.arguments']);
+    let value = stored;
+    if (typeof stored === 'string') {
+        try {
+            value = JSON.parse(stored);
+        }
+        catch {
+            return stored ? [['', stored]] : [];
+        }
+    }
+    if (!isRecord(value))
+        return [];
+    return Object.entries(value).flatMap(([key, item]) => typeof item === 'string' || typeof item === 'number' || typeof item === 'boolean' ? [[key, String(item)]] : []);
+}
+/** A stored JSON or text value, parsed when it is JSON. */
+export function storedValue(node, key) {
+    const value = inlineContent(node.record, node.span.attributes[key]);
+    if (typeof value !== 'string')
+        return value;
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return value;
+    }
+}
+function optionOf(key, value) {
+    return typeof value === 'string' ? traceAttributeMeta(key)?.options?.[value] : undefined;
+}
+export function traceOutcome(node) {
+    const { attributes, status } = node.span;
+    const outcome = attributes['theorem.tool.outcome'];
+    const code = attributes['theorem.tool.failure.code'];
+    const codePart = typeof code === 'string' ? { code } : {};
+    const tool = outcome !== 'ok' ? optionOf('theorem.tool.outcome', outcome) : undefined;
+    if (tool)
+        return { ...tool, ...codePart, tone: status.code === 'ERROR' || outcome === 'error' ? 'error' : 'warning' };
+    if (status.code !== 'ERROR')
+        return undefined;
+    const type = attributes['error.type'];
+    const kind = optionOf('error.type', type);
+    if (kind)
+        return { ...kind, ...codePart, tone: 'error' };
+    return { label: typeof type === 'string' ? type : (status.message ?? ''), ...codePart, tone: 'error' };
+}
+function guardrailHits(value) {
+    if (!Array.isArray(value))
+        return [];
+    const fields = traceEventAttributeMeta('theorem.guardrail', 'hits')?.fields;
+    return value.filter(isRecord).flatMap((hit) => {
+        if (typeof hit.rule !== 'string')
+            return [];
+        const severity = typeof hit.severity === 'string' ? hit.severity : '';
+        // A host names its own rules on the hit; Theorem's are named in the catalog.
+        const rule = fields?.rule?.options?.[hit.rule];
+        const ruleDoc = typeof hit.doc === 'string' ? hit.doc : rule?.doc;
+        return [
+            {
+                rule: hit.rule,
+                ruleLabel: typeof hit.label === 'string' ? hit.label : (rule?.label ?? hit.rule),
+                ...(ruleDoc && { ruleDoc }),
+                severity,
+                severityLabel: fields?.severity?.options?.[severity]?.label ?? severity,
+                ...(typeof hit.match === 'string' && { match: hit.match }),
+            },
+        ];
+    });
+}
+function eventOption(key, value) {
+    return traceEventAttributeMeta('theorem.guardrail', key)?.options?.[value];
+}
+/** Every guardrail check the turn recorded, in the order they ran: the passes as well as what they caught. */
+export function traceGuardrails(root) {
+    return traceSpans([root])
+        .flatMap((node) => node.span.events.flatMap((event, index) => {
+        if (event.name !== 'theorem.guardrail')
+            return [];
+        const { attributes } = event;
+        const stage = String(attributes.stage ?? '');
+        const action = String(attributes.action ?? '');
+        const actionMeta = eventOption('action', action);
+        const check = typeof attributes.check === 'string' ? attributes.check : undefined;
+        const provenance = isRecord(attributes.provenance) ? attributes.provenance : undefined;
+        const reason = inlineContent(node.record, attributes.error);
+        return [
+            {
+                id: `${node.id}:guardrail:${index}`,
+                node,
+                offsetMs: nanosToMs(event.timeUnixNano) - root.startMs,
+                stage,
+                stageLabel: eventOption('stage', stage)?.label ?? stage,
+                ...(check && { checkLabel: eventOption('check', check)?.label ?? check }),
+                action,
+                actionLabel: actionMeta?.label ?? action,
+                ...(actionMeta && { actionDoc: actionMeta.doc }),
+                ...(typeof attributes.duration_ms === 'number' && { durationMs: attributes.duration_ms }),
+                hits: guardrailHits(attributes.hits),
+                ...(typeof provenance?.tool === 'string' && { tool: provenance.tool }),
+                ...(typeof reason === 'string' && reason && { reason }),
+            },
+        ];
+    }))
+        .sort((a, b) => a.offsetMs - b.offsetMs);
+}
+function union(intervals) {
+    const sorted = [...intervals].sort((a, b) => a[0] - b[0]);
+    let total = 0;
+    let end = Number.NEGATIVE_INFINITY;
+    for (const [from, to] of sorted) {
+        if (to <= end)
+            continue;
+        total += to - Math.max(from, end);
+        end = to;
+    }
+    return total;
+}
+function intervalsOf(nodes, types) {
+    return nodes.filter((node) => types.has(node.meta.type)).map((node) => [node.startMs, node.startMs + node.durationMs]);
+}
+const MODEL_TYPES = new Set(['call', 'response', 'decision']);
+const TOOL_TYPES = new Set(['tool']);
+/** Milliseconds a span's events of `name` recorded under `key`. */
+function eventMs(node, name, key) {
+    return node.span.events.reduce((total, event) => {
+        const value = event.name === name ? event.attributes[key] : undefined;
+        return total + (typeof value === 'number' ? value : 0);
+    }, 0);
+}
+/** Milliseconds all of `nodes`' events of `name` recorded under `key`. */
+function eventsMs(nodes, name, key) {
+    return nodes.reduce((total, node) => total + eventMs(node, name, key), 0);
+}
+/**
+ * Where a turn's time went. Guardrail checks and hooks run inside the turn's
+ * gaps and inside tool calls; their recorded time moves out of those into
+ * their own share, so the model and tools keep only their own work.
+ */
+function traceTimeSplit(root) {
+    // A host's call is its own root: the tool span is the whole trace.
+    const spans = TOOL_TYPES.has(root.meta.type) ? [root, ...traceSpans(root.children)] : traceSpans(root.children);
+    const model = intervalsOf(spans, MODEL_TYPES);
+    const tools = intervalsOf(spans, TOOL_TYPES);
+    const toolNodes = spans.filter((node) => TOOL_TYPES.has(node.meta.type));
+    const streamGuard = spans.reduce((total, node) => {
+        const value = node.span.attributes['theorem.guardrail.stream_ms'];
+        return total + (typeof value === 'number' ? value : 0);
+    }, 0);
+    const modelMs = union(model);
+    const toolsMs = union(tools);
+    const toolGuard = Math.min(toolsMs, eventsMs(toolNodes, 'theorem.guardrail', 'duration_ms'));
+    const toolHooks = Math.min(toolsMs - toolGuard, eventsMs(toolNodes, 'theorem.stage', 'hook_ms'));
+    const between = Math.max(0, root.durationMs - union([...model, ...tools]));
+    const turnGuard = Math.min(between, eventMs(root, 'theorem.guardrail', 'duration_ms'));
+    const turnHooks = Math.min(between - turnGuard, eventMs(root, 'theorem.stage', 'hook_ms'));
+    return {
+        model: Math.max(0, modelMs - streamGuard),
+        tools: toolsMs - toolGuard - toolHooks,
+        guardrails: turnGuard + Math.min(streamGuard, modelMs) + toolGuard,
+        hooks: turnHooks + toolHooks,
+        other: between - turnGuard - turnHooks,
+    };
+}
+const MS_PER_S = 1000;
+function secondsAttribute(node, key) {
+    const value = node.span.attributes[key];
+    return typeof value === 'number' ? value * MS_PER_S : undefined;
+}
+export function traceLatency(root) {
+    const spans = traceSpans([root]);
+    const calls = spans.filter((node) => MODEL_TYPES.has(node.meta.type));
+    const firstTextMs = secondsAttribute(root, 'theorem.turn.time_to_first_text');
+    // When the model first wrote, measured from the turn's start.
+    const written = calls
+        .map((node) => {
+        const at = secondsAttribute(node, 'theorem.response.time_to_first_text');
+        return at === undefined ? undefined : { node, at };
+    })
+        .filter((entry) => entry !== undefined);
+    const firstWritten = Math.min(...written.map(({ node, at }) => node.startMs - root.startMs + at));
+    let tokens = 0;
+    let writingMs = 0;
+    for (const { node, at } of written) {
+        const output = node.span.attributes['gen_ai.usage.output_tokens'];
+        if (typeof output !== 'number' || node.durationMs <= at)
+            continue;
+        tokens += output;
+        writingMs += node.durationMs - at;
+    }
+    let checks = 0;
+    let flagged = 0;
+    let guardMs = 0;
+    for (const node of spans) {
+        let streamChecksMs = 0;
+        for (const event of node.span.events) {
+            if (event.name !== 'theorem.guardrail')
+                continue;
+            checks += 1;
+            if (event.attributes.action !== 'allow')
+                flagged += 1;
+            const took = event.attributes.duration_ms;
+            if (typeof took !== 'number')
+                continue;
+            guardMs += took;
+            if (typeof event.attributes.runs === 'number')
+                streamChecksMs += took;
+        }
+        // A call's stream checks carry their own time; the call's total adds only what they do not (a trace from before they did).
+        const stream = node.span.attributes['theorem.guardrail.stream_ms'];
+        if (typeof stream === 'number')
+            guardMs += Math.max(0, stream - streamChecksMs);
+    }
+    return {
+        ...(firstTextMs !== undefined && { firstTextMs }),
+        ...(firstTextMs !== undefined && Number.isFinite(firstWritten) && { heldMs: Math.max(0, firstTextMs - firstWritten) }),
+        ...(writingMs > 0 && { tokensPerSecond: tokens / (writingMs / MS_PER_S) }),
+        ...(checks > 0 && { guardrails: { checks, ms: guardMs, flagged } }),
+    };
+}
+export function traceBars(root) {
+    const bars = [];
+    const walk = (node, depth) => {
+        const from = node.startMs - root.startMs;
+        bars.push({ node, depth, from, to: from + node.durationMs });
+        for (const child of node.children)
+            walk(child, depth + 1);
+    };
+    walk(root, 0);
+    return bars;
+}
+/** How busy the model and the tools were across the turn, in `buckets` equal slices (0 to 1 each). */
+export function traceActivity(root, buckets) {
+    const spans = traceSpans(root.children);
+    const model = intervalsOf(spans, MODEL_TYPES);
+    const tools = intervalsOf(spans, TOOL_TYPES);
+    const width = Math.max(root.durationMs, 1) / buckets;
+    const share = (intervals, from, to) => Math.min(1, union(intervals.map(([a, b]) => [Math.max(a, from), Math.min(b, to)]).filter(([a, b]) => b > a)) / width);
+    return Array.from({ length: buckets + 1 }, (_, index) => {
+        const from = root.startMs + index * width;
+        return { at: index * width, model: share(model, from, from + width), tools: share(tools, from, from + width) };
+    });
+}

package/client/trace-view.d.ts

@@ -0,0 +1,130 @@
+import { type TraceAttributeMeta, type TraceRecord, type TraceSpan, type TraceSpanMeta } from '@theoremjs/agents';
+/**
+ * The trace panel's reading of the records a run delivered: one span tree,
+ * totals over it, and search over its spans. Every word comes from the trace
+ * catalog; the panel only lays it out.
+ *
+ * @module
+ */
+/** One span, placed: the record that carries it, what it is, its timing, and the spans inside it. */
+export interface TraceNode {
+    /** `traceId:spanId`, unique across records. */
+    id: string;
+    span: TraceSpan;
+    record: TraceRecord;
+    meta: TraceSpanMeta;
+    /** Unix milliseconds. */
+    startMs: number;
+    durationMs: number;
+    /** In start order. */
+    children: TraceNode[];
+}
+/** A unix-nanosecond timestamp string as unix milliseconds. */
+export declare function nanosToMs(nanos: string): number;
+/**
+ * Every span of `records` as one tree. A span written twice keeps its later
+ * copy; a span whose parent has not arrived yet (a Live response before its
+ * session root) is a root until the parent does.
+ */
+export declare function traceTree(records: readonly TraceRecord[]): TraceNode[];
+/** Every node of `nodes`, depth first. */
+export declare function traceSpans(nodes: readonly TraceNode[]): TraceNode[];
+/** A sum over the model calls; `complete` is false when some call did not report it. */
+export interface TraceSum {
+    value: number;
+    complete: boolean;
+}
+/** A token sum; `estimated` when THEOREM estimated some call's count rather than the provider reporting it. */
+export interface TraceTokenSum extends TraceSum {
+    estimated: boolean;
+}
+/**
+ * Totals over a set of spans. Usage and cost sum the model calls (and Live
+ * responses) themselves, not the roots, so a specialist's or a compaction's
+ * calls count once. A total no call reported is absent: unknown, not zero.
+ */
+export interface TraceTotals {
+    spans: number;
+    errors: number;
+    /** First start to last end. */
+    durationMs: number;
+    /** Model calls and Live responses. */
+    calls: number;
+    input?: TraceTokenSum;
+    cached?: TraceSum;
+    output?: TraceTokenSum;
+    cost?: TraceSum;
+}
+export declare function traceTotals(nodes: readonly TraceNode[]): TraceTotals;
+/** How a search field compares: one of a set, a number, text, or yes/no. */
+export type TraceSearchKind = 'options' | 'number' | 'text' | 'flag';
+/** The operators each kind offers, by PowerSearch operator key. */
+export declare const TRACE_SEARCH_OPERATORS: {
+    readonly options: readonly ["isAnyOf", "isNoneOf"];
+    readonly number: readonly ["greaterThan", "lessThan"];
+    readonly text: readonly ["contains", "notContains"];
+    readonly flag: readonly ["isTrue", "isFalse"];
+};
+/** One searchable field of a span, worded by the catalog. */
+export interface TraceSearchField {
+    key: string;
+    label: string;
+    doc: string;
+    kind: TraceSearchKind;
+    /** The attribute group's label; absent for a span's own fields and for keys the catalog does not name. */
+    group?: string;
+    /** For `options`: each value seen, with its meaning when the catalog knows it. */
+    options?: readonly TraceSearchOption[];
+    /** For a number field: its unit, when it has one. */
+    format?: TraceAttributeMeta['format'];
+}
+export interface TraceSearchOption {
+    value: string;
+    label: string;
+    doc?: string;
+}
+/** The field that searches any text a span holds: its name, its values, its stored text. */
+export declare const TRACE_TEXT_FIELD = "text";
+/** A filter as PowerSearch emits it. */
+export interface TraceFilter {
+    readonly field: string;
+    readonly operator: string;
+    readonly value: {
+        readonly type: string;
+        readonly value?: unknown;
+    };
+}
+/**
+ * The fields a search over `nodes` offers: type, status, duration and events,
+ * then every attribute the spans recorded that compares as a value. Stored
+ * text, JSON and structured values are reached by the text field.
+ */
+export declare function traceSearchFields(nodes: readonly TraceNode[]): TraceSearchField[];
+/**
+ * The tree cut to the spans that meet every filter, each kept under its
+ * ancestors so it reads in place; `matches` counts the spans that met them.
+ */
+export declare function filterTraceTree(nodes: readonly TraceNode[], filters: readonly TraceFilter[]): {
+    nodes: TraceNode[];
+    matches: number;
+};
+/** The ids of the spans under `nodes` that meet every filter; undefined when there are none to meet. */
+export declare function traceMatchIds(nodes: readonly TraceNode[], filters: readonly TraceFilter[]): Set<string> | undefined;
+/** How to show an attribute value: by its catalog entry first, then by its shape. */
+export type TraceValueShape = {
+    kind: 'stored';
+} | {
+    kind: 'text';
+    value: string;
+} | {
+    kind: 'scalar';
+    value: number | boolean;
+} | {
+    kind: 'list';
+    value: string[];
+} | {
+    kind: 'fields';
+    fields: Readonly<Record<string, TraceAttributeMeta>>;
+    items: Record<string, unknown>[];
+};
+export declare function traceValueShape(meta: TraceAttributeMeta | undefined, value: unknown): TraceValueShape;