@qvac/harness 1.0.0-rc.0 → 1.0.0-rc.2

package/dist-lib/plugins/run-report/flow.js

@@ -0,0 +1,201 @@
+const PER_LINE = 42;
+const ROUND_W = 250;
+const CALL_W = 300;
+const GAP_X = 44;
+const GAP_Y = 30;
+export function flow(model) {
+    const nodes = graph(model);
+    own(nodes, model.steps);
+    const rows = rowsOf(nodes);
+    const edges = [];
+    let y = 0;
+    let w = 0;
+    for (const row of rows) {
+        const [lead, ...calls] = row;
+        const h = Math.max(...row.map((node) => node.h));
+        Object.assign(lead ?? {}, { x: 0, y, w: ROUND_W });
+        calls.forEach((call, index) => Object.assign(call, { x: ROUND_W + GAP_X + index * (CALL_W + 24), y }));
+        for (const [index, node] of row.slice(1).entries()) {
+            const from = row[index];
+            if (from) {
+                edges.push(across(from, node));
+            }
+        }
+        w = Math.max(w, ...row.map((node) => node.x + node.w));
+        y += h + GAP_Y;
+    }
+    for (const [index, row] of rows.slice(1).entries()) {
+        const above = rows[index]?.[0];
+        const lead = row[0];
+        if (above && lead) {
+            edges.push(down(above, lead));
+        }
+    }
+    return { w, h: y - GAP_Y, nodes, edges };
+}
+function rowsOf(nodes) {
+    const rows = [];
+    for (const node of nodes) {
+        if (node.kind === 'call') {
+            rows.at(-1)?.push(node);
+        }
+        else {
+            rows.push([node]);
+        }
+    }
+    return rows;
+}
+function across(from, to) {
+    const y = from.y + 21;
+    return {
+        source: from.id,
+        target: to.id,
+        points: [
+            [from.x + from.w, y],
+            [to.x, y]
+        ]
+    };
+}
+function down(from, to) {
+    const x = ROUND_W / 2;
+    return {
+        source: from.id,
+        target: to.id,
+        points: [
+            [x, from.y + from.h],
+            [x, to.y]
+        ]
+    };
+}
+function graph(model) {
+    const { run, story } = model;
+    const nodes = [
+        node({
+            id: 'start',
+            kind: 'start',
+            status: 'start',
+            title: 'Request',
+            lines: wrap(story.prompt)
+        })
+    ];
+    for (const round of story.rounds) {
+        nodes.push(roundNode(round), ...round.calls.map((call, index) => callNode(call, index)));
+    }
+    const good = run.outcome === 'success' || run.status === 'done';
+    nodes.push(node({
+        id: 'end',
+        kind: 'end',
+        status: good ? 'done' : 'error',
+        title: good ? 'Answer' : `Ended: ${run.outcome}`,
+        lines: wrap(good ? run.response : run.failure || run.response)
+    }));
+    return nodes;
+}
+function roundNode(round) {
+    const meta = [seconds(round.durationMs), round.tokens === null ? '' : `${round.tokens} tokens`];
+    return node({
+        id: `round-${round.round}`,
+        kind: 'round',
+        status: 'model',
+        title: `Round ${round.round + 1}`,
+        meta: meta.filter(Boolean).join(' · '),
+        lines: wrap(round.text),
+        round: round.round,
+        from: round.step
+    });
+}
+function callNode(call, index) {
+    const asked = call.approved === null ? '' : `${call.approved ? 'approved' : 'declined'} by a person`;
+    const said = call.outcome === 'ok' ? gist(call.args) : call.detail;
+    const lines = call.outcome === 'denied' && asked ? [] : wrap(said);
+    return node({
+        id: `call-${call.round}-${index}`,
+        kind: 'call',
+        status: call.outcome,
+        title: call.name,
+        meta: seconds(call.durationMs),
+        lines,
+        badge: asked,
+        round: call.round,
+        call: index,
+        from: call.step
+    });
+}
+function node(input) {
+    const lines = input.lines ?? [];
+    const badge = input.badge ?? '';
+    return {
+        meta: '',
+        badge,
+        round: null,
+        call: null,
+        from: null,
+        ...input,
+        lines,
+        steps: [],
+        x: 0,
+        y: 0,
+        w: input.kind === 'call' ? CALL_W : ROUND_W,
+        h: 40 + lines.length * 17 + (badge ? 22 : 0)
+    };
+}
+function own(nodes, steps) {
+    const started = nodes.filter((node) => node.from !== null);
+    const end = nodes.at(-1);
+    let finishing = false;
+    for (const step of steps) {
+        finishing ||=
+            step.kind === 'transition' &&
+                step.entered.some((id) => /\.(answering|wrapping|finishing)$/.test(id));
+        const owner = finishing
+            ? end
+            : (started.filter((node) => Number(node.from) <= step.index).at(-1) ?? nodes[0]);
+        owner?.steps.push(step.index);
+    }
+}
+function wrap(text, max = 2) {
+    const words = text.replace(/\s+/g, ' ').trim().split(' ').filter(Boolean);
+    const lines = [];
+    let line = '';
+    for (const word of words) {
+        if (line && line.length + word.length + 1 > PER_LINE) {
+            lines.push(line);
+            line = '';
+        }
+        line = line ? `${line} ${word}` : word.slice(0, PER_LINE);
+        if (lines.length === max) {
+            break;
+        }
+    }
+    if (line && lines.length < max) {
+        lines.push(line);
+    }
+    if (lines.length === max && words.join(' ').length > lines.join(' ').length) {
+        lines[max - 1] = `${(lines[max - 1] ?? '').slice(0, PER_LINE - 1)}…`;
+    }
+    return lines;
+}
+const NAMED = ['intent', 'summary', 'query', 'q', 'path', 'url', 'to', 'text', 'name'];
+function gist(args) {
+    const texts = [];
+    const walk = (value, key, depth) => {
+        if (typeof value === 'string') {
+            texts.push([key, value]);
+        }
+        else if (value && typeof value === 'object' && depth < 4) {
+            for (const [child, item] of Object.entries(value)) {
+                walk(item, Array.isArray(value) ? key : child, depth + 1);
+            }
+        }
+    };
+    walk(args, '', 0);
+    const named = NAMED.flatMap((name) => texts.filter(([key]) => key === name))[0];
+    const longest = [...texts].sort((a, b) => b[1].length - a[1].length)[0];
+    return (named ?? longest)?.[1] ?? '';
+}
+function seconds(ms) {
+    if (ms === null) {
+        return '';
+    }
+    return ms < 1000 ? `${Math.round(ms)} ms` : `${(ms / 1000).toFixed(1)} s`;
+}

package/dist-lib/plugins/run-report/index.d.ts

@@ -0,0 +1,17 @@
+import { type EngineBoot, type EngineStep, type MachineLog, type RunRecord } from '../../lib/run-log/index.ts';
+export interface RunReportOptions {
+    title?: string;
+}
+export interface RunRecordingOptions {
+    boot: EngineBoot;
+    onStep: (step: EngineStep) => Promise<void>;
+    requestDigestKey: Uint8Array;
+}
+export interface RunRecording {
+    records: RunRecord[];
+    options: RunRecordingOptions;
+}
+export declare class RunReport {
+    static record(): RunRecording;
+    static render(source: MachineLog | RunRecord[], { title }?: RunReportOptions): Promise<string>;
+}

package/dist-lib/plugins/run-report/index.js

@@ -0,0 +1,27 @@
+import crypto from 'hypercore-crypto';
+import { RunLog } from '../../lib/run-log/index.js';
+import { flow } from './flow.js';
+import { layout } from './layout.js';
+import { reportModel } from './model.js';
+import { page } from './page.js';
+export class RunReport {
+    static record() {
+        const records = [];
+        let boot = RunLog.createEngineBoot();
+        const onStep = (step) => {
+            boot = RunLog.applyEngineStep(boot, step);
+            records.push(...step.records.map((record) => JSON.parse(JSON.stringify(record))));
+            return Promise.resolve();
+        };
+        return { records, options: { boot, onStep, requestDigestKey: crypto.randomBytes(32) } };
+    }
+    static async render(source, { title } = {}) {
+        const log = Array.isArray(source) ? RunLog.fromRecords([...source]) : source;
+        const model = await reportModel(log);
+        const machine = model.definition
+            ? { run: await layout(model.definition, model), all: await layout(model.definition) }
+            : null;
+        const views = { flow: flow(model), machine };
+        return page(model, views, title ?? `${model.run.agent} on ${model.run.model}`);
+    }
+}

package/dist-lib/plugins/run-report/layout.d.ts

@@ -0,0 +1,39 @@
+import type { MachineDefinitionNode } from '../../lib/machine/index.ts';
+export type NodeKind = 'state' | 'final' | 'history' | 'start' | 'any';
+export interface ChartNode {
+    id: string;
+    label: string;
+    group: string;
+    kind: NodeKind;
+    x: number;
+    y: number;
+    w: number;
+    h: number;
+    stays: string[];
+}
+export interface ChartEdge {
+    key: string;
+    source: string;
+    target: string;
+    event: string;
+    guard: string | null;
+    points: [number, number][];
+    label: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+}
+export interface Chart {
+    w: number;
+    h: number;
+    nodes: ChartNode[];
+    edges: ChartEdge[];
+}
+export interface Ran {
+    visits: Record<string, number[]>;
+    edges: Record<string, number[]>;
+}
+export declare function layout(definition: MachineDefinitionNode, ran?: Ran): Promise<Chart>;
+export declare function eventName(type: string): string;

package/dist-lib/plugins/run-report/layout.js

@@ -0,0 +1,175 @@
+import ELK, {} from 'elkjs/lib/elk.bundled.js';
+const KINDS = { final: 'final', history: 'history' };
+const CHAR = 7.4;
+const LINE = 15;
+const OPTIONS = {
+    'elk.algorithm': 'layered',
+    'elk.direction': 'DOWN',
+    'elk.edgeRouting': 'ORTHOGONAL',
+    'elk.layered.spacing.nodeNodeBetweenLayers': '36',
+    'elk.spacing.nodeNode': '32',
+    'elk.spacing.edgeNode': '16',
+    'elk.spacing.edgeEdge': '10',
+    'elk.layered.spacing.edgeNodeBetweenLayers': '16',
+    'elk.layered.nodePlacement.strategy': 'NETWORK_SIMPLEX',
+    'elk.layered.considerModelOrder.strategy': 'NODES_AND_EDGES',
+    'elk.layered.cycleBreaking.strategy': 'MODEL_ORDER',
+    'elk.edgeLabels.placement': 'CENTER',
+    'elk.edgeLabels.inline': 'true',
+    'elk.layered.edgeLabels.sideSelection': 'SMART_DOWN'
+};
+export async function layout(definition, ran) {
+    const graph = new Graph(definition);
+    graph.walk(definition);
+    const { nodes, edges } = ran ? only(graph, ran) : graph;
+    const laid = await new ELK().layout({
+        id: '__root',
+        layoutOptions: OPTIONS,
+        children: [...nodes.values()].map((node) => ({
+            id: node.id,
+            width: node.w,
+            height: node.h
+        })),
+        edges: edges.map((edge, index) => ({
+            id: String(index),
+            sources: [edge.source],
+            targets: [edge.target],
+            labels: edge.event ? [{ text: edge.key, ...labelSize(edge) }] : []
+        }))
+    });
+    for (const child of laid.children ?? []) {
+        Object.assign(nodes.get(child.id) ?? {}, { x: child.x, y: child.y });
+    }
+    for (const laidEdge of laid.edges ?? []) {
+        route(edges[Number(laidEdge.id)], laidEdge);
+    }
+    return {
+        w: laid.width ?? 0,
+        h: laid.height ?? 0,
+        nodes: [...nodes.values()],
+        edges: edges.filter((edge) => edge.points.length > 0)
+    };
+}
+function route(edge, laid) {
+    const section = laid.sections?.[0];
+    if (!edge || !section) {
+        return;
+    }
+    edge.points = [section.startPoint, ...(section.bendPoints ?? []), section.endPoint].map(({ x, y }) => [x, y]);
+    const label = laid.labels?.[0];
+    edge.label = {
+        x: label?.x ?? 0,
+        y: label?.y ?? 0,
+        w: label?.width ?? 0,
+        h: label?.height ?? 0
+    };
+}
+function only(graph, ran) {
+    const edges = graph.edges.filter((edge) => edge.event ? edge.key in ran.edges : edge.target in ran.visits);
+    const ends = new Set(edges.flatMap((edge) => [edge.source, edge.target]));
+    const nodes = new Map([...graph.nodes].filter(([id, node]) => node.kind === 'start' || node.kind === 'any' ? ends.has(id) : id in ran.visits));
+    return { nodes, edges: edges.filter((edge) => nodes.has(edge.source) && nodes.has(edge.target)) };
+}
+class Graph {
+    nodes = new Map();
+    edges = [];
+    states = new Map();
+    constructor(definition) {
+        const index = (state) => {
+            this.states.set(state.id, state);
+            Object.values(state.states ?? {}).forEach(index);
+        };
+        index(definition);
+        if (definition.initial) {
+            const id = `${definition.id}::start`;
+            this.add({ id, label: '', group: '', kind: 'start', w: 14, h: 14 });
+            this.edges.push(blank(id, this.landing(`${definition.id}.${definition.initial}`)));
+        }
+    }
+    walk(state) {
+        const children = Object.values(state.states ?? {});
+        const group = groupOf(state.id);
+        if (children.length === 0) {
+            const kind = KINDS[state.type] ?? 'state';
+            const label = state.id.split('.').at(-1) ?? state.id;
+            const size = kind === 'history' ? { w: 30, h: 30 } : { w: width(label, group), h: 42 };
+            this.add({ id: state.id, label, group, kind, ...size });
+            this.routes(state, state.id);
+            return;
+        }
+        if (hasRoutes(state)) {
+            const id = `${state.id}::any`;
+            const name = state.id.split('.').at(-1) ?? state.id;
+            const label = state.id.includes('.') ? `any ${name} state` : 'any state';
+            this.add({ id, label, group: '', kind: 'any', w: width(label, ''), h: 34 });
+            this.routes(state, id);
+        }
+        children.forEach((child) => this.walk(child));
+    }
+    add(node) {
+        this.nodes.set(node.id, { ...node, x: 0, y: 0, stays: [] });
+    }
+    routes(state, from) {
+        const lists = [
+            ...Object.entries(state.on ?? {}),
+            ['always', state.always]
+        ];
+        for (const [event, list] of lists) {
+            for (const [index, route] of (list ?? []).entries()) {
+                if (!route.target?.length) {
+                    this.nodes.get(from)?.stays.push(eventName(event));
+                    continue;
+                }
+                for (const target of route.target) {
+                    this.edges.push({
+                        ...blank(from, this.landing(target)),
+                        key: `${state.id}|${event}|${index}`,
+                        event: eventName(event),
+                        guard: route.guard ?? null
+                    });
+                }
+            }
+        }
+    }
+    landing(id) {
+        const state = this.states.get(id);
+        return state?.initial && state.states ? this.landing(`${id}.${state.initial}`) : id;
+    }
+}
+function hasRoutes(state) {
+    return Object.keys(state.on ?? {}).length > 0 || (state.always?.length ?? 0) > 0;
+}
+function groupOf(id) {
+    return id.split('.').slice(2, -1).join('.');
+}
+function width(label, group) {
+    return Math.max(label.length, group.length * 0.85) * CHAR + 64;
+}
+function blank(source, target) {
+    return {
+        key: `${source}|start|0`,
+        source,
+        target,
+        event: '',
+        guard: null,
+        points: [],
+        label: { x: 0, y: 0, w: 0, h: 0 }
+    };
+}
+function labelSize(edge) {
+    const lines = [edge.event, edge.guard ? `[${edge.guard}]` : ''].filter(Boolean);
+    return {
+        width: Math.max(...lines.map((line) => line.length)) * CHAR * 0.85 + 40,
+        height: lines.length * LINE + 6
+    };
+}
+export function eventName(type) {
+    if (type === 'always' || type === '')
+        return 'always';
+    if (type.startsWith('xstate.done.actor.'))
+        return 'done';
+    if (type.startsWith('xstate.error.actor.')) {
+        return 'error';
+    }
+    return type.startsWith('xstate.after.') ? `after ${type.split('.')[2] ?? ''}`.trim() : type;
+}

package/dist-lib/plugins/run-report/meaning.d.ts

@@ -0,0 +1,6 @@
+export type Tone = 'trouble' | 'person' | 'limit' | 'end' | 'error';
+export interface Meaning {
+    says: string;
+    tone?: Tone;
+}
+export declare const MEANINGS: Record<string, Meaning>;

package/dist-lib/plugins/run-report/meaning.js

@@ -0,0 +1,43 @@
+export const MEANINGS = {
+    'completion.active.idle': { says: 'Waiting for the run to start' },
+    'completion.active.preparing': { says: 'Gets the run ready: tools, skills and files' },
+    'completion.active.prepareChecking': { says: 'Checks whether a prepare step is needed first' },
+    'completion.active.prepareRunning': { says: 'Runs the prepare step before the first round' },
+    'completion.active.searching': { says: 'Runs a web search the request asked for outright' },
+    'completion.active.retrieving': { says: 'Looks up what a follow-up question needs' },
+    'completion.active.round': {
+        says: 'Decides what is next: another model round, or wrapping up when time or rounds run out'
+    },
+    'completion.active.generating': { says: 'The model writes its reply' },
+    'completion.active.resolving': {
+        says: 'Reads the reply: tool calls to run, or an answer'
+    },
+    'completion.active.tools.selecting': { says: 'Picks the next queued tool call' },
+    'completion.active.tools.batching': { says: 'Runs reads that need no approval together' },
+    'completion.active.tools.inspecting': {
+        says: 'Checks the call: a known tool, valid arguments, allowed, and whether a person must approve'
+    },
+    'completion.active.tools.rejecting': {
+        says: 'Turns the call down before it runs; the model gets the reason and can try again',
+        tone: 'trouble'
+    },
+    'completion.active.tools.approving': {
+        says: 'Waits for a person to approve or decline the call',
+        tone: 'person'
+    },
+    'completion.active.tools.executing': { says: 'Runs the tool and keeps its result' },
+    'completion.active.answering': { says: 'Streams the answer' },
+    'completion.active.wrapping': {
+        says: 'Time or rounds ran out; asks the model for a final answer',
+        tone: 'limit'
+    },
+    'completion.active.finishing': { says: 'Settles the run and its result' },
+    'completion.active.history': { says: 'Returns to where the run was before it paused' },
+    'completion.pausing': { says: 'Stopping safely so the run can resume later', tone: 'limit' },
+    'completion.paused': { says: 'Paused; the run can resume from here', tone: 'limit' },
+    'completion.resuming': { says: 'Picking the run back up' },
+    'completion.done': { says: 'The run ended with an answer', tone: 'end' },
+    'completion.failed': { says: 'The run ended with an error', tone: 'error' },
+    'completion::any': { says: 'Routes that fire from any state, such as an error or a pause' },
+    'completion.active::any': { says: 'Routes that fire from any active state, such as a pause' }
+};

package/dist-lib/plugins/run-report/model.d.ts

@@ -0,0 +1,86 @@
+import { type MachineDefinitionNode } from '../../lib/machine/index.ts';
+import { type MachineLog, type RunRecord } from '../../lib/run-log/index.ts';
+import { type Story } from './story.ts';
+import { type Candidate } from './why.ts';
+export interface Change {
+    path: string;
+    before: unknown;
+    after: unknown;
+    kept?: number;
+    appended?: unknown[];
+}
+export interface Microstep {
+    eventType: string;
+    from: unknown;
+    to: unknown;
+    candidates: Candidate[];
+}
+export interface TransitionStep {
+    kind: 'transition';
+    index: number;
+    round: number;
+    at: number;
+    from: unknown;
+    to: unknown;
+    events: Record<string, unknown>[];
+    micro: Microstep[];
+    recorded: RunRecordTrace[];
+    changes: Change[];
+    entered: string[];
+    taken: string[];
+    facts: boolean;
+    drift: boolean;
+}
+export interface ReceiptStep {
+    kind: 'receipt';
+    index: number;
+    round: number;
+    id: string;
+    name: string;
+    effect: string;
+    args: unknown;
+    result: unknown;
+    settled: boolean;
+}
+export interface RequestStep {
+    kind: 'request';
+    index: number;
+    round: number;
+    attempt: number;
+    droppedTurns: number;
+    appended: unknown[];
+    tools: string[];
+}
+export type Step = TransitionStep | ReceiptStep | RequestStep;
+type RunRecordTrace = NonNullable<Extract<RunRecord, {
+    type: 'delta';
+}>['trace']>[number];
+export interface RunSummary {
+    runId: string;
+    model: string;
+    agent: string;
+    startedAt: number | null;
+    endedAt: number | null;
+    tools: string[];
+    maxToolRounds: number | null;
+    deadline: number | null;
+    status: string;
+    outcome: string;
+    response: string;
+    rounds: number;
+    tokens: number | null;
+    machine: number | null;
+    replay: string;
+    failure: string;
+}
+export interface ReportModel {
+    run: RunSummary;
+    definition: MachineDefinitionNode | null;
+    steps: Step[];
+    visits: Record<string, number[]>;
+    edges: Record<string, number[]>;
+    story: Story;
+}
+export declare function reportModel(core: MachineLog): Promise<ReportModel>;
+export declare function activeIds(value: unknown, prefix: string): string[];
+export {};