@theoremjs/agents 0.0.0-stage → 0.3.0

package/esm/src/cli/commands/eval.js

@@ -0,0 +1,190 @@
+import { groupSummaries } from '../../evals/breakdown.js';
+import { runSuite } from '../../evals/run.js';
+import { loadSuite, readTraceRecords } from '../../evals/suite.js';
+import { summarizeRun } from '../../evals/summary.js';
+import { jsonlSink } from '../../observability/jsonl.js';
+const GREEN = '\x1b[32m';
+const RED = '\x1b[31m';
+const DIM = '\x1b[2m';
+const YELLOW = '\x1b[33m';
+const RESET = '\x1b[0m';
+function failed(message) {
+    console.error(`\n${RED}Eval Failed${RESET}: ${message}\n`);
+    return false;
+}
+/** Passed over decided cases: a case every trial of which errored is not a failure. */
+function fraction(run) {
+    const decided = run.verdicts.filter((verdict) => verdict.decided);
+    if (decided.length === 0)
+        return 0;
+    return decided.filter((verdict) => verdict.passed).length / decided.length;
+}
+/** `part/whole pct`, with what the pass rate leaves out after it. */
+function rate(part, whole, left, what) {
+    return `${part}/${whole} ${percent(part, whole)}${left > 0 ? ` (+${left} ${what})` : ''}`;
+}
+function blame(report) {
+    return report.results.flatMap((result) => {
+        if (result.passed === true)
+            return [];
+        if (result.errorType) {
+            const why = result.explanation ? ` (${result.explanation})` : '';
+            return [`${result.name}: ${result.errorType}${why}`];
+        }
+        if (result.passed === false)
+            return [`${result.name}: ${result.explanation ?? 'failed'}`];
+        return [];
+    });
+}
+/** The run's cost, never a zero standing in for calls whose provider reported none. */
+function costLine(run) {
+    if (run.unpriced === 0)
+        return `cost $${run.costUsd.toFixed(4)}`;
+    const calls = `${run.unpriced} call${run.unpriced === 1 ? '' : 's'}`;
+    if (run.priced === 0)
+        return `cost not reported (${calls})`;
+    return `cost $${run.costUsd.toFixed(4)}, plus ${calls} whose cost went unreported`;
+}
+/** What a case passing means under the rule, over `k` trials. */
+function ruleName(rule) {
+    if (rule === 'all')
+        return 'pass^k';
+    if (rule === 'any')
+        return 'pass@k';
+    return `>=${rule.atLeast}/k`;
+}
+function percent(part, whole) {
+    return whole === 0 ? '-' : `${Math.round((part / whole) * 100)}%`;
+}
+function spreadText(spread, unit = '') {
+    return spread ? `${spread.median}${unit} / ${spread.p90}${unit}` : '-';
+}
+/** Every cased trial, then each tag's: how often right, how consistently, how long, how many loops. */
+function printGroups(run) {
+    const groups = groupSummaries(run);
+    const rows = groups.map((group) => [
+        group.group,
+        rate(group.casesPassed, group.cases - group.casesUndecided, group.casesUndecided, 'undecided'),
+        rate(group.trialsPassed, group.trials - group.trialsErrored, group.trialsErrored, 'errored'),
+        group.answers
+            ? `${group.answers.accepted}/${group.answers.partial}/${group.answers.wrong}${group.answers.none > 0 ? ` (+${group.answers.none} none)` : ''}`
+            : '-',
+        spreadText(group.durationMs, 'ms'),
+        spreadText(group.modelCalls),
+        spreadText(group.toolCalls),
+        Object.entries(group.stops)
+            .toSorted(([a], [b]) => a.localeCompare(b))
+            .map(([stop, count]) => `${stop} ${count}`)
+            .join(', ') || '-',
+    ]);
+    const header = [
+        'group',
+        `cases ${ruleName(run.passRule)}`,
+        'trials passed',
+        'accepted/partial/wrong',
+        'turn median / p90',
+        'model calls',
+        'tool calls',
+        'stops',
+    ];
+    const widths = header.map((title, column) => Math.max(title.length, ...rows.map((row) => row[column]?.length ?? 0)));
+    const line = (cells) => `  ${cells.map((cell, column) => cell.padEnd(widths[column] ?? 0)).join('  ')}`;
+    console.log('');
+    console.log(line(header));
+    for (const row of rows)
+        console.log(line(row));
+}
+function printTable(run) {
+    const width = Math.max(4, ...run.verdicts.map((verdict) => verdict.case.length));
+    console.log(`\n▶ [EVAL] Suite: ${run.suite} (${run.mode}, ${run.repeat} trials per case)\n`);
+    console.log(`  ${'case'.padEnd(width)}  kind        passed  result`);
+    for (const verdict of run.verdicts) {
+        const mark = verdict.passed
+            ? `${GREEN}pass${RESET}`
+            : verdict.decided
+                ? `${RED}fail${RESET}`
+                : `${YELLOW}undecided${RESET}`;
+        const counts = `${verdict.trialsPassed}/${verdict.trials - verdict.trialsErrored}`;
+        const extra = [
+            verdict.trialsErrored > 0 ? `${verdict.trialsErrored} errored` : '',
+            verdict.trialsUngraded > 0 ? `${verdict.trialsUngraded} ungraded` : '',
+        ]
+            .filter(Boolean)
+            .join(', ');
+        console.log(`  ${verdict.case.padEnd(width)}  ${verdict.kind.padEnd(10)}  ${counts.padEnd(6)}  ${mark}${extra ? ` ${DIM}(${extra})${RESET}` : ''}`);
+    }
+    for (const verdict of run.verdicts) {
+        if (verdict.passed)
+            continue;
+        const reports = run.trials.filter((report) => report.case?.id === verdict.case);
+        for (const report of reports) {
+            const reasons = blame(report);
+            if (reasons.length === 0)
+                continue;
+            console.log(`\n  ${verdict.case} trial ${report.index}:`);
+            for (const reason of reasons)
+                console.log(`    ${DIM}·${RESET} ${reason}`);
+        }
+    }
+    if (run.caseless.length > 0) {
+        console.log(`\n  ${DIM}${run.caseless.length} trace(s) carried no case stamp for this suite; graded without a case, no verdict.${RESET}`);
+    }
+    printGroups(run);
+    const passed = run.verdicts.filter((verdict) => verdict.passed).length;
+    const decided = run.verdicts.filter((verdict) => verdict.decided).length;
+    const undecided = run.verdicts.length - decided;
+    console.log(`\n  ${passed}/${decided} cases passed${undecided > 0 ? ` (+${undecided} undecided)` : ''}; ${costLine(run)}`);
+    if (run.stopped)
+        console.log(`  ${YELLOW}stopped on ${run.stopped}${RESET}`);
+    for (const warning of run.warnings)
+        console.log(`  ${YELLOW}warning${RESET}: ${warning}`);
+    console.log('');
+}
+function printJson(run) {
+    console.log(JSON.stringify(summarizeRun(run), null, 2));
+}
+/**
+ * Returns whether the run met the threshold; a run stopped on budget never does. The CLI never
+ * creates a provider or reads a key: the host passes them or the suite module exports them.
+ */
+export async function evalCommand(options, host = {}) {
+    const { provider, judgeProvider, judgeDecision, media } = host;
+    const threshold = options.threshold ?? 1;
+    if (!(threshold >= 0 && threshold <= 1))
+        return failed('--threshold is a fraction from 0 to 1.');
+    if (options.trials !== undefined && !(Number.isInteger(options.trials) && options.trials > 0)) {
+        return failed('--trials is a whole number of trials per case, 1 or more.');
+    }
+    if (options.concurrency !== undefined &&
+        !(Number.isInteger(options.concurrency) && options.concurrency > 0)) {
+        return failed('--concurrency is a whole number of trials in flight, 1 or more.');
+    }
+    let run;
+    try {
+        const loaded = await loadSuite(options.suite);
+        const recorded = options.recorded ? await readTraceRecords(options.recorded) : undefined;
+        const live = provider ?? loaded.provider;
+        if (!recorded && !live) {
+            return failed('Theorem CLI does not create providers or read keys. Export `provider` from the suite module, or pass --recorded <traces>.');
+        }
+        run = await runSuite(loaded, {
+            ...(recorded ? { recorded } : {}),
+            ...(live ? { provider: live } : {}),
+            ...(judgeProvider ? { judgeProvider } : {}),
+            ...(judgeDecision ? { judgeDecision } : {}),
+            ...(media ? { media } : {}),
+            ...(options.trials !== undefined ? { repeat: options.trials } : {}),
+            ...(options.concurrency !== undefined ? { concurrency: options.concurrency } : {}),
+            ...(options.maxCostUsd !== undefined ? { maxCostUsd: options.maxCostUsd } : {}),
+            ...(options.traceDir?.trim() ? { sink: jsonlSink(options.traceDir.trim()) } : {}),
+        });
+    }
+    catch (error) {
+        return failed(error instanceof Error ? error.message : String(error));
+    }
+    if (options.json)
+        printJson(run);
+    else
+        printTable(run);
+    return run.stopped === undefined && fraction(run) >= threshold;
+}

package/esm/src/cli/commands/fuzz-canary.d.ts

@@ -0,0 +1,15 @@
+import { type CanaryEgressAttack } from '../../guardrails/corpus/canary-egress-attacks.js';
+export interface ChannelResult {
+    attack: CanaryEgressAttack;
+    channel: string;
+    blocked: boolean;
+    bypassed: boolean;
+    falseAlarm: boolean;
+    clientWire: string;
+}
+/** One result per attack per channel (runTurn stream, Live batch). */
+export declare function runCanaryFuzz(canary?: string): Promise<ChannelResult[]>;
+/** Run adversarial canary fuzz; returns true when no bypasses or false alarms. */
+export declare function fuzzCanaryCommand(options?: {
+    canary?: string;
+}): Promise<boolean>;

package/esm/src/cli/commands/fuzz-canary.js

@@ -0,0 +1,202 @@
+// A bypass is judged from the attack's own encoded canary, not from the detector under test.
+import { bindCanary, isStreamedCanaryEvent, mintCanary, scanTextForCanaryLeak, } from '../../guardrails/canary.js';
+import { buildCanaryEgressAttacks, FIXED_CANARY, FUZZ_SYSTEM, } from '../../guardrails/corpus/canary-egress-attacks.js';
+import { givenUrlSets } from '../../guardrails/egress-urls.js';
+import { createLiveOutboundGateSession, finalizeLiveOutboundTurn, processLiveOutboundBatch, } from '../../guardrails/live-outbound-gate.js';
+import { scanTextOf } from '../../guardrails/serialize.js';
+import { clearProfiles, getProfile, registerProfile, resolveTurn, } from '../../kernel/default-scope.js';
+import { yieldProviderEvents, } from '../../kernel/engine/runner/stream.js';
+import { providerCompleteRequest } from '../../kernel/registry/provider-request.js';
+import { defaultKernelScope } from '../../kernel/scope.js';
+const FUZZ_PROFILE_ID = '__fuzz_canary__';
+function registerFuzzCanaryProfile() {
+    registerProfile({
+        type: 'text',
+        id: FUZZ_PROFILE_ID,
+        identity: { handle: 'fuzz-canary', system: 'Canary fuzz profile.' },
+        models: {
+            'fuzz-model': {
+                protocol: 'openAi',
+                provider: 'openrouter',
+                apiId: 'fuzz-model',
+                key: 'fuzz',
+                efforts: { normal: 'none' },
+                summaries: false,
+                maxOutputTokens: 4096,
+                temperature: 0,
+                builtInTools: [],
+            },
+        },
+        defaultModel: 'fuzz-model',
+        tools: { allow: [] },
+        inputs: { text: true },
+        guardrails: { canary: true },
+    });
+}
+function resolveFuzzGeneration(canary) {
+    const { generation } = resolveTurn({
+        profile: FUZZ_PROFILE_ID,
+        input: { text: 'fuzz' },
+    });
+    return { ...generation, canary };
+}
+async function collectEvents(gen) {
+    const out = [];
+    for await (const event of gen) {
+        out.push(event);
+    }
+    return out;
+}
+function clientVisible(events) {
+    return events.filter((event) => event.type !== 'error');
+}
+function clientVisibleWire(events) {
+    return JSON.stringify(clientVisible(events));
+}
+/**
+ * Whether the attack's encoded canary reached the client: in the reply as the
+ * client assembles it across chunks and turns, or in any other event.
+ */
+function leakReachedClient(attack, events) {
+    if (!attack.shouldBlock) {
+        return false;
+    }
+    const visible = clientVisible(events);
+    const reply = visible
+        .filter(isStreamedCanaryEvent)
+        .map((event) => event.text ?? '')
+        .join('');
+    return (reply.includes(attack.leak) ||
+        visible.some((event) => !isStreamedCanaryEvent(event) && scanTextOf(event).includes(attack.leak)));
+}
+function channelResult(attack, channel, blocked, events) {
+    const bypassed = leakReachedClient(attack, events);
+    return {
+        attack,
+        channel,
+        blocked,
+        bypassed,
+        falseAlarm: !attack.shouldBlock && blocked,
+        clientWire: clientVisibleWire(events),
+    };
+}
+async function* replay(turn) {
+    await Promise.resolve();
+    yield* turn;
+}
+async function runStreamChannel(attack, generation, canary) {
+    const events = [];
+    // Every provider call of one turn shares its canary: each attack turn is a step.
+    const control = { withholdVisible: false };
+    for (const turn of attack.turns) {
+        const streamed = await collectEvents(yieldProviderEvents({
+            profile: getProfile(FUZZ_PROFILE_ID),
+            generation,
+            request: providerCompleteRequest(defaultKernelScope.tools, generation, bindCanary(FUZZ_SYSTEM, canary)),
+            privateSystem: [bindCanary(FUZZ_SYSTEM, canary)],
+            provider: { complete: () => replay(turn) },
+            // The fuzz reads what reaches the client, not the trace.
+            call: { tap: () => { }, observe: () => { } },
+            control,
+            givenUrls: givenUrlSets(),
+        }));
+        // The runner reads the call's `done`; the client never receives it.
+        const turnEvents = streamed.flatMap((event) => (event.type === 'done' ? [] : [event]));
+        events.push(...turnEvents);
+        if (turnEvents.some((event) => event.type === 'error')) {
+            return channelResult(attack, 'runTurn.stream', true, events);
+        }
+    }
+    return channelResult(attack, 'runTurn.stream', false, events);
+}
+/** One Live session; each turn is a cycle the provider finishes before the next. */
+async function runLiveBatchChannel(attack, canary) {
+    const session = createLiveOutboundGateSession(getProfile(FUZZ_PROFILE_ID), canary, [
+        bindCanary(FUZZ_SYSTEM, canary),
+    ]);
+    const events = [];
+    for (const turn of attack.turns) {
+        for (const result of [
+            await processLiveOutboundBatch(session, turn),
+            await finalizeLiveOutboundTurn(session),
+        ]) {
+            if (result.action === 'withhold') {
+                return channelResult(attack, 'live.batch', true, events);
+            }
+            if (result.action === 'emit') {
+                events.push(...result.events);
+            }
+        }
+    }
+    return channelResult(attack, 'live.batch', false, events);
+}
+function truncate(s, max) {
+    if (s.length <= max)
+        return s;
+    return `${s.slice(0, max - 3)}...`;
+}
+function printResults(results) {
+    const bypassed = results.filter((r) => r.bypassed);
+    const falseAlarms = results.filter((r) => r.falseAlarm);
+    const caught = results.filter((r) => r.attack.shouldBlock && r.blocked && !r.bypassed);
+    console.log(`\n${'═'.repeat(72)}`);
+    console.log(`  TOTAL: ${results.length} | CAUGHT: ${caught.length} | BYPASSED: ${bypassed.length} | FALSE ALARM: ${falseAlarms.length}`);
+    console.log(`${'═'.repeat(72)}`);
+    if (bypassed.length > 0) {
+        console.log('\n\x1b[31mBYPASSED (canary reached client-visible output):\x1b[0m\n');
+        for (const r of bypassed) {
+            console.log(`  ✗ ${r.attack.category}/${r.attack.name} [${r.channel}]`);
+            console.log(`    \x1b[2m${truncate(r.clientWire, 90)}\x1b[0m`);
+        }
+    }
+    if (falseAlarms.length > 0) {
+        console.log('\n\x1b[33mFALSE ALARMS (benign payload blocked):\x1b[0m\n');
+        for (const r of falseAlarms) {
+            console.log(`  ⚠ ${r.attack.category}/${r.attack.name} [${r.channel}]`);
+        }
+    }
+    if (bypassed.length === 0 && falseAlarms.length === 0) {
+        console.log('\n\x1b[32mAll leak attempts blocked; benign controls passed.\x1b[0m\n');
+    }
+    return bypassed.length === 0 && falseAlarms.length === 0;
+}
+/** One result per attack per channel (runTurn stream, Live batch). */
+export async function runCanaryFuzz(canary = FIXED_CANARY) {
+    clearProfiles();
+    registerFuzzCanaryProfile();
+    try {
+        const generation = resolveFuzzGeneration(canary);
+        const results = [];
+        for (const attack of buildCanaryEgressAttacks(canary)) {
+            results.push(await runStreamChannel(attack, generation, canary));
+            results.push(await runLiveBatchChannel(attack, canary));
+        }
+        return results;
+    }
+    finally {
+        clearProfiles();
+    }
+}
+/** Run adversarial canary fuzz; returns true when no bypasses or false alarms. */
+export async function fuzzCanaryCommand(options) {
+    console.log('\n🔐 Theorem Canary Egress Adversarial Fuzzer\n');
+    const canary = options?.canary ?? FIXED_CANARY;
+    if (!/^[0-9a-f]{32}$/.test(canary)) {
+        console.error(`Invalid canary shape (expected 32 hex): ${canary}`);
+        return false;
+    }
+    const minted = mintCanary();
+    if (!/^[0-9a-f]{32}$/.test(minted)) {
+        console.error(`mintCanary produced unexpected shape: ${minted}`);
+        return false;
+    }
+    if (!scanTextForCanaryLeak(canary, canary)) {
+        console.error('scanTextForCanaryLeak failed to detect literal canary');
+        return false;
+    }
+    const results = await runCanaryFuzz(canary);
+    console.log(`Canary: ${canary.slice(0, 12)}… (${results.length / 2} attacks × 2 channels)`);
+    const ok = printResults(results);
+    console.log('');
+    return ok;
+}

package/esm/src/cli/commands/fuzz-guardrails.d.ts

@@ -0,0 +1,2 @@
+/** Run inbound fuzz; returns true when no expected catches are missed. */
+export declare function fuzzGuardrailsCommand(): boolean;

package/esm/src/cli/commands/fuzz-guardrails.js

@@ -0,0 +1,5 @@
+import { runInboundGuardrailFuzz } from '../../guardrails/corpus/fuzz-inbound.js';
+/** Run inbound fuzz; returns true when no expected catches are missed. */
+export function fuzzGuardrailsCommand() {
+    return runInboundGuardrailFuzz();
+}

package/esm/src/cli/commands/profile.d.ts

@@ -0,0 +1,2 @@
+export declare function listProfilesCommand(): void;
+export declare function showProfileCommand(profileId: string): void;

package/esm/src/cli/commands/profile.js

@@ -0,0 +1,80 @@
+import { getProfile, listProfiles } from '../../kernel/default-scope.js';
+import { profileToolAllow } from '../../kernel/tools/resolve.js';
+function formatProfileInputs(p) {
+    if (p.type === 'speech') {
+        return 'text (speech)';
+    }
+    if (p.type === 'live') {
+        return 'audio/video (live)';
+    }
+    const inputs = [];
+    const { inputs: spec } = p;
+    if (!spec) {
+        return 'none';
+    }
+    if (spec.text !== false)
+        inputs.push('text');
+    if (p.type !== 'image' && p.inputs?.voice)
+        inputs.push('voice');
+    if (spec.attachments) {
+        inputs.push(`attachments [${spec.attachments.accept?.join(', ')}]`);
+    }
+    return inputs.join(' | ') || 'none';
+}
+function formatProfileTools(p) {
+    return profileToolAllow(p).join(', ') || 'none';
+}
+function printProfileCard(p) {
+    const tools = formatProfileTools(p);
+    if (p.type === 'host') {
+        console.log(` • Profile: ${p.id.padEnd(16)} [host]`);
+        console.log(`   - Tools:      ${tools}`);
+        console.log('-'.repeat(70));
+        return;
+    }
+    if (p.type === 'decision') {
+        console.log(` • Profile: ${p.id.padEnd(16)} (handle: ${p.identity.handle}) [decision]`);
+        console.log(`   - Model:      ${Object.keys(p.models).join(', ')}`);
+        console.log(`   - Inputs:     JSON state`);
+        console.log(`   - Contract:   ${p.decision.contract}`);
+        console.log(`   - Key Slot: ${p.key ?? '(unset)'}`);
+        console.log('-'.repeat(70));
+        return;
+    }
+    const models = Object.keys(p.models).join(', ') || 'default';
+    const structured = p.type === 'live' || p.type === 'speech' ? undefined : p.outputs?.structured;
+    const structuredLabel = typeof structured === 'string' ? structured : structured ? 'custom' : 'none';
+    console.log(` • Profile: ${p.id.padEnd(16)} (handle: ${p.identity.handle}) [${p.type}]`);
+    console.log(`   - Models:     ${models}`);
+    console.log(`   - Inputs:     ${formatProfileInputs(p)}`);
+    console.log(`   - Tools:      ${tools}`);
+    console.log(`   - Structured: ${structuredLabel}`);
+    console.log(`   - Key Slot: ${p.key ?? '(unset)'}`);
+    console.log('-'.repeat(70));
+}
+export function listProfilesCommand() {
+    const profiles = listProfiles();
+    console.log('\n Registered Theorem Profiles:');
+    console.log('='.repeat(70));
+    if (profiles.length === 0) {
+        console.log('  (No profiles registered in runtime)');
+        console.log(`${'='.repeat(70)}\n`);
+        return;
+    }
+    for (const p of profiles) {
+        printProfileCard(p);
+    }
+    console.log(`Total: ${profiles.length} profiles registered.\n`);
+}
+export function showProfileCommand(profileId) {
+    try {
+        const profile = getProfile(profileId);
+        console.log(`\n Profile Details: ${profileId}`);
+        console.log('='.repeat(70));
+        console.log(JSON.stringify(profile, null, 2));
+        console.log(`${'='.repeat(70)}\n`);
+    }
+    catch (err) {
+        console.error(`\n Error: ${err instanceof Error ? err.message : String(err)}\n`);
+    }
+}

package/esm/src/cli/commands/run.d.ts

@@ -0,0 +1,13 @@
+import type { ModelProvider } from '../../kernel/types.js';
+export interface RunOptions {
+    profile: string;
+    prompt?: string;
+    mode?: string;
+    search?: boolean;
+    map?: boolean;
+    provider?: ModelProvider;
+    verbose?: boolean;
+    trace?: boolean;
+    traceDir?: string;
+}
+export declare function runCommand(options: RunOptions): Promise<void>;

package/esm/src/cli/commands/run.js

@@ -0,0 +1,45 @@
+import { getProfile, runTurn } from '../../kernel/default-scope.js';
+import { requireModelProfile } from '../../kernel/registry/resolve.js';
+import { createCliTraceCapture, printRunEvent, printTraceRecord } from '../event-log.js';
+export async function runCommand(options) {
+    requireModelProfile(getProfile(options.profile), 'agents run');
+    const provider = options.provider;
+    if (!provider) {
+        console.error('\n\x1b[31mExecution Failed\x1b[0m: Theorem CLI does not create providers or read keys. Run turns from a host app with an explicit ModelProvider.\n');
+        return;
+    }
+    const prompt = options.prompt || 'Hello! Please introduce your capabilities.';
+    if (options.search || options.map) {
+        const profile = requireModelProfile(getProfile(options.profile), 'agents run');
+        const selected = options.mode ?? profile.defaultModel;
+        const builtins = new Set(profile.models[selected]?.builtInTools ?? []);
+        if (options.search && !builtins.has('googleSearch')) {
+            console.error('\n\x1b[31mExecution Failed\x1b[0m: --search requires googleSearch on models.*.builtInTools for the selected model.\n');
+            return;
+        }
+        if (options.map && !builtins.has('googleMaps')) {
+            console.error('\n\x1b[31mExecution Failed\x1b[0m: --map requires googleMaps on models.*.builtInTools for the selected model.\n');
+            return;
+        }
+    }
+    const req = {
+        profile: options.profile,
+        model: options.mode,
+        input: { text: prompt },
+    };
+    console.log(`\n▶ [RUNNING TURN] Profile: ${options.profile} (${options.mode ?? 'default'})`);
+    console.log(`Prompt: "${prompt}"\n`);
+    const traceCapture = options.trace ? createCliTraceCapture(options.traceDir) : undefined;
+    try {
+        for await (const event of runTurn(req, provider, traceCapture?.sink)) {
+            printRunEvent(event, { verbose: options.verbose });
+        }
+        console.log('\n');
+        if (options.trace) {
+            printTraceRecord(traceCapture?.records.at(-1), options.verbose === true);
+        }
+    }
+    catch (err) {
+        console.error(`\n\x1b[31mExecution Failed\x1b[0m: ${err instanceof Error ? err.message : String(err)}\n`);
+    }
+}

package/esm/src/cli/commands/test.d.ts

@@ -0,0 +1,25 @@
+import type { ModelProvider, TurnRequest, TurnTokens } from '../../kernel/types.js';
+import { type MatrixOptions } from '../matrix/synthesizer.js';
+export interface TestRunResult {
+    passed: boolean;
+    name: string;
+    durationSec: number;
+    /** The turn's usage, summed over every model call (`sumTokens`). */
+    tokens?: TurnTokens;
+    error?: string;
+}
+interface CliTestOptions {
+    verbose?: boolean;
+    trace?: boolean;
+    traceDir?: string;
+}
+export declare function executeSingleTest(req: TurnRequest, testName: string, provider?: ModelProvider, cliOptions?: CliTestOptions): Promise<TestRunResult>;
+export declare function testProfileCommand(profileId?: string, options?: MatrixOptions & {
+    all?: boolean;
+    matrix?: boolean;
+    provider?: ModelProvider;
+    verbose?: boolean;
+    trace?: boolean;
+    traceDir?: string;
+}): Promise<boolean>;
+export {};