@theoremjs/agents 0.0.0-stage → 0.3.0

package/esm/src/cli/commands/bench.js

@@ -0,0 +1,458 @@
+// TTFE: time to first event (profile resolve + provider setup). TTFT: time to first text delta.
+// Overhead: total wall-clock delta vs. raw provider consumption.
+import { canaryNote, eventHasCanary, mintCanary } from '../../guardrails/canary.js';
+import { sanitizeTurnRequest } from '../../guardrails/sanitize.js';
+import { clearProfiles, getProfile, registerProfile, resolveTurn, runTurn, } from '../../kernel/default-scope.js';
+import { startCallUsage } from '../../kernel/engine/runner/usage.js';
+import { startCallTrace } from '../../kernel/engine/turn-trace.js';
+import { pickSystemRole } from '../../kernel/registry/system-role.js';
+import { bindSystem } from '../../kernel/system-parts.js';
+import { resolveObservabilityPolicy } from '../../observability/resolve-policy.js';
+import { buildRecord } from '../../observability/trace-record.js';
+import { startTrace } from '../../observability/trace-span.js';
+const DEFAULT_CHUNKS = 200;
+const DEFAULT_ITERATIONS = 50;
+const DEFAULT_WARMUP = 5;
+const MS_PER_SEC = 1000;
+const BENCH_PROFILE_ID = '__bench__';
+function registerBenchProfile() {
+    registerProfile({
+        type: 'text',
+        id: BENCH_PROFILE_ID,
+        identity: {
+            handle: 'bench',
+            system: 'You are a benchmark stub.',
+        },
+        models: {
+            'bench-model': {
+                protocol: 'openAi',
+                provider: 'openrouter',
+                apiId: 'bench-model',
+                efforts: { normal: 'none' },
+                summaries: false,
+                maxOutputTokens: 4096,
+                temperature: 0,
+                builtInTools: [],
+            },
+        },
+        defaultModel: 'bench-model',
+        tools: { allow: [] },
+        inputs: { text: true },
+        guardrails: {
+            canary: true,
+            sanitizeInput: true,
+            redactSensitive: true,
+        },
+    });
+}
+function generateChunks(count) {
+    const events = [];
+    for (let i = 0; i < count; i++) {
+        events.push({ type: 'text', text: `token_${i} ` });
+    }
+    events.push({
+        type: 'tokens',
+        tokens: { input: 10, output: count, total: 10 + count },
+    });
+    events.push({ type: 'done', stop: { kind: 'completed' } });
+    return events;
+}
+function createMockProvider(chunks) {
+    return {
+        async *complete(_req) {
+            for (const chunk of chunks) {
+                yield chunk;
+            }
+        },
+    };
+}
+function buildBenchRequest() {
+    return {
+        profile: BENCH_PROFILE_ID,
+        input: { text: 'Benchmark prompt.' },
+    };
+}
+/** What a bare consumer sends a provider: the bench prompt, no kernel shaping. */
+function benchProviderRequest(text) {
+    return {
+        model: 'bench-model',
+        apiId: 'bench-model',
+        thinking: 'none',
+        summaries: undefined,
+        maxOutputTokens: 4096,
+        temperature: 0,
+        builtins: [],
+        system: '',
+        input: [{ type: 'text', text }],
+        structured: null,
+        image: null,
+    };
+}
+async function measureRawProvider(provider) {
+    const req = buildBenchRequest();
+    const start = performance.now();
+    let firstEvent = 0;
+    let firstText = 0;
+    let textEvents = 0;
+    let gotFirst = false;
+    let gotFirstText = false;
+    const fakeReq = benchProviderRequest(req.input?.text ?? '');
+    for await (const event of provider.complete(fakeReq)) {
+        if (!gotFirst) {
+            firstEvent = performance.now() - start;
+            gotFirst = true;
+        }
+        if (!gotFirstText && event.type === 'text') {
+            firstText = performance.now() - start;
+            gotFirstText = true;
+        }
+        if (event.type === 'text') {
+            textEvents++;
+        }
+    }
+    return {
+        ttfe: firstEvent,
+        ttft: firstText,
+        totalMs: performance.now() - start,
+        textEvents,
+    };
+}
+async function measureKernelPipeline(provider) {
+    const req = buildBenchRequest();
+    const start = performance.now();
+    let firstEvent = 0;
+    let firstText = 0;
+    let textEvents = 0;
+    let gotFirst = false;
+    let gotFirstText = false;
+    for await (const event of runTurn(req, provider)) {
+        if (!gotFirst) {
+            firstEvent = performance.now() - start;
+            gotFirst = true;
+        }
+        if (!gotFirstText && event.type === 'text') {
+            firstText = performance.now() - start;
+            gotFirstText = true;
+        }
+        if (event.type === 'text') {
+            textEvents++;
+        }
+    }
+    return {
+        ttfe: firstEvent,
+        ttft: firstText,
+        totalMs: performance.now() - start,
+        textEvents,
+    };
+}
+function percentile(sorted, p) {
+    const idx = Math.ceil((p / 100) * sorted.length) - 1;
+    return sorted[Math.max(0, idx)];
+}
+function aggregate(results) {
+    const ttfe = results.map((r) => r.ttfe).sort((a, b) => a - b);
+    const ttft = results.map((r) => r.ttft).sort((a, b) => a - b);
+    const total = results.map((r) => r.totalMs).sort((a, b) => a - b);
+    const tps = results.map((r) => (r.textEvents / r.totalMs) * MS_PER_SEC).sort((a, b) => a - b);
+    const mean = (arr) => arr.reduce((a, b) => a + b, 0) / arr.length;
+    return {
+        ttfeMs: {
+            median: percentile(ttfe, 50),
+            p95: percentile(ttfe, 95),
+            mean: mean(ttfe),
+        },
+        ttftMs: {
+            median: percentile(ttft, 50),
+            p95: percentile(ttft, 95),
+            mean: mean(ttft),
+        },
+        tokensPerSec: {
+            median: percentile(tps, 50),
+            p95: percentile(tps, 5), // lower is worse for throughput
+            mean: mean(tps),
+        },
+        totalMs: {
+            median: percentile(total, 50),
+            p95: percentile(total, 95),
+            mean: mean(total),
+        },
+    };
+}
+function fmtMs(ms) {
+    if (ms < 1) {
+        return `${(ms * MS_PER_SEC).toFixed(0)}µs`;
+    }
+    return `${ms.toFixed(2)}ms`;
+}
+function fmtRate(tps) {
+    if (tps >= MS_PER_SEC * MS_PER_SEC) {
+        return `${(tps / (MS_PER_SEC * MS_PER_SEC)).toFixed(1)}M/s`;
+    }
+    if (tps >= MS_PER_SEC) {
+        return `${(tps / MS_PER_SEC).toFixed(1)}K/s`;
+    }
+    return `${tps.toFixed(0)}/s`;
+}
+function printMetricRow(label, raw, fmt) {
+    console.log(`  ${label.padEnd(14)} median ${fmt(raw.median).padStart(10)}   p95 ${fmt(raw.p95).padStart(10)}   mean ${fmt(raw.mean).padStart(10)}`);
+}
+function printResults(label, metrics) {
+    console.log(`\n${label}`);
+    console.log('─'.repeat(72));
+    printMetricRow('TTFE', metrics.ttfeMs, fmtMs);
+    printMetricRow('TTFT', metrics.ttftMs, fmtMs);
+    printMetricRow('Throughput', metrics.tokensPerSec, fmtRate);
+    printMetricRow('Total', metrics.totalMs, fmtMs);
+}
+function printOverhead(raw, kernel) {
+    console.log('\nOverhead (kernel − raw)');
+    console.log('─'.repeat(72));
+    const ttfeDelta = kernel.ttfeMs.median - raw.ttfeMs.median;
+    const ttftDelta = kernel.ttftMs.median - raw.ttftMs.median;
+    const totalDelta = kernel.totalMs.median - raw.totalMs.median;
+    const tpsDelta = kernel.tokensPerSec.median - raw.tokensPerSec.median;
+    console.log(`  TTFE           ${ttfeDelta >= 0 ? '+' : ''}${fmtMs(ttfeDelta)}`);
+    console.log(`  TTFT           ${ttftDelta >= 0 ? '+' : ''}${fmtMs(ttftDelta)}`);
+    console.log(`  Throughput     ${tpsDelta >= 0 ? '+' : ''}${fmtRate(tpsDelta)}`);
+    console.log(`  Total          ${totalDelta >= 0 ? '+' : ''}${fmtMs(totalDelta)}`);
+    if (raw.totalMs.median > 0) {
+        const pct = ((totalDelta / raw.totalMs.median) * 100).toFixed(1);
+        console.log(`  Relative       ${totalDelta >= 0 ? '+' : ''}${pct}%`);
+    }
+}
+function measureSetupPhases() {
+    const req = buildBenchRequest();
+    const t0 = performance.now();
+    const safe = sanitizeTurnRequest(req, getProfile(req.profile));
+    const t1 = performance.now();
+    const { profile, generation } = resolveTurn(safe);
+    const t2 = performance.now();
+    pickSystemRole(profile, safe.input?.role);
+    const t3 = performance.now();
+    bindSystem(generation.resolvedSystem, [canaryNote(generation.canary)]);
+    const t4 = performance.now();
+    return {
+        sanitizeMs: t1 - t0,
+        resolveMs: t2 - t1,
+        systemMs: t3 - t2,
+        canaryMs: t4 - t3,
+        totalSetupMs: t4 - t0,
+    };
+}
+function printPhaseBreakdown(iterations) {
+    const results = [];
+    for (let i = 0; i < iterations; i++) {
+        results.push(measureSetupPhases());
+    }
+    const median = (arr) => {
+        const s = arr.slice().sort((a, b) => a - b);
+        return percentile(s, 50);
+    };
+    const sanitize = median(results.map((r) => r.sanitizeMs));
+    const resolve = median(results.map((r) => r.resolveMs));
+    const system = median(results.map((r) => r.systemMs));
+    const canary = median(results.map((r) => r.canaryMs));
+    const total = median(results.map((r) => r.totalSetupMs));
+    console.log('\nSetup Phase Breakdown (median)');
+    console.log('─'.repeat(72));
+    console.log(`  sanitizeTurnRequest    ${fmtMs(sanitize).padStart(10)}`);
+    console.log(`  resolveTurn            ${fmtMs(resolve).padStart(10)}`);
+    console.log(`  pickSystemRole         ${fmtMs(system).padStart(10)}`);
+    console.log(`  bindSystem             ${fmtMs(canary).padStart(10)}`);
+    console.log(`  ─────────────────────────────`);
+    console.log(`  Total setup            ${fmtMs(total).padStart(10)}`);
+}
+function microCanaryCheck(chunks, iterations) {
+    const canary = mintCanary();
+    const total = chunks.length * iterations;
+    const start = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        for (const chunk of chunks) {
+            eventHasCanary(chunk, canary);
+        }
+    }
+    const elapsed = performance.now() - start;
+    return {
+        label: 'eventHasCanary',
+        totalMs: elapsed,
+        per: 'event',
+        perUnitNs: (elapsed / total) * 1e6,
+    };
+}
+function microArrayPush(chunks, iterations) {
+    const total = chunks.length * iterations;
+    const start = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        const seen = [];
+        const allEmitted = [];
+        const attempt = [];
+        for (const chunk of chunks) {
+            seen.push(chunk);
+            allEmitted.push(chunk);
+            attempt.push(chunk);
+        }
+    }
+    const elapsed = performance.now() - start;
+    return {
+        label: 'Array.push ×3',
+        totalMs: elapsed,
+        per: 'event',
+        perUnitNs: (elapsed / total) * 1e6,
+    };
+}
+async function microAsyncGenOverhead(chunks, iterations) {
+    async function* layer1(events) {
+        for (const e of events)
+            yield e;
+    }
+    async function* layer2(events) {
+        for await (const e of layer1(events))
+            yield e;
+    }
+    async function* layer3(events) {
+        for await (const e of layer2(events))
+            yield e;
+    }
+    async function* layer4(events) {
+        for await (const e of layer3(events))
+            yield e;
+    }
+    const total = chunks.length * iterations;
+    const start = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        for await (const _e of layer4(chunks)) {
+            // consume
+        }
+    }
+    const elapsed = performance.now() - start;
+    return {
+        label: 'AsyncGen ×4 layers',
+        totalMs: elapsed,
+        per: 'event',
+        perUnitNs: (elapsed / total) * 1e6,
+    };
+}
+function microAbortCheck(chunks, iterations) {
+    const controller = new AbortController();
+    const { signal } = controller;
+    const total = chunks.length * iterations;
+    const start = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        for (const _chunk of chunks) {
+            if (signal.aborted)
+                throw signal.reason;
+        }
+    }
+    const elapsed = performance.now() - start;
+    return {
+        label: 'signal.aborted check',
+        totalMs: elapsed,
+        per: 'event',
+        perUnitNs: (elapsed / total) * 1e6,
+    };
+}
+/** Record one model call over `chunks`, then build its trace record, as a turn does. */
+async function microTraceRecord(chunks, iterations) {
+    const text = buildBenchRequest().input?.text ?? '';
+    const req = benchProviderRequest(text);
+    const policy = resolveObservabilityPolicy(undefined);
+    const start = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        const tree = startTrace(`invoke_agent ${BENCH_PROFILE_ID}`);
+        const call = startCallTrace((name, options) => tree.root.child(name, options), {
+            req,
+            usage: startCallUsage('', { history: [], input: req.input }),
+            binding: undefined,
+            transport: 'interactions',
+            step: 0,
+            attempt: 0,
+        });
+        for (const chunk of chunks) {
+            call.observe(chunk);
+        }
+        call.end({ stop: { kind: 'completed' } });
+        tree.root.end();
+        await buildRecord({ spans: tree.collect(), policy });
+    }
+    const elapsed = performance.now() - start;
+    return {
+        label: 'record call + buildRecord (trace)',
+        totalMs: elapsed,
+        per: 'turn',
+        perUnitNs: (elapsed / iterations) * 1e6,
+    };
+}
+function microSanitizeScaling() {
+    const sizes = [50, 200, 1000, 5000];
+    console.log('\nSanitize Scaling (input text length)');
+    console.log('─'.repeat(72));
+    for (const size of sizes) {
+        const text = 'A'.repeat(size);
+        const req = {
+            profile: BENCH_PROFILE_ID,
+            input: { text },
+        };
+        const runs = 500;
+        const start = performance.now();
+        for (let i = 0; i < runs; i++) {
+            sanitizeTurnRequest(req, getProfile(req.profile));
+        }
+        const elapsed = performance.now() - start;
+        const perCall = elapsed / runs;
+        console.log(`  ${String(size).padStart(5)} chars          ${fmtMs(perCall).padStart(10)}/call`);
+    }
+}
+function fmtNs(ns) {
+    if (ns < 1000) {
+        return `${ns.toFixed(0)}ns`;
+    }
+    return `${(ns / 1000).toFixed(1)}µs`;
+}
+async function printMicroBenchmarks(chunks, iterations) {
+    const results = [];
+    results.push(microCanaryCheck(chunks, iterations));
+    results.push(microArrayPush(chunks, iterations));
+    results.push(await microAsyncGenOverhead(chunks, iterations));
+    results.push(microAbortCheck(chunks, iterations));
+    results.push(await microTraceRecord(chunks, Math.min(iterations, 20)));
+    console.log('\nPer-Event Micro-Benchmarks');
+    console.log('─'.repeat(72));
+    for (const r of results) {
+        const perUnit = r.per === 'turn'
+            ? `${fmtMs(r.perUnitNs / 1e6).padStart(10)}/turn`
+            : `${fmtNs(r.perUnitNs).padStart(8)}/event`;
+        console.log(`  ${r.label.padEnd(24)} ${perUnit}    (${fmtMs(r.totalMs)} total)`);
+    }
+    microSanitizeScaling();
+}
+export async function benchCommand(options = {}) {
+    const chunkCount = options.chunks ?? DEFAULT_CHUNKS;
+    const iterations = options.iterations ?? DEFAULT_ITERATIONS;
+    const warmup = options.warmup ?? DEFAULT_WARMUP;
+    console.log(`\n⏱  Theorem Kernel Benchmark`);
+    console.log(`   ${chunkCount} chunks × ${iterations} iterations (${warmup} warmup)\n`);
+    const chunks = generateChunks(chunkCount);
+    const provider = createMockProvider(chunks);
+    registerBenchProfile();
+    for (let i = 0; i < warmup; i++) {
+        await measureRawProvider(provider);
+        await measureKernelPipeline(provider);
+    }
+    const rawResults = [];
+    for (let i = 0; i < iterations; i++) {
+        rawResults.push(await measureRawProvider(provider));
+    }
+    const kernelResults = [];
+    for (let i = 0; i < iterations; i++) {
+        kernelResults.push(await measureKernelPipeline(provider));
+    }
+    const rawMetrics = aggregate(rawResults);
+    const kernelMetrics = aggregate(kernelResults);
+    printResults('Raw Provider (baseline)', rawMetrics);
+    printResults('Kernel Pipeline (runTurn)', kernelMetrics);
+    printOverhead(rawMetrics, kernelMetrics);
+    printPhaseBreakdown(iterations);
+    await printMicroBenchmarks(chunks, iterations);
+    clearProfiles();
+    console.log('\n');
+}

package/esm/src/cli/commands/egress-compile.d.ts

@@ -0,0 +1,17 @@
+/**
+ * `agents egress-compile`: reads a module's exported egress rules and writes
+ * the table `egressPolicy` loads with them.
+ *
+ * @module
+ */
+interface EgressCompileOptions {
+    /** Path or URL of the module exporting the rules. */
+    module: string;
+    /** The export holding the rules. Default `rules`. */
+    exportName?: string;
+    /** Where the compiled module is written. */
+    out: string;
+}
+/** Compiles the rules and writes the module; returns how many rules it read. */
+declare function egressCompileCommand({ module, exportName, out, }: EgressCompileOptions): Promise<number>;
+export { egressCompileCommand };

package/esm/src/cli/commands/egress-compile.js

@@ -0,0 +1,25 @@
+/**
+ * `agents egress-compile`: reads a module's exported egress rules and writes
+ * the table `egressPolicy` loads with them.
+ *
+ * @module
+ */
+import { writeFile } from 'node:fs/promises';
+import { cwd } from 'node:process';
+/** `specifier` as a URL, relative paths read from the working directory. */
+function moduleUrl(specifier) {
+    return new URL(specifier, `file://${cwd()}/`).href;
+}
+/** Compiles the rules and writes the module; returns how many rules it read. */
+async function egressCompileCommand({ module, exportName = 'rules', out, }) {
+    const loaded = await import(moduleUrl(module));
+    const rules = loaded[exportName];
+    if (!Array.isArray(rules)) {
+        throw new Error(`${module} has no array export named ${exportName}`);
+    }
+    // Loaded here so no other command loads the regex engine.
+    const { compiledEgressModule, compileEgressRules } = await import('../../guardrails/egress-compiler.js');
+    await writeFile(out, compiledEgressModule(compileEgressRules(rules)));
+    return rules.length;
+}
+export { egressCompileCommand };

package/esm/src/cli/commands/eval.d.ts

@@ -0,0 +1,37 @@
+import type { EvalMediaResolver } from '../../evals/types.js';
+import type { RunDecisionOptions } from '../../kernel/engine/decision.js';
+import type { ModelProvider } from '../../kernel/types.js';
+export interface EvalOptions {
+    /** Path of the suite module (`export default` an `EvalSuite`). */
+    suite: string;
+    /** Grade these records (a JSONL file or a directory of them) instead of running the profile. */
+    recorded?: string;
+    /** Trials per case, overriding the suite's `trials.repeat`. */
+    trials?: number;
+    /** Trials in flight at once (default 1). */
+    concurrency?: number;
+    /** Append trial and run records as JSONL under this directory. */
+    traceDir?: string;
+    /** Print the run as one JSON document instead of the table. */
+    json?: boolean;
+    /** Stop starting trials once the run's cost passes this. */
+    maxCostUsd?: number;
+    /** The fraction of cases that must pass for exit 0 (default 1: every case). */
+    threshold?: number;
+}
+/** What the host hands the eval command: providers and keys it never creates or reads itself. */
+export interface EvalHost {
+    /** The provider for the profile under test. */
+    provider?: ModelProvider;
+    /** The provider for text judge profiles. */
+    judgeProvider?: ModelProvider;
+    /** The key for decision judge profiles (Jev). */
+    judgeDecision?: Omit<RunDecisionOptions, 'sink'>;
+    /** Where a judge finds media a trace names only by hash. */
+    media?: EvalMediaResolver;
+}
+/**
+ * Returns whether the run met the threshold; a run stopped on budget never does. The CLI never
+ * creates a provider or reads a key: the host passes them or the suite module exports them.
+ */
+export declare function evalCommand(options: EvalOptions, host?: EvalHost): Promise<boolean>;