@parcha/agentrun-pi 0.1.0-beta.2 → 0.1.0-beta.21

package/dist/benchmark-contract.js

@@ -1,50 +0,0 @@
-import { createHash } from 'node:crypto';
-import { readFileSync } from 'node:fs';
-import { Compile } from 'typebox/compile';
-const cases = [
-    { id: 'docs-editing', split: 'development', question: 'Can contributors edit our docs in the browser?', topics: [{ id: 'editing', question: 'Did contributors edit a page in the browser during our trial?' }], sources: [{ id: 'd1', text: 'All three contributors edited a page in the browser during the trial.' }, { id: 'd2', text: 'The documentation product has a new logo.' }], expected: [{ topicId: 'editing', status: 'supported', sourceIds: ['d1'] }] },
-    { id: 'migration', split: 'development', question: 'What changes with a repository-based wiki?', topics: [{ id: 'review', question: 'How are changes reviewed?' }, { id: 'editing', question: 'Could non-developers edit documentation?' }, { id: 'search', question: 'What did the search trial find?' }], sources: [{ id: 'm1', text: 'Documentation changes require an assigned reviewer to approve the pull request.' }, { id: 'm2', text: 'Non-developers edited in the browser but two needed help with merge conflicts.' }, { id: 'm3', text: 'Search found exact words but missed synonyms during our trial.' }, { id: 'm4', text: 'A revolutionary documentation experience.' }], expected: [{ topicId: 'review', status: 'supported', sourceIds: ['m1'] }, { topicId: 'editing', status: 'supported', sourceIds: ['m2'] }, { topicId: 'search', status: 'supported', sourceIds: ['m3'] }] },
-    { id: 'missing-conflicting', split: 'development', question: 'Can we rely on restricted editing and synonym search?', topics: [{ id: 'permissions', question: 'Who can edit restricted pages?' }, { id: 'search', question: 'Did synonym search work in this trial?' }], sources: [{ id: 'c1', text: 'Restricted-page permissions: details pending.' }, { id: 'c2', text: 'The trial report says all ten synonym searches found their target pages.' }, { id: 'c3', text: 'An audit of the same trial says none of those ten searches found their target pages.' }], expected: [{ topicId: 'permissions', status: 'insufficient', sourceIds: [] }, { topicId: 'search', status: 'conflicting', sourceIds: ['c2', 'c3'] }] },
-    { id: 'backup', split: 'heldout', question: 'Did the restore drill recover the archive?', topics: [{ id: 'restore', question: 'Was the archive recovered during the drill?' }], sources: [{ id: 'h1', text: 'The drill restored the archive and every file matched its recorded checksum.' }, { id: 'h2', text: 'Backups that give you peace of mind.' }], expected: [{ topicId: 'restore', status: 'supported', sourceIds: ['h1'] }] },
-    { id: 'queue-adoption', split: 'heldout', question: 'What should we know before adopting this task queue?', topics: [{ id: 'retries', question: 'What happens to jobs after worker failure?' }, { id: 'ordering', question: 'Is task order guaranteed?' }, { id: 'visibility', question: 'Can operators inspect stuck jobs?' }], sources: [{ id: 'q1', text: 'In the failure test, an unfinished job returned to the queue when its lease expired.' }, { id: 'q2', text: 'Tasks from different workers completed out of submission order in the trial.' }, { id: 'q3', text: 'Operators used the dashboard to inspect job age and the last worker error.' }, { id: 'q4', text: 'Supercharge your asynchronous future.' }], expected: [{ topicId: 'retries', status: 'supported', sourceIds: ['q1'] }, { topicId: 'ordering', status: 'supported', sourceIds: ['q2'] }, { topicId: 'visibility', status: 'supported', sourceIds: ['q3'] }] },
-    { id: 'cache-evidence', split: 'heldout', question: 'What do the cache trials tell us about deletion and encryption?', topics: [{ id: 'deletion', question: 'Did deleted entries disappear immediately in the trial?' }, { id: 'encryption', question: 'Which encryption protects cached entries?' }], sources: [{ id: 'k1', text: 'The trial log says every deleted entry disappeared immediately.' }, { id: 'k2', text: 'A second observer of that same trial recorded deleted entries remaining readable for a full minute.' }, { id: 'k3', text: 'Encryption: documentation forthcoming.' }], expected: [{ topicId: 'deletion', status: 'conflicting', sourceIds: ['k1', 'k2'] }, { topicId: 'encryption', status: 'insufficient', sourceIds: [] }] },
-];
-export const benchmarkHash = (value) => createHash('sha256').update(JSON.stringify(value)).digest('hex');
-export const benchmarkInstructions = 'Answer using only the fictional source corpus. Produce exactly one finding for each supplied topic. Status supported means concrete evidence establishes a finding, including a negative or qualified answer. Conflicting means incompatible accounts. Insufficient means no concrete information answers the question. Preserve conditions and contrary evidence; cite every material source by ID. Do not invent facts or citations. Submit the final structured report.';
-const str = { type: 'string', minLength: 1 };
-export const benchmarkReportSchema = { type: 'object', additionalProperties: false, required: ['findings'], properties: { findings: { type: 'array', minItems: 1, items: { type: 'object', additionalProperties: false, required: ['topicId', 'status', 'summary', 'sourceIds'], properties: { topicId: str, status: { enum: ['supported', 'conflicting', 'insufficient'] }, summary: str, sourceIds: { type: 'array', uniqueItems: true, items: str } } } } } };
-export const benchmarkEvidenceSchema = { type: 'object', additionalProperties: false, required: ['relevant'], properties: { relevant: { type: 'boolean', description: 'Does the passage contain a concrete fact, condition, limitation or contrary evidence bearing on the question?', criteria: { true: 'Concrete evidence relevant to the question, including qualified or contradictory answers.', false: 'Only topic mentions, marketing promises, missing details, or unrelated information.' } } } };
-const validator = Compile(benchmarkReportSchema);
-export function benchmarkCases(split) {
-    if (split !== 'development' && split !== 'heldout')
-        throw new Error('Choose development or heldout explicitly');
-    return structuredClone(cases.filter(row => row.split === split));
-}
-/** Frozen independent checks. Never return these results to an authoring agent. */
-export function evaluatePiBenchmark(caseId, output) {
-    const item = cases.find(row => row.id === caseId);
-    if (!item)
-        throw new Error('Unknown benchmark case');
-    if (!validator.Check(output))
-        return { passed: false, issues: ['schema'] };
-    const findings = output.findings;
-    const issues = [];
-    if (findings.length !== item.expected.length || new Set(findings.map(row => row.topicId)).size !== findings.length)
-        issues.push('coverage');
-    for (const expected of item.expected) {
-        const actual = findings.find(row => row.topicId === expected.topicId);
-        if (actual?.status !== expected.status)
-            issues.push(`${expected.topicId}:status`);
-        if (!actual || JSON.stringify([...actual.sourceIds].sort()) !== JSON.stringify([...expected.sourceIds].sort()))
-            issues.push(`${expected.topicId}:citations`);
-    }
-    return { passed: issues.length === 0, issues };
-}
-export function benchmarkContract() {
-    return {
-        version: 1, fixturesSha256: benchmarkHash(cases),
-        verifierSha256: createHash('sha256').update(readFileSync(new URL('./benchmark-contract.js', import.meta.url))).digest('hex'),
-        outputSchemaSha256: benchmarkHash(benchmarkReportSchema),
-        instructionsSha256: benchmarkHash(benchmarkInstructions),
-    };
-}

package/dist/benchmark-freeze.d.ts

@@ -1,9 +0,0 @@
-/** Frozen before any live sample. Changing these pins requires a new benchmark
- * version and a fresh baseline; never update them to improve a skill's score. */
-export declare const frozenBenchmarkContract: {
-    readonly version: 1;
-    readonly fixturesSha256: "1a8f9ff022d5cdaf2455e327e59ae4593aeffcc8ae4c510a870b36ea0d9e7de2";
-    readonly verifierSha256: "f21d0c8b790186f571003907a5a6c7a6360423d77d1125c4398994cb1ae11f9a";
-    readonly outputSchemaSha256: "38a0d7ccd8590ad8a486a5b7b7edd36c0b8bd32e642ead5345a3f93d12f3e574";
-    readonly instructionsSha256: "d8d3fc6721ed99bb163ff3dbd93dffc15a1a92ab400f11f781bb25e9f8a61535";
-};

package/dist/benchmark-freeze.js

@@ -1,9 +0,0 @@
-/** Frozen before any live sample. Changing these pins requires a new benchmark
- * version and a fresh baseline; never update them to improve a skill's score. */
-export const frozenBenchmarkContract = {
-    version: 1,
-    fixturesSha256: '1a8f9ff022d5cdaf2455e327e59ae4593aeffcc8ae4c510a870b36ea0d9e7de2',
-    verifierSha256: 'f21d0c8b790186f571003907a5a6c7a6360423d77d1125c4398994cb1ae11f9a',
-    outputSchemaSha256: '38a0d7ccd8590ad8a486a5b7b7edd36c0b8bd32e642ead5345a3f93d12f3e574',
-    instructionsSha256: 'd8d3fc6721ed99bb163ff3dbd93dffc15a1a92ab400f11f781bb25e9f8a61535',
-};

package/dist/benchmark.d.ts

@@ -1,127 +0,0 @@
-import { type WorkflowDeps } from '@parcha/agentrun-dsl';
-import type { PiHostContext } from './types.js';
-import { evaluatePiBenchmark, type BenchmarkSplit } from './benchmark-contract.js';
-export interface PiBenchmarkBudgets {
-    timeoutMs: number;
-    maxModelRequests: number;
-    maxToolCalls: number;
-    maxTurns: number;
-    maxSubmissions: number;
-}
-export interface PiSkillBenchmarkOptions {
-    host: PiHostContext;
-    runJudge: NonNullable<WorkflowDeps['runJudge']>;
-    /** Exact native skill plus its loaded references, supplied by the caller. No hidden authoring contract is added. */
-    skillText: string;
-    split: BenchmarkSplit;
-    mode: 'live' | 'scripted-transport';
-    signal?: AbortSignal;
-    budgets?: Partial<PiBenchmarkBudgets>;
-    onProgress?: (event: {
-        caseId: string;
-        arm: 'plain-pi' | 'skill-dsl';
-        stage: 'arm' | 'outer' | 'workflow';
-        event: 'start' | 'end';
-        status?: string;
-    }) => unknown;
-}
-type Metrics = {
-    pi: number;
-    jev: number;
-    tools: number;
-    submissions: number;
-    tokens: number;
-    usageSamples: number;
-    knownCostUsd: number;
-    costSamples: number;
-};
-type Arm = 'plain-pi' | 'skill-dsl';
-export interface PiBenchmarkArm {
-    arm: Arm;
-    status: 'passed' | 'rejected' | 'failed' | 'cancelled';
-    elapsedMs: number;
-    total: Metrics;
-    stages: {
-        outer: Metrics;
-        workflow: Metrics;
-    };
-    costComplete: boolean;
-    usageComplete: boolean;
-    candidates: {
-        digest?: string;
-        accepted: boolean;
-        workflow?: unknown;
-    }[];
-    workflowRuns: {
-        status: string;
-        digest: string;
-        output?: unknown;
-    }[];
-    output?: unknown;
-    evaluation?: ReturnType<typeof evaluatePiBenchmark>;
-    failure?: string;
-}
-/** End-to-end comparison: DSL authoring, repair, execution and delivery all count.
- * Uses the selected host model, never a model selector or alternate provider. */
-export declare function runPiSkillBenchmark(options: PiSkillBenchmarkOptions): Promise<{
-    mode: "live" | "scripted-transport";
-    split: BenchmarkSplit;
-    status: "complete" | "cancelled";
-    passed: boolean;
-    contract: {
-        version: number;
-        fixturesSha256: string;
-        verifierSha256: string;
-        outputSchemaSha256: string;
-        instructionsSha256: string;
-    };
-    skillSha256: string;
-    budgets: {
-        timeoutMs: number;
-        maxModelRequests: number;
-        maxToolCalls: number;
-        maxTurns: number;
-        maxSubmissions: number;
-    };
-    runtime: {
-        model: string;
-        provider: string;
-        api: string;
-        thinking: import("./types.js").PiThinkingLevel;
-    };
-    runs: {
-        caseId: string;
-        inputSha256: string;
-        order: Arm[];
-        arms: PiBenchmarkArm[];
-    }[];
-    limitations: string[];
-}>;
-export type PiSkillBenchmarkReport = Awaited<ReturnType<typeof runPiSkillBenchmark>>;
-/** A development comparison receipt, not automatic promotion. Heldout is a final
- * confirmation set, never the optimization target for this helper. */
-export declare function comparePiSkillBenchmarks(baseline: PiSkillBenchmarkReport, candidate: PiSkillBenchmarkReport): {
-    mode: "live" | "scripted-transport";
-    baselineSkillSha256: string;
-    candidateSkillSha256: string;
-    regressions: string[];
-    previous: {
-        passed: number;
-        requests: number;
-        tokens: number;
-        usageComplete: boolean;
-        knownCostUsd: number;
-        costComplete: boolean;
-    };
-    current: {
-        passed: number;
-        requests: number;
-        tokens: number;
-        usageComplete: boolean;
-        knownCostUsd: number;
-        costComplete: boolean;
-    };
-    improved: boolean;
-    eligibleForLiveHeldout: boolean;
-};
-export {};

package/dist/benchmark.js

@@ -1,260 +0,0 @@
-import { isDeepStrictEqual } from 'node:util';
-import { frozenBenchmarkContract } from './benchmark-freeze.js';
-import { Compile } from 'typebox/compile';
-import { formatWorkflowTree, runWorkflowSlice } from '@parcha/agentrun-dsl';
-import { createPiHostSessionFactory } from './host-session.js';
-import { createPiRunner, PiRunError } from './runner.js';
-import { WorkflowExtensionService, ExtensionServiceError } from './extension-service.js';
-import { benchmarkCases, benchmarkContract, benchmarkHash, benchmarkInstructions, benchmarkReportSchema, benchmarkEvidenceSchema, evaluatePiBenchmark } from './benchmark-contract.js';
-const emptyMetrics = () => ({ pi: 0, jev: 0, tools: 0, submissions: 0, tokens: 0, usageSamples: 0, knownCostUsd: 0, costSamples: 0 });
-const ceilings = { timeoutMs: 600_000, maxModelRequests: 24, maxToolCalls: 48, maxTurns: 24, maxSubmissions: 4 };
-class BenchmarkStop extends Error {
-    code;
-    constructor(code) {
-        super(code);
-        this.code = code;
-    }
-}
-const textResult = (value) => ({ content: [{ type: 'text', text: JSON.stringify(value) }], details: value });
-const selectionWorkflow = { v: 2, name: 'benchmark-evidence', schemas: { Evidence: benchmarkEvidenceSchema, Result: { type: 'object' } }, output: { schemaId: 'Result', path: 'evidence' }, root: { node: 'sift', label: 'screen', itemsPath: 'sources', state: { question: '{question}' }, out: 'Evidence', as: 'evidence', keep: { path: 'relevant', gte: 0.8 } } };
-/** End-to-end comparison: DSL authoring, repair, execution and delivery all count.
- * Uses the selected host model, never a model selector or alternate provider. */
-export async function runPiSkillBenchmark(options) {
-    if (options.mode !== 'live' && options.mode !== 'scripted-transport')
-        throw new Error('Choose live or scripted-transport explicitly');
-    if (typeof options.skillText !== 'string' || !options.skillText.trim() || Buffer.byteLength(options.skillText) > 128 * 1024)
-        throw new Error('Provide the exact native skill bundle, at most 128 KiB');
-    if (typeof options.runJudge !== 'function')
-        throw new Error('Both arms require the same Jev capability');
-    const suite = benchmarkCases(options.split);
-    const contract = benchmarkContract();
-    if (!isDeepStrictEqual(contract, frozenBenchmarkContract))
-        throw new Error('Frozen benchmark contract changed; establish a new version and baseline');
-    const skillText = options.skillText;
-    const budgets = { ...ceilings, ...options.budgets };
-    for (const key of Object.keys(ceilings))
-        if (!Number.isSafeInteger(budgets[key]) || budgets[key] < 1 || budgets[key] > ceilings[key])
-            throw new Error(`Invalid benchmark budget: ${key}`);
-    const captured = { ...options.host };
-    createPiHostSessionFactory(captured);
-    const model = captured.model;
-    const runs = [];
-    for (const [index, item] of suite.entries()) {
-        if (options.signal?.aborted)
-            break;
-        const order = index % 2 ? ['skill-dsl', 'plain-pi'] : ['plain-pi', 'skill-dsl'];
-        const arms = [];
-        for (const arm of order) {
-            if (options.signal?.aborted)
-                break;
-            arms.push(await runArm(item, arm));
-        }
-        runs.push({ caseId: item.id, inputSha256: benchmarkHash({ question: item.question, topics: item.topics, sources: item.sources }), order: arms.map(row => row.arm), arms });
-    }
-    if (!isDeepStrictEqual(contract, benchmarkContract()))
-        throw new Error('Benchmark verifier changed during execution');
-    return {
-        mode: options.mode, split: options.split, status: options.signal?.aborted ? 'cancelled' : 'complete',
-        passed: runs.length === suite.length && runs.every(row => row.arms.length === 2 && row.arms.every(arm => arm.status === 'passed')),
-        contract, skillSha256: benchmarkHash(skillText), budgets,
-        runtime: { model: model.id, provider: model.provider, api: model.api, thinking: captured.thinkingLevel ?? 'low' }, runs,
-        limitations: ['Three fictional cases per split are not a statistical or real-world benchmark.', 'Gold checks cover statuses and citation sets; summary entailment remains unscored.', 'All outer-agent authoring, repair, final delivery, child execution and failed attempts count; initial package/skill development does not.', 'Request caps count adapter invocations, not provider-internal HTTP retries. There is no token or spending cap.', 'Known cost uses returned provider/SDK accounting only; missing samples are unknown, including failed requests.', 'Host caches are not reset; execution order alternates. Heldout cases must not be used to tune the skill.', 'The DSL arm uses the native bridge and service with a constrained benchmark tool surface; this is not a test of interactive skill discovery.'],
-    };
-    async function runArm(item, arm) {
-        const started = performance.now();
-        const notify = (stage, event, status) => {
-            try {
-                void Promise.resolve(options.onProgress?.({ caseId: item.id, arm, stage, event, ...(status ? { status } : {}) })).catch(() => { });
-            }
-            catch { /* UI hooks cannot change the measurement. */ }
-        };
-        notify('arm', 'start');
-        notify('outer', 'start');
-        const controller = new AbortController();
-        const signal = controller.signal;
-        const stop = (code) => { if (!signal.aborted)
-            controller.abort(new BenchmarkStop(code)); };
-        const externalAbort = () => stop('cancelled');
-        options.signal?.addEventListener('abort', externalAbort, { once: true });
-        if (options.signal?.aborted)
-            externalAbort();
-        const timer = setTimeout(() => stop('deadline'), budgets.timeoutMs);
-        const interrupted = new Promise((_resolve, reject) => {
-            signal.addEventListener('abort', () => reject(signal.reason), { once: true });
-            if (signal.aborted)
-                reject(signal.reason);
-        });
-        void interrupted.catch(() => { });
-        const bounded = (promise) => Promise.race([promise, interrupted]);
-        let stage = 'outer';
-        const stages = { outer: emptyMetrics(), workflow: emptyMetrics() };
-        const candidates = [], workflowRuns = [];
-        const total = () => Object.fromEntries(Object.keys(stages.outer).map(key => [key, stages.outer[key] + stages.workflow[key]]));
-        const admit = (kind) => {
-            signal.throwIfAborted();
-            const counts = total();
-            if (kind === 'tools' ? counts.tools >= budgets.maxToolCalls : counts.pi + counts.jev >= budgets.maxModelRequests)
-                stop(kind === 'tools' ? 'tool_limit' : 'model_limit');
-            signal.throwIfAborted();
-            stages[stage][kind]++;
-        };
-        const usage = (bucket, tokens, cost) => {
-            if (typeof tokens === 'number' && Number.isFinite(tokens) && tokens >= 0) {
-                bucket.tokens += tokens;
-                bucket.usageSamples++;
-            }
-            if (typeof cost === 'number' && Number.isFinite(cost) && cost >= 0) {
-                bucket.knownCostUsd += cost;
-                bucket.costSamples++;
-            }
-        };
-        const host = { ...captured, modelRegistry: {
-                getAll: () => captured.modelRegistry.getAll(),
-                streamSimple(selected, transcript, opts) {
-                    admit('pi');
-                    const bucket = stages[stage];
-                    const stream = captured.modelRegistry.streamSimple(selected, transcript, opts);
-                    void stream.result().then(result => {
-                        const value = result;
-                        usage(bucket, value.usage?.totalTokens, value.usage?.cost?.total);
-                    }).catch(() => { });
-                    return stream;
-                },
-            } };
-        const runJudge = async (request) => {
-            request.signal?.throwIfAborted();
-            admit('jev');
-            const bucket = stages[stage];
-            const result = await bounded(options.runJudge({ ...request, signal: request.signal ? AbortSignal.any([signal, request.signal]) : signal }));
-            usage(bucket, result.usage ? result.usage.input_tokens + result.usage.output_tokens : undefined, result.cost_usd);
-            return result;
-        };
-        const domainTools = [
-            { name: 'read_sources', label: 'Read sources', description: 'Return {sources:[{id,text}]} for this fictional task. Direct workflow calls return this same structured value. This is not web search.', parameters: { type: 'object', properties: {}, additionalProperties: false }, async execute() { return textResult({ sources: structuredClone(item.sources) }); } },
-            { name: 'screen_evidence', label: 'Screen evidence', description: 'Jev System One decisions select concrete evidence including limitations and contradictions at threshold 0.8.', parameters: { type: 'object', additionalProperties: false, required: ['topicId', 'sourceIds'], properties: { topicId: { type: 'string' }, sourceIds: { type: 'array', uniqueItems: true, items: { type: 'string' } } } },
-                async execute(_id, value, toolSignal) {
-                    const args = value;
-                    const topic = item.topics.find(row => row.id === args.topicId);
-                    if (!topic || args.sourceIds.some(id => !item.sources.some(source => source.id === id)))
-                        throw new Error('Unknown topic or source');
-                    const result = await bounded(runWorkflowSlice(selectionWorkflow, { question: topic.question, sources: item.sources.filter(source => args.sourceIds.includes(source.id)) }, { from: 'screen' }, { runJudge: request => runJudge({ ...request, signal: toolSignal }) }));
-                    if (result.status !== 'complete')
-                        throw new Error('Screening did not complete');
-                    return textResult(result.state.evidence);
-                } },
-        ];
-        const service = new WorkflowExtensionService({ allowedTools: domainTools.map(tool => tool.name), limits: { deadlineMs: budgets.timeoutMs } });
-        const input = { question: item.question, topics: structuredClone(item.topics) };
-        let lastOutput;
-        let workflowInput = input;
-        try {
-            const factory = createPiHostSessionFactory(host);
-            const makeRunner = (tools) => createPiRunner({ model, thinkingLevel: captured.thinkingLevel, cwd: captured.cwd, signal, tools,
-                maxTurns: budgets.maxTurns, maxSubmissions: budgets.maxSubmissions, timeoutMs: budgets.timeoutMs,
-                sessionFactory: opts => factory({ ...opts, customTools: opts.customTools.map(tool => ({ ...tool, async execute(...args) {
-                            args[2]?.throwIfAborted();
-                            admit('tools');
-                            if (tool.name === 'submit')
-                                stages[stage].submissions++;
-                            const validator = Compile(tool.parameters);
-                            if (!validator.Check(args[1]))
-                                throw new Error('Tool input does not match its schema');
-                            return tool.execute(...args);
-                        } })) }),
-            });
-            const child = makeRunner(domainTools);
-            const agentrun = { name: 'agentrun', label: 'AgentRun workflow', description: 'Describe available tools, inspect a workflow, then run the inspected workflow. Never run an inline candidate. No shell, code, file access or model selection.', parameters: { type: 'object', additionalProperties: false, required: ['action'], properties: { action: { enum: ['describe', 'inspect', 'run'] }, workflow: {}, input: { type: 'object' } } }, async execute(_id, value) {
-                    const args = value;
-                    if (args.action === 'inspect' && args.input !== undefined)
-                        workflowInput = args.input;
-                    if (args.action === 'describe')
-                        return textResult({ tools: domainTools.map(tool => ({ name: tool.name, description: tool.description, parameters: tool.parameters })), skill: true, pi: true, jev: true, running: false, workflow: !!service.inspect().current });
-                    if (args.action === 'inspect') {
-                        try {
-                            if (args.workflow === undefined) {
-                                const current = service.inspect().current;
-                                if (!current)
-                                    throw new Error('Inspect a workflow first');
-                                return textResult({ digest: current.digest, tree: formatWorkflowTree(current.inspection) });
-                            }
-                            const prepared = service.prepare(args.workflow);
-                            candidates.push({ digest: prepared.digest, accepted: true, workflow: prepared.workflow });
-                            return textResult({ digest: prepared.digest, tree: formatWorkflowTree(prepared.inspection) });
-                        }
-                        catch (error) {
-                            candidates.push({ accepted: false, workflow: structuredClone(args.workflow) });
-                            throw error;
-                        }
-                    }
-                    if (args.action !== 'run' || args.workflow !== undefined)
-                        throw new Error('Inspect the workflow first; run without a workflow argument');
-                    notify('outer', 'end');
-                    stage = 'workflow';
-                    notify('workflow', 'start');
-                    try {
-                        const report = await bounded(service.run(args.input ?? workflowInput, { signal, deps: {
-                                // A candidate cannot change the captured host thinking setting.
-                                runNode: request => child({ ...request, thinking: undefined }), runJudge,
-                                runEffect: async (request) => {
-                                    request.signal.throwIfAborted();
-                                    admit('tools');
-                                    const tool = domainTools.find(tool => tool.name === request.node.tool);
-                                    if (!tool || !Compile(tool.parameters).Check(request.input))
-                                        throw new Error('Unsupported tool input');
-                                    return (await bounded(tool.execute('benchmark-effect', request.input, request.signal, undefined, host))).details;
-                                },
-                            } }));
-                        workflowRuns.push({ status: report.status, digest: report.digest, ...(report.output === undefined ? {} : { output: report.output }) });
-                        lastOutput = report.status === 'complete' ? report.output : undefined;
-                        return textResult(report);
-                    }
-                    finally {
-                        notify('workflow', 'end');
-                        stage = 'outer';
-                        notify('outer', 'start');
-                    }
-                } };
-            const outer = makeRunner(arm === 'skill-dsl' ? [...domainTools, agentrun] : domainTools);
-            const output = await bounded(outer({ kind: 'agent', label: 'benchmark-outer',
-                system: [benchmarkInstructions, ...(arm === 'skill-dsl' ? [skillText, 'Use agentrun to inspect and execute your workflow. Deliver its completed output unchanged with submit. The benchmark forbids code, shell, files and changing models. Available task tools and their contracts are returned by agentrun describe.'] : [])],
-                user: JSON.stringify(input), schema: benchmarkReportSchema }));
-            signal.throwIfAborted();
-            if (arm === 'skill-dsl' && (lastOutput === undefined || !isDeepStrictEqual(lastOutput, output)))
-                return finish({ status: 'rejected', output, failure: 'workflow_delivery_missing_or_changed' });
-            const evaluation = evaluatePiBenchmark(item.id, output);
-            return finish({ status: evaluation.passed ? 'passed' : 'rejected', output, evaluation });
-        }
-        catch (error) {
-            return finish({ status: options.signal?.aborted ? 'cancelled' : 'failed', failure: signal.reason instanceof BenchmarkStop ? signal.reason.code : error instanceof PiRunError ? `pi_${error.reason}` : error instanceof ExtensionServiceError ? error.code : 'execution_failed' });
-        }
-        finally {
-            clearTimeout(timer);
-            options.signal?.removeEventListener('abort', externalAbort);
-            stop('arm_finished');
-            void service.dispose();
-        }
-        function finish(result) {
-            const counts = total();
-            notify('outer', 'end', result.status);
-            notify('arm', 'end', result.status);
-            return structuredClone({ arm, ...result, elapsedMs: Math.round(performance.now() - started), total: counts, stages, candidates, workflowRuns, costComplete: counts.costSamples === counts.pi + counts.jev, usageComplete: counts.usageSamples === counts.pi + counts.jev });
-        }
-    }
-}
-/** A development comparison receipt, not automatic promotion. Heldout is a final
- * confirmation set, never the optimization target for this helper. */
-export function comparePiSkillBenchmarks(baseline, candidate) {
-    if (baseline.split !== 'development' || candidate.split !== 'development')
-        throw new Error('Hillclimb comparisons use development only');
-    if (baseline.status !== 'complete' || candidate.status !== 'complete' || baseline.mode !== candidate.mode || !isDeepStrictEqual(baseline.contract, candidate.contract) || !isDeepStrictEqual(baseline.budgets, candidate.budgets) || !isDeepStrictEqual(baseline.runtime, candidate.runtime))
-        throw new Error('Incomparable benchmark receipts');
-    const select = (report) => report.runs.map(run => ({ id: run.caseId, input: run.inputSha256, arm: run.arms.find(arm => arm.arm === 'skill-dsl') }));
-    const before = select(baseline), after = select(candidate);
-    if (before.length !== 3 || after.length !== 3 || !isDeepStrictEqual(before.map(row => [row.id, row.input]), after.map(row => [row.id, row.input])) || [...before, ...after].some(row => !row.arm))
-        throw new Error('Incomplete benchmark receipts');
-    const regressions = before.filter((row, i) => row.arm.status === 'passed' && after[i].arm.status !== 'passed').map(row => row.id);
-    const sums = (rows) => ({ passed: rows.filter(row => row.arm.status === 'passed').length, requests: rows.reduce((sum, row) => sum + row.arm.total.pi + row.arm.total.jev, 0), tokens: rows.reduce((sum, row) => sum + row.arm.total.tokens, 0), usageComplete: rows.every(row => row.arm.usageComplete), knownCostUsd: rows.reduce((sum, row) => sum + row.arm.total.knownCostUsd, 0), costComplete: rows.every(row => row.arm.costComplete) });
-    const previous = sums(before), current = sums(after);
-    const improved = !regressions.length && (current.passed > previous.passed || current.passed === previous.passed && current.passed === 3 && previous.usageComplete && current.usageComplete && current.tokens < previous.tokens);
-    return { mode: candidate.mode, baselineSkillSha256: baseline.skillSha256, candidateSkillSha256: candidate.skillSha256, regressions, previous, current, improved, eligibleForLiveHeldout: candidate.mode === 'live' && improved && current.passed === 3 };
-}

package/dist/comparison.d.ts

@@ -1,69 +0,0 @@
-import { type WorkflowDeps } from '@parcha/agentrun-dsl';
-import type { PiHostContext } from './types.js';
-export declare function evaluatePiComparison(caseId: string, output: unknown): {
-    passed: boolean;
-    issues: string[];
-};
-export interface PiComparisonBudgets {
-    timeoutMs: number;
-    maxModelRequests: number;
-    maxToolCalls: number;
-    maxTurns: number;
-    maxSubmissions: number;
-}
-export interface PiComparisonOptions {
-    host: PiHostContext;
-    runJudge: NonNullable<WorkflowDeps['runJudge']>;
-    signal?: AbortSignal;
-    mode: 'live' | 'scripted-transport';
-    budgets?: Partial<PiComparisonBudgets>;
-}
-export interface PiComparisonArm {
-    arm: 'plain-pi' | 'workflow';
-    status: 'passed' | 'rejected' | 'failed' | 'cancelled';
-    elapsedMs: number;
-    requests: {
-        pi: number;
-        jev: number;
-        total: number;
-    };
-    toolCalls: number;
-    submissions: number;
-    tokens: {
-        pi: number;
-        jev: number;
-        piUsageSamples: number;
-        jevUsageSamples: number;
-    };
-    output?: unknown;
-    evaluation?: ReturnType<typeof evaluatePiComparison>;
-    failure?: string;
-}
-export declare function runPiComparison(options: PiComparisonOptions): Promise<{
-    mode: "live" | "scripted-transport";
-    status: "complete" | "cancelled";
-    passed: boolean;
-    runtime: {
-        provider: string;
-        model: string;
-        api: string;
-        thinking: import("./types.js").PiThinkingLevel;
-    };
-    budgets: {
-        timeoutMs: number;
-        maxModelRequests: number;
-        maxToolCalls: number;
-        maxTurns: number;
-        maxSubmissions: number;
-    };
-    workflowSha256: string;
-    suiteSha256: string;
-    runs: {
-        caseId: string;
-        inputSha256: string;
-        expectedSha256: string;
-        executionOrder: string[];
-        arms: PiComparisonArm[];
-    }[];
-    limits: string[];
-}>;