@parcha/agentrun-pi 0.1.0-beta.2 → 0.1.0-beta.3

package/dist/benchmark.js

@@ -1,260 +0,0 @@
-import { isDeepStrictEqual } from 'node:util';
-import { frozenBenchmarkContract } from './benchmark-freeze.js';
-import { Compile } from 'typebox/compile';
-import { formatWorkflowTree, runWorkflowSlice } from '@parcha/agentrun-dsl';
-import { createPiHostSessionFactory } from './host-session.js';
-import { createPiRunner, PiRunError } from './runner.js';
-import { WorkflowExtensionService, ExtensionServiceError } from './extension-service.js';
-import { benchmarkCases, benchmarkContract, benchmarkHash, benchmarkInstructions, benchmarkReportSchema, benchmarkEvidenceSchema, evaluatePiBenchmark } from './benchmark-contract.js';
-const emptyMetrics = () => ({ pi: 0, jev: 0, tools: 0, submissions: 0, tokens: 0, usageSamples: 0, knownCostUsd: 0, costSamples: 0 });
-const ceilings = { timeoutMs: 600_000, maxModelRequests: 24, maxToolCalls: 48, maxTurns: 24, maxSubmissions: 4 };
-class BenchmarkStop extends Error {
-    code;
-    constructor(code) {
-        super(code);
-        this.code = code;
-    }
-}
-const textResult = (value) => ({ content: [{ type: 'text', text: JSON.stringify(value) }], details: value });
-const selectionWorkflow = { v: 2, name: 'benchmark-evidence', schemas: { Evidence: benchmarkEvidenceSchema, Result: { type: 'object' } }, output: { schemaId: 'Result', path: 'evidence' }, root: { node: 'sift', label: 'screen', itemsPath: 'sources', state: { question: '{question}' }, out: 'Evidence', as: 'evidence', keep: { path: 'relevant', gte: 0.8 } } };
-/** End-to-end comparison: DSL authoring, repair, execution and delivery all count.
- * Uses the selected host model, never a model selector or alternate provider. */
-export async function runPiSkillBenchmark(options) {
-    if (options.mode !== 'live' && options.mode !== 'scripted-transport')
-        throw new Error('Choose live or scripted-transport explicitly');
-    if (typeof options.skillText !== 'string' || !options.skillText.trim() || Buffer.byteLength(options.skillText) > 128 * 1024)
-        throw new Error('Provide the exact native skill bundle, at most 128 KiB');
-    if (typeof options.runJudge !== 'function')
-        throw new Error('Both arms require the same Jev capability');
-    const suite = benchmarkCases(options.split);
-    const contract = benchmarkContract();
-    if (!isDeepStrictEqual(contract, frozenBenchmarkContract))
-        throw new Error('Frozen benchmark contract changed; establish a new version and baseline');
-    const skillText = options.skillText;
-    const budgets = { ...ceilings, ...options.budgets };
-    for (const key of Object.keys(ceilings))
-        if (!Number.isSafeInteger(budgets[key]) || budgets[key] < 1 || budgets[key] > ceilings[key])
-            throw new Error(`Invalid benchmark budget: ${key}`);
-    const captured = { ...options.host };
-    createPiHostSessionFactory(captured);
-    const model = captured.model;
-    const runs = [];
-    for (const [index, item] of suite.entries()) {
-        if (options.signal?.aborted)
-            break;
-        const order = index % 2 ? ['skill-dsl', 'plain-pi'] : ['plain-pi', 'skill-dsl'];
-        const arms = [];
-        for (const arm of order) {
-            if (options.signal?.aborted)
-                break;
-            arms.push(await runArm(item, arm));
-        }
-        runs.push({ caseId: item.id, inputSha256: benchmarkHash({ question: item.question, topics: item.topics, sources: item.sources }), order: arms.map(row => row.arm), arms });
-    }
-    if (!isDeepStrictEqual(contract, benchmarkContract()))
-        throw new Error('Benchmark verifier changed during execution');
-    return {
-        mode: options.mode, split: options.split, status: options.signal?.aborted ? 'cancelled' : 'complete',
-        passed: runs.length === suite.length && runs.every(row => row.arms.length === 2 && row.arms.every(arm => arm.status === 'passed')),
-        contract, skillSha256: benchmarkHash(skillText), budgets,
-        runtime: { model: model.id, provider: model.provider, api: model.api, thinking: captured.thinkingLevel ?? 'low' }, runs,
-        limitations: ['Three fictional cases per split are not a statistical or real-world benchmark.', 'Gold checks cover statuses and citation sets; summary entailment remains unscored.', 'All outer-agent authoring, repair, final delivery, child execution and failed attempts count; initial package/skill development does not.', 'Request caps count adapter invocations, not provider-internal HTTP retries. There is no token or spending cap.', 'Known cost uses returned provider/SDK accounting only; missing samples are unknown, including failed requests.', 'Host caches are not reset; execution order alternates. Heldout cases must not be used to tune the skill.', 'The DSL arm uses the native bridge and service with a constrained benchmark tool surface; this is not a test of interactive skill discovery.'],
-    };
-    async function runArm(item, arm) {
-        const started = performance.now();
-        const notify = (stage, event, status) => {
-            try {
-                void Promise.resolve(options.onProgress?.({ caseId: item.id, arm, stage, event, ...(status ? { status } : {}) })).catch(() => { });
-            }
-            catch { /* UI hooks cannot change the measurement. */ }
-        };
-        notify('arm', 'start');
-        notify('outer', 'start');
-        const controller = new AbortController();
-        const signal = controller.signal;
-        const stop = (code) => { if (!signal.aborted)
-            controller.abort(new BenchmarkStop(code)); };
-        const externalAbort = () => stop('cancelled');
-        options.signal?.addEventListener('abort', externalAbort, { once: true });
-        if (options.signal?.aborted)
-            externalAbort();
-        const timer = setTimeout(() => stop('deadline'), budgets.timeoutMs);
-        const interrupted = new Promise((_resolve, reject) => {
-            signal.addEventListener('abort', () => reject(signal.reason), { once: true });
-            if (signal.aborted)
-                reject(signal.reason);
-        });
-        void interrupted.catch(() => { });
-        const bounded = (promise) => Promise.race([promise, interrupted]);
-        let stage = 'outer';
-        const stages = { outer: emptyMetrics(), workflow: emptyMetrics() };
-        const candidates = [], workflowRuns = [];
-        const total = () => Object.fromEntries(Object.keys(stages.outer).map(key => [key, stages.outer[key] + stages.workflow[key]]));
-        const admit = (kind) => {
-            signal.throwIfAborted();
-            const counts = total();
-            if (kind === 'tools' ? counts.tools >= budgets.maxToolCalls : counts.pi + counts.jev >= budgets.maxModelRequests)
-                stop(kind === 'tools' ? 'tool_limit' : 'model_limit');
-            signal.throwIfAborted();
-            stages[stage][kind]++;
-        };
-        const usage = (bucket, tokens, cost) => {
-            if (typeof tokens === 'number' && Number.isFinite(tokens) && tokens >= 0) {
-                bucket.tokens += tokens;
-                bucket.usageSamples++;
-            }
-            if (typeof cost === 'number' && Number.isFinite(cost) && cost >= 0) {
-                bucket.knownCostUsd += cost;
-                bucket.costSamples++;
-            }
-        };
-        const host = { ...captured, modelRegistry: {
-                getAll: () => captured.modelRegistry.getAll(),
-                streamSimple(selected, transcript, opts) {
-                    admit('pi');
-                    const bucket = stages[stage];
-                    const stream = captured.modelRegistry.streamSimple(selected, transcript, opts);
-                    void stream.result().then(result => {
-                        const value = result;
-                        usage(bucket, value.usage?.totalTokens, value.usage?.cost?.total);
-                    }).catch(() => { });
-                    return stream;
-                },
-            } };
-        const runJudge = async (request) => {
-            request.signal?.throwIfAborted();
-            admit('jev');
-            const bucket = stages[stage];
-            const result = await bounded(options.runJudge({ ...request, signal: request.signal ? AbortSignal.any([signal, request.signal]) : signal }));
-            usage(bucket, result.usage ? result.usage.input_tokens + result.usage.output_tokens : undefined, result.cost_usd);
-            return result;
-        };
-        const domainTools = [
-            { name: 'read_sources', label: 'Read sources', description: 'Return {sources:[{id,text}]} for this fictional task. Direct workflow calls return this same structured value. This is not web search.', parameters: { type: 'object', properties: {}, additionalProperties: false }, async execute() { return textResult({ sources: structuredClone(item.sources) }); } },
-            { name: 'screen_evidence', label: 'Screen evidence', description: 'Jev System One decisions select concrete evidence including limitations and contradictions at threshold 0.8.', parameters: { type: 'object', additionalProperties: false, required: ['topicId', 'sourceIds'], properties: { topicId: { type: 'string' }, sourceIds: { type: 'array', uniqueItems: true, items: { type: 'string' } } } },
-                async execute(_id, value, toolSignal) {
-                    const args = value;
-                    const topic = item.topics.find(row => row.id === args.topicId);
-                    if (!topic || args.sourceIds.some(id => !item.sources.some(source => source.id === id)))
-                        throw new Error('Unknown topic or source');
-                    const result = await bounded(runWorkflowSlice(selectionWorkflow, { question: topic.question, sources: item.sources.filter(source => args.sourceIds.includes(source.id)) }, { from: 'screen' }, { runJudge: request => runJudge({ ...request, signal: toolSignal }) }));
-                    if (result.status !== 'complete')
-                        throw new Error('Screening did not complete');
-                    return textResult(result.state.evidence);
-                } },
-        ];
-        const service = new WorkflowExtensionService({ allowedTools: domainTools.map(tool => tool.name), limits: { deadlineMs: budgets.timeoutMs } });
-        const input = { question: item.question, topics: structuredClone(item.topics) };
-        let lastOutput;
-        let workflowInput = input;
-        try {
-            const factory = createPiHostSessionFactory(host);
-            const makeRunner = (tools) => createPiRunner({ model, thinkingLevel: captured.thinkingLevel, cwd: captured.cwd, signal, tools,
-                maxTurns: budgets.maxTurns, maxSubmissions: budgets.maxSubmissions, timeoutMs: budgets.timeoutMs,
-                sessionFactory: opts => factory({ ...opts, customTools: opts.customTools.map(tool => ({ ...tool, async execute(...args) {
-                            args[2]?.throwIfAborted();
-                            admit('tools');
-                            if (tool.name === 'submit')
-                                stages[stage].submissions++;
-                            const validator = Compile(tool.parameters);
-                            if (!validator.Check(args[1]))
-                                throw new Error('Tool input does not match its schema');
-                            return tool.execute(...args);
-                        } })) }),
-            });
-            const child = makeRunner(domainTools);
-            const agentrun = { name: 'agentrun', label: 'AgentRun workflow', description: 'Describe available tools, inspect a workflow, then run the inspected workflow. Never run an inline candidate. No shell, code, file access or model selection.', parameters: { type: 'object', additionalProperties: false, required: ['action'], properties: { action: { enum: ['describe', 'inspect', 'run'] }, workflow: {}, input: { type: 'object' } } }, async execute(_id, value) {
-                    const args = value;
-                    if (args.action === 'inspect' && args.input !== undefined)
-                        workflowInput = args.input;
-                    if (args.action === 'describe')
-                        return textResult({ tools: domainTools.map(tool => ({ name: tool.name, description: tool.description, parameters: tool.parameters })), skill: true, pi: true, jev: true, running: false, workflow: !!service.inspect().current });
-                    if (args.action === 'inspect') {
-                        try {
-                            if (args.workflow === undefined) {
-                                const current = service.inspect().current;
-                                if (!current)
-                                    throw new Error('Inspect a workflow first');
-                                return textResult({ digest: current.digest, tree: formatWorkflowTree(current.inspection) });
-                            }
-                            const prepared = service.prepare(args.workflow);
-                            candidates.push({ digest: prepared.digest, accepted: true, workflow: prepared.workflow });
-                            return textResult({ digest: prepared.digest, tree: formatWorkflowTree(prepared.inspection) });
-                        }
-                        catch (error) {
-                            candidates.push({ accepted: false, workflow: structuredClone(args.workflow) });
-                            throw error;
-                        }
-                    }
-                    if (args.action !== 'run' || args.workflow !== undefined)
-                        throw new Error('Inspect the workflow first; run without a workflow argument');
-                    notify('outer', 'end');
-                    stage = 'workflow';
-                    notify('workflow', 'start');
-                    try {
-                        const report = await bounded(service.run(args.input ?? workflowInput, { signal, deps: {
-                                // A candidate cannot change the captured host thinking setting.
-                                runNode: request => child({ ...request, thinking: undefined }), runJudge,
-                                runEffect: async (request) => {
-                                    request.signal.throwIfAborted();
-                                    admit('tools');
-                                    const tool = domainTools.find(tool => tool.name === request.node.tool);
-                                    if (!tool || !Compile(tool.parameters).Check(request.input))
-                                        throw new Error('Unsupported tool input');
-                                    return (await bounded(tool.execute('benchmark-effect', request.input, request.signal, undefined, host))).details;
-                                },
-                            } }));
-                        workflowRuns.push({ status: report.status, digest: report.digest, ...(report.output === undefined ? {} : { output: report.output }) });
-                        lastOutput = report.status === 'complete' ? report.output : undefined;
-                        return textResult(report);
-                    }
-                    finally {
-                        notify('workflow', 'end');
-                        stage = 'outer';
-                        notify('outer', 'start');
-                    }
-                } };
-            const outer = makeRunner(arm === 'skill-dsl' ? [...domainTools, agentrun] : domainTools);
-            const output = await bounded(outer({ kind: 'agent', label: 'benchmark-outer',
-                system: [benchmarkInstructions, ...(arm === 'skill-dsl' ? [skillText, 'Use agentrun to inspect and execute your workflow. Deliver its completed output unchanged with submit. The benchmark forbids code, shell, files and changing models. Available task tools and their contracts are returned by agentrun describe.'] : [])],
-                user: JSON.stringify(input), schema: benchmarkReportSchema }));
-            signal.throwIfAborted();
-            if (arm === 'skill-dsl' && (lastOutput === undefined || !isDeepStrictEqual(lastOutput, output)))
-                return finish({ status: 'rejected', output, failure: 'workflow_delivery_missing_or_changed' });
-            const evaluation = evaluatePiBenchmark(item.id, output);
-            return finish({ status: evaluation.passed ? 'passed' : 'rejected', output, evaluation });
-        }
-        catch (error) {
-            return finish({ status: options.signal?.aborted ? 'cancelled' : 'failed', failure: signal.reason instanceof BenchmarkStop ? signal.reason.code : error instanceof PiRunError ? `pi_${error.reason}` : error instanceof ExtensionServiceError ? error.code : 'execution_failed' });
-        }
-        finally {
-            clearTimeout(timer);
-            options.signal?.removeEventListener('abort', externalAbort);
-            stop('arm_finished');
-            void service.dispose();
-        }
-        function finish(result) {
-            const counts = total();
-            notify('outer', 'end', result.status);
-            notify('arm', 'end', result.status);
-            return structuredClone({ arm, ...result, elapsedMs: Math.round(performance.now() - started), total: counts, stages, candidates, workflowRuns, costComplete: counts.costSamples === counts.pi + counts.jev, usageComplete: counts.usageSamples === counts.pi + counts.jev });
-        }
-    }
-}
-/** A development comparison receipt, not automatic promotion. Heldout is a final
- * confirmation set, never the optimization target for this helper. */
-export function comparePiSkillBenchmarks(baseline, candidate) {
-    if (baseline.split !== 'development' || candidate.split !== 'development')
-        throw new Error('Hillclimb comparisons use development only');
-    if (baseline.status !== 'complete' || candidate.status !== 'complete' || baseline.mode !== candidate.mode || !isDeepStrictEqual(baseline.contract, candidate.contract) || !isDeepStrictEqual(baseline.budgets, candidate.budgets) || !isDeepStrictEqual(baseline.runtime, candidate.runtime))
-        throw new Error('Incomparable benchmark receipts');
-    const select = (report) => report.runs.map(run => ({ id: run.caseId, input: run.inputSha256, arm: run.arms.find(arm => arm.arm === 'skill-dsl') }));
-    const before = select(baseline), after = select(candidate);
-    if (before.length !== 3 || after.length !== 3 || !isDeepStrictEqual(before.map(row => [row.id, row.input]), after.map(row => [row.id, row.input])) || [...before, ...after].some(row => !row.arm))
-        throw new Error('Incomplete benchmark receipts');
-    const regressions = before.filter((row, i) => row.arm.status === 'passed' && after[i].arm.status !== 'passed').map(row => row.id);
-    const sums = (rows) => ({ passed: rows.filter(row => row.arm.status === 'passed').length, requests: rows.reduce((sum, row) => sum + row.arm.total.pi + row.arm.total.jev, 0), tokens: rows.reduce((sum, row) => sum + row.arm.total.tokens, 0), usageComplete: rows.every(row => row.arm.usageComplete), knownCostUsd: rows.reduce((sum, row) => sum + row.arm.total.knownCostUsd, 0), costComplete: rows.every(row => row.arm.costComplete) });
-    const previous = sums(before), current = sums(after);
-    const improved = !regressions.length && (current.passed > previous.passed || current.passed === previous.passed && current.passed === 3 && previous.usageComplete && current.usageComplete && current.tokens < previous.tokens);
-    return { mode: candidate.mode, baselineSkillSha256: baseline.skillSha256, candidateSkillSha256: candidate.skillSha256, regressions, previous, current, improved, eligibleForLiveHeldout: candidate.mode === 'live' && improved && current.passed === 3 };
-}

package/dist/comparison.d.ts

@@ -1,69 +0,0 @@
-import { type WorkflowDeps } from '@parcha/agentrun-dsl';
-import type { PiHostContext } from './types.js';
-export declare function evaluatePiComparison(caseId: string, output: unknown): {
-    passed: boolean;
-    issues: string[];
-};
-export interface PiComparisonBudgets {
-    timeoutMs: number;
-    maxModelRequests: number;
-    maxToolCalls: number;
-    maxTurns: number;
-    maxSubmissions: number;
-}
-export interface PiComparisonOptions {
-    host: PiHostContext;
-    runJudge: NonNullable<WorkflowDeps['runJudge']>;
-    signal?: AbortSignal;
-    mode: 'live' | 'scripted-transport';
-    budgets?: Partial<PiComparisonBudgets>;
-}
-export interface PiComparisonArm {
-    arm: 'plain-pi' | 'workflow';
-    status: 'passed' | 'rejected' | 'failed' | 'cancelled';
-    elapsedMs: number;
-    requests: {
-        pi: number;
-        jev: number;
-        total: number;
-    };
-    toolCalls: number;
-    submissions: number;
-    tokens: {
-        pi: number;
-        jev: number;
-        piUsageSamples: number;
-        jevUsageSamples: number;
-    };
-    output?: unknown;
-    evaluation?: ReturnType<typeof evaluatePiComparison>;
-    failure?: string;
-}
-export declare function runPiComparison(options: PiComparisonOptions): Promise<{
-    mode: "live" | "scripted-transport";
-    status: "complete" | "cancelled";
-    passed: boolean;
-    runtime: {
-        provider: string;
-        model: string;
-        api: string;
-        thinking: import("./types.js").PiThinkingLevel;
-    };
-    budgets: {
-        timeoutMs: number;
-        maxModelRequests: number;
-        maxToolCalls: number;
-        maxTurns: number;
-        maxSubmissions: number;
-    };
-    workflowSha256: string;
-    suiteSha256: string;
-    runs: {
-        caseId: string;
-        inputSha256: string;
-        expectedSha256: string;
-        executionOrder: string[];
-        arms: PiComparisonArm[];
-    }[];
-    limits: string[];
-}>;

package/dist/comparison.js

@@ -1,229 +0,0 @@
-import { createHash } from 'node:crypto';
-import { Compile } from 'typebox/compile';
-import { runWorkflow, runWorkflowSlice, workflowSha256 } from '@parcha/agentrun-dsl';
-import { createPiHostSessionFactory } from './host-session.js';
-import { createPiRunner, PiRunError } from './runner.js';
-import { demoData } from './demo-data.js';
-const cases = [
-    { id: 'single-fact', question: 'Can our contributors edit documentation in a browser?',
-        topics: [{ id: 'editing', question: 'Can contributors edit documentation in a browser?' }],
-        sources: [{ id: 'browser-trial', text: 'In our trial, all three contributors successfully edited a documentation page in the browser.' }, { id: 'advert', text: 'A beautiful new logo for documentation.' }],
-        expected: [{ topicId: 'editing', status: 'supported', sourceIds: ['browser-trial'] }] },
-    { id: 'parallel-research', question: 'What would change if our team moved its wiki into the code repository?',
-        topics: [{ id: 'reviews', question: demoData.subquestions[0] }, { id: 'editing', question: demoData.subquestions[1] }, { id: 'search', question: demoData.subquestions[2] }],
-        sources: structuredClone(demoData.sources),
-        expected: [{ topicId: 'reviews', status: 'supported', sourceIds: ['review-guide'] }, { topicId: 'editing', status: 'supported', sourceIds: ['editor-trial'] }, { topicId: 'search', status: 'supported', sourceIds: ['search-trial'] }] },
-    { id: 'gaps-and-conflicts', question: 'Does our repository trial support restricted editing and synonym search?',
-        topics: [{ id: 'permissions', question: 'Who can edit restricted documentation pages in our trial?' }, { id: 'search', question: 'Did synonym search work in our trial?' }],
-        sources: [{ id: 'permissions-heading', text: 'Restricted documentation permissions: details to be supplied.' }, { id: 'search-positive', text: 'The trial report says all ten synonym searches found the intended pages.' }, { id: 'search-negative', text: 'The audit of the same trial says none of those ten synonym searches found the intended pages.' }],
-        expected: [{ topicId: 'permissions', status: 'insufficient', sourceIds: [] }, { topicId: 'search', status: 'conflicting', sourceIds: ['search-positive', 'search-negative'] }] },
-];
-const str = { type: 'string', minLength: 1 };
-const findingSchema = { type: 'object', additionalProperties: false, required: ['topicId', 'status', 'summary', 'sourceIds'], properties: {
-        topicId: str, status: { enum: ['supported', 'conflicting', 'insufficient'] }, summary: str, sourceIds: { type: 'array', uniqueItems: true, items: str },
-    } };
-const reportSchema = { type: 'object', additionalProperties: false, required: ['answer', 'findings'], properties: {
-        answer: str, findings: { type: 'array', minItems: 1, items: findingSchema },
-    } };
-const sourcesSchema = { type: 'object', required: ['sources'], additionalProperties: false, properties: { sources: { type: 'array', items: { type: 'object', required: ['id', 'text'], additionalProperties: false, properties: { id: str, text: str } } } } };
-const evidenceSchema = demoData.workflow.root.steps[1].body.workflow.schemas.EvidenceDecision;
-const instructions = 'Use only the supplied fictional corpus. For each topic preserve all material sources, including contrary evidence, conditions and uncertainty. Status supported means the sources establish a finding, not necessarily a positive answer; conflicting means incompatible accounts; insufficient means no concrete information answers the topic. Cite source IDs. Do not turn qualified evidence into unconditional claims. You may read_sources or screen_evidence with Jev System One decisions. Submit the requested structured output.';
-const screen = { node: 'sift', label: 'screen-evidence', itemsPath: 'search.sources', state: { subquestion: '{item.question}' }, out: 'EvidenceDecision', as: 'evidence', keep: { path: 'answersQuestion', gte: 0.8 } };
-const workflow = {
-    v: 2, name: 'comparison-research', schemas: { Input: { type: 'object' }, Sources: sourcesSchema, Finding: findingSchema, Report: reportSchema, EvidenceDecision: evidenceSchema },
-    input: { schemaId: 'Input' }, output: { schemaId: 'Report', path: 'report' },
-    root: { node: 'chain', steps: [
-            { node: 'map', label: 'research-topics', itemsPath: 'topics', maxConcurrency: 3, resultPath: 'finding', as: 'findings', body: { node: 'chain', steps: [
-                        { node: 'call', label: 'read-sources', via: 'tool', tool: 'read_sources', args: {}, out: 'Sources', as: 'search', deadline_s: 15 }, screen,
-                        { node: 'agent', label: 'finding', instructions, state: { topic: '{item}', sources: '{evidence.items}' }, out: 'Finding', as: 'finding' },
-                    ] } },
-            { node: 'agent', label: 'report', instructions, state: { question: '{question}', topics: '{topics}', findings: '{findings}' }, out: 'Report', as: 'report' },
-        ] },
-};
-const selector = { ...workflow, name: 'comparison-screen', root: { node: 'chain', steps: [screen] } };
-const hash = (value) => createHash('sha256').update(JSON.stringify(value)).digest('hex');
-const validator = Compile(reportSchema);
-export function evaluatePiComparison(caseId, output) {
-    const item = cases.find(row => row.id === caseId);
-    if (!item)
-        throw new Error('Unknown comparison case');
-    if (!validator.Check(output))
-        return { passed: false, issues: ['output_schema'] };
-    const findings = output.findings;
-    const issues = [];
-    if (findings.length !== item.expected.length || new Set(findings.map(row => row.topicId)).size !== findings.length)
-        issues.push('topic_coverage');
-    for (const expected of item.expected) {
-        const actual = findings.find(row => row.topicId === expected.topicId);
-        if (!actual || actual.status !== expected.status)
-            issues.push(`${expected.topicId}:status`);
-        if (!actual || JSON.stringify([...actual.sourceIds].sort()) !== JSON.stringify([...expected.sourceIds].sort()))
-            issues.push(`${expected.topicId}:citations`);
-    }
-    return { passed: issues.length === 0, issues };
-}
-class ComparisonStop extends Error {
-    code;
-    constructor(code) {
-        super(code);
-        this.code = code;
-    }
-}
-const defaults = { timeoutMs: 120_000, maxModelRequests: 12, maxToolCalls: 24, maxTurns: 12, maxSubmissions: 3 };
-export async function runPiComparison(options) {
-    if (options.mode !== 'live' && options.mode !== 'scripted-transport')
-        throw new Error('Comparison requires an explicit execution mode');
-    if (typeof options.runJudge !== 'function')
-        throw new Error('Both arms require the same Jev capability');
-    const budgets = { ...defaults, ...options.budgets };
-    for (const key of Object.keys(defaults)) {
-        if (!Number.isSafeInteger(budgets[key]) || budgets[key] < 1 || budgets[key] > defaults[key])
-            throw new Error(`Comparison budget ${key} must be between 1 and ${defaults[key]}`);
-    }
-    const originalRegistry = options.host.modelRegistry;
-    const capturedHost = { ...options.host, modelRegistry: originalRegistry };
-    createPiHostSessionFactory(capturedHost);
-    const model = capturedHost.model;
-    const runs = [];
-    for (const [index, item] of cases.entries()) {
-        if (options.signal?.aborted)
-            break;
-        const order = index % 2 ? ['workflow', 'plain-pi'] : ['plain-pi', 'workflow'];
-        const arms = [];
-        for (const arm of order) {
-            if (options.signal?.aborted)
-                break;
-            arms.push(await runArm(item, arm));
-        }
-        runs.push({ caseId: item.id, inputSha256: hash({ question: item.question, topics: item.topics, sources: item.sources }), expectedSha256: hash(item.expected), executionOrder: arms.map(arm => arm.arm), arms });
-    }
-    return {
-        mode: options.mode, status: options.signal?.aborted ? 'cancelled' : 'complete',
-        passed: runs.length === cases.length && runs.every(run => run.arms.length === 2 && run.arms.every(arm => arm.status === 'passed')),
-        runtime: { provider: model.provider, model: model.id, api: model.api, thinking: capturedHost.thinkingLevel ?? 'low' },
-        budgets, workflowSha256: workflowSha256(workflow), suiteSha256: hash(cases), runs,
-        limits: ['Three fictional cases, one run per arm; not a statistical benchmark.', 'Workflow authoring and setup costs excluded; both arms receive identical prewritten topics.', 'Both arms can use the same corpus reader and Jev rubric; the plain loop chooses whether to call them.', 'Evaluation checks topic status and citation sets, not arbitrary prose entailment.', 'Token totals cover reported usage only; compare usage sample counts with request counts.', 'Host caches are not reset; execution order alternates.'],
-    };
-    async function runArm(item, arm) {
-        const started = performance.now();
-        const controller = new AbortController();
-        const signal = controller.signal;
-        const abort = (code) => { if (!signal.aborted)
-            controller.abort(new ComparisonStop(code)); };
-        const externalAbort = () => abort('cancelled');
-        options.signal?.addEventListener('abort', externalAbort, { once: true });
-        if (options.signal?.aborted)
-            externalAbort();
-        const timer = setTimeout(() => abort('deadline'), budgets.timeoutMs);
-        let rejectAbort;
-        const interrupted = new Promise((_resolve, reject) => { rejectAbort = reject; });
-        const onAbort = () => rejectAbort(signal.reason);
-        signal.addEventListener('abort', onAbort, { once: true });
-        if (signal.aborted)
-            onAbort();
-        void interrupted.catch(() => { });
-        const bounded = (work) => Promise.race([work, interrupted]);
-        const requests = { pi: 0, jev: 0, total: 0 };
-        const tokens = { pi: 0, jev: 0, piUsageSamples: 0, jevUsageSamples: 0 };
-        let toolCalls = 0, submissions = 0;
-        const admit = (kind) => {
-            signal.throwIfAborted();
-            if (kind === 'tool') {
-                if (toolCalls >= budgets.maxToolCalls)
-                    abort('tool_call_limit');
-                signal.throwIfAborted();
-                toolCalls++;
-            }
-            else {
-                if (requests.total >= budgets.maxModelRequests)
-                    abort('model_request_limit');
-                signal.throwIfAborted();
-                requests.total++;
-                requests[kind]++;
-            }
-        };
-        const host = { ...capturedHost, modelRegistry: {
-                getAll: () => originalRegistry.getAll(),
-                streamSimple(selected, transcript, streamOptions) {
-                    admit('pi');
-                    const stream = originalRegistry.streamSimple(selected, transcript, streamOptions);
-                    void stream.result().then(result => {
-                        const usage = result?.usage;
-                        if (Number.isFinite(usage?.totalTokens) && usage.totalTokens >= 0) {
-                            tokens.pi += usage.totalTokens;
-                            tokens.piUsageSamples++;
-                        }
-                    }).catch(() => { });
-                    return stream;
-                },
-            } };
-        const runJudge = async (request) => {
-            request.signal?.throwIfAborted();
-            admit('jev');
-            const result = await bounded(options.runJudge({ ...request, signal: request.signal ? AbortSignal.any([signal, request.signal]) : signal }));
-            if (result.usage && Number.isFinite(result.usage.input_tokens) && Number.isFinite(result.usage.output_tokens) && result.usage.input_tokens >= 0 && result.usage.output_tokens >= 0) {
-                tokens.jev += result.usage.input_tokens + result.usage.output_tokens;
-                tokens.jevUsageSamples++;
-            }
-            return result;
-        };
-        const read = () => ({ sources: structuredClone(item.sources) });
-        const tools = [
-            { name: 'read_sources', label: 'Read sources', description: 'Read the complete fictional corpus for this task.', parameters: { type: 'object', properties: {}, additionalProperties: false },
-                async execute() { const value = read(); return { content: [{ type: 'text', text: JSON.stringify(value) }], details: value }; } },
-            { name: 'screen_evidence', label: 'Screen evidence', description: 'Use Jev System One decisions to retain concrete evidence for a topic, including qualified and contrary evidence. Same rubric and 0.8 threshold as the workflow.',
-                parameters: { type: 'object', properties: { topicId: str, sourceIds: { type: 'array', uniqueItems: true, items: str } }, required: ['topicId', 'sourceIds'], additionalProperties: false },
-                async execute(_id, value, toolSignal) {
-                    const args = value;
-                    const topic = item.topics.find(row => row.id === args.topicId);
-                    if (!topic || args.sourceIds.some(id => !item.sources.some(source => source.id === id)))
-                        throw new Error('Unknown topic or source');
-                    const selected = item.sources.filter(source => args.sourceIds.includes(source.id));
-                    const result = await bounded(runWorkflowSlice(selector, {}, { from: 'screen-evidence', seed: { item: topic, search: { sources: selected } } }, { runJudge: request => runJudge({ ...request, signal: toolSignal }) }));
-                    if (result.status !== 'complete')
-                        throw new Error('Evidence screening did not complete');
-                    const evidence = result.state.evidence;
-                    return { content: [{ type: 'text', text: JSON.stringify(evidence) }], details: evidence };
-                } },
-        ];
-        try {
-            const factory = createPiHostSessionFactory(host);
-            const runNode = createPiRunner({ model, thinkingLevel: capturedHost.thinkingLevel, cwd: capturedHost.cwd, signal,
-                maxTurns: budgets.maxTurns, maxSubmissions: budgets.maxSubmissions, timeoutMs: budgets.timeoutMs, tools,
-                sessionFactory: opts => factory({ ...opts, customTools: opts.customTools.map(tool => ({ ...tool,
-                        async execute(...args) { args[2]?.throwIfAborted(); admit('tool'); if (tool.name === 'submit')
-                            submissions++; return tool.execute(...args); },
-                    })) }),
-            });
-            const input = { question: item.question, topics: structuredClone(item.topics) };
-            const output = await bounded((async () => {
-                if (arm === 'plain-pi')
-                    return runNode({ kind: 'agent', label: 'plain-pi', system: [instructions], user: JSON.stringify(input), schema: reportSchema });
-                const result = await runWorkflow(workflow, input, { runNode, runJudge, runEffect: async ({ node }) => {
-                        if (node.tool !== 'read_sources')
-                            throw new Error('Unknown comparison tool');
-                        admit('tool');
-                        return read();
-                    } });
-                if (result.status !== 'complete')
-                    throw new Error('Workflow did not complete');
-                return result.output;
-            })());
-            signal.throwIfAborted();
-            const evaluation = evaluatePiComparison(item.id, output);
-            return finish({ status: evaluation.passed ? 'passed' : 'rejected', output, evaluation });
-        }
-        catch (error) {
-            return finish({ status: options.signal?.aborted ? 'cancelled' : 'failed', failure: signal.reason instanceof ComparisonStop ? signal.reason.code : error instanceof PiRunError ? `pi_${error.reason}` : 'execution_failed' });
-        }
-        finally {
-            clearTimeout(timer);
-            options.signal?.removeEventListener('abort', externalAbort);
-            signal.removeEventListener('abort', onAbort);
-            abort('arm_finished');
-        }
-        function finish(result) {
-            return { arm, ...result, elapsedMs: Math.round(performance.now() - started), requests: { ...requests }, toolCalls, submissions, tokens: { ...tokens } };
-        }
-    }
-}

package/dist/skill-bundle.d.ts

@@ -1,5 +0,0 @@
-/** Shipped with the extension, including hosts without filesystem-reading tools. */
-export declare function loadPiJevGuide(): string;
-/** The same core contract for skill-enabled and tool-only Pi hosts. */
-export declare function loadPiWorkflowGuide(): string;
-export declare function loadPiAuthorSkillBundle(): string;

package/dist/skill-bundle.js

@@ -1,15 +0,0 @@
-import { readFileSync, readdirSync } from 'node:fs';
-/** Shipped with the extension, including hosts without filesystem-reading tools. */
-export function loadPiJevGuide() {
-    return readFileSync(new URL('../skills/author/references/jev-decisions.md', import.meta.url), 'utf8');
-}
-/** The same core contract for skill-enabled and tool-only Pi hosts. */
-export function loadPiWorkflowGuide() {
-    return readFileSync(new URL('../skills/author/references/workflow-format.md', import.meta.url), 'utf8');
-}
-export function loadPiAuthorSkillBundle() {
-    const root = new URL('../skills/author/', import.meta.url);
-    const files = ['SKILL.md', 'references/workflow-format.md', 'references/jev-decisions.md', ...readdirSync(new URL('examples/', root))
-            .filter(name => name.endsWith('.json')).sort().map(name => `examples/${name}`)];
-    return files.map(path => `\n--- ${path} ---\n${readFileSync(new URL(path, root), 'utf8')}`).join('\n');
-}