@parcha/agentrun-pi 0.1.0-beta.2 → 0.1.0-beta.4

package/dist/admission.js

@@ -1,51 +0,0 @@
-export function workflowPolicyErrors(candidate, options) {
-    const errors = [];
-    const sections = Object.keys(options.rubricSections ?? {});
-    const record = (value) => value !== null && typeof value === "object" && !Array.isArray(value) ? value : undefined;
-    const visit = (value) => {
-        const node = record(value);
-        if (!node)
-            return;
-        const kind = node.node;
-        if (!options.allowExecutableCandidates && ["code", "call", "artifact"].includes(String(kind)) && !(kind === "call" && node.via === "tool" && typeof node.tool === "string" && options.allowedEffectTools?.includes(node.tool)))
-            errors.push(`${kind} requires allowExecutableCandidates`);
-        if (sections.length && ["judge", "pick", "sift", "route"].includes(String(kind)))
-            errors.push(`${String(kind)} cannot carry supplied SOP sections; use an LLM node with sopSection`);
-        const llm = ["agent", "decide", "extract", "report"].includes(String(kind));
-        const predicate = kind === "loop" ? record(node.until) : kind === "escalate" ? record(node.when) : undefined;
-        if (sections.length && (predicate?.predicate === "ask" || (llm && node.verify !== undefined)))
-            errors.push("Semantic predicates and verify clauses require a separately reviewed question contract when rubric sections are supplied");
-        const proseArtifact = kind === "artifact" && (node.type === "markdown" || node.type === "report");
-        if (llm || proseArtifact) {
-            const included = Array.isArray(node.sopSection) ? node.sopSection : [node.sopSection];
-            for (const section of sections)
-                if (!included.includes(section))
-                    errors.push(`${String(node.label)} must include rubric section ${section}`);
-        }
-        switch (kind) {
-            case "chain":
-                if (Array.isArray(node.steps))
-                    node.steps.forEach(visit);
-                break;
-            case "parallel":
-                if (Array.isArray(node.branches))
-                    node.branches.forEach(visit);
-                break;
-            case "map":
-            case "loop":
-                visit(node.body);
-                break;
-            case "route": {
-                const branches = record(node.branches);
-                if (branches)
-                    Object.values(branches).forEach(branch => visit(record(branch)?.body));
-                break;
-            }
-            case "workflow":
-                visit(record(node.workflow)?.root);
-                break;
-        }
-    };
-    visit(record(candidate)?.root);
-    return errors;
-}

package/dist/author.d.ts

@@ -1,21 +0,0 @@
-import { type Workflow } from "@parcha/agentrun-dsl";
-import { type PiRunnerOptions } from "./runner.js";
-export declare const AUTHOR_CONTRACT = "You author AgentRun DSL v2 workflows as JSON data.\nDeliver with the submit tool, never prose or a filename. Make the smallest workflow satisfying the request.\nThe document has v:2, name, schemas (JSON Schema catalog), optional input:{schemaId}, output:{schemaId,path?}, root.\nEvery referenced schema must exist. Each LLM node has node:agent|decide|extract, label, instructions, out (schema id), as (state key), optional requires/tools/sopSection.\nA chain has node:chain, steps:[nodes]. A map has label, itemsPath, body, as, maxConcurrency. A loop has label, body, until, maxIters.\nA judge has label,state,out,as. Judge schemas use flat boolean/enum/integer properties; each property's description states its question. A route has label,state,instructions,branches:{name:{criteria,body}}, optional unsure:{branch,gte} and as.\nLLM instructions are literal text, never interpolated. The node receives workflow state as a JSON user message; refer to its keys in ordinary prose and declare required keys in requires. Use {stateKey} placeholders only in interpolated fields such as judge.state and call.args; whole placeholders preserve their value's type. Output.path selects a final state value validated by output.schemaId.\nOnly use tools named by the host. Never invent a tool, capability, evidence source or successful test.\nKeep the full rubric in its original source text. Every LLM node's sopSection must include ALL supplied rubric section names. Merged judgments inherit their parents' union. Do not paraphrase policy into instructions. If rubric sections are supplied, use LLM nodes for judgment; Jev nodes do not carry sopSection and require a separately reviewed question contract.\nKeep independent acceptance checks host-owned; never change their implementation, fixtures or thresholds to pass. A valid candidate is not automatically activated.\nCode, shell/effect calls and artifact delivery require explicit host authorization. Bound loops and parallelism; uncertainty should escalate or select an explicit fallback.\nExample: {\"v\":2,\"name\":\"summarize\",\"schemas\":{\"Result\":{\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"string\"}},\"required\":[\"summary\"],\"additionalProperties\":false}},\"output\":{\"schemaId\":\"Result\",\"path\":\"result\"},\"root\":{\"node\":\"extract\",\"label\":\"summarize\",\"instructions\":\"Summarize the text field from the JSON input faithfully.\",\"out\":\"Result\",\"as\":\"result\",\"requires\":[\"text\"]}}";
-export interface AuthorWorkflowOptions {
-    request: string;
-    outputDir: string;
-    pi: PiRunnerOptions;
-    inputKeys?: string[];
-    rubricSections?: Record<string, string>;
-    allowExecutableCandidates?: boolean;
-    maxCandidates?: number;
-    acceptance?: (candidate: Workflow) => Promise<string[]> | string[];
-}
-export interface AuthoredWorkflow {
-    workflow: Workflow;
-    path: string;
-    directory: string;
-    candidates: number;
-    checks: "structural" | "structural-and-host";
-}
-export declare function authorWorkflow(options: AuthorWorkflowOptions): Promise<AuthoredWorkflow>;

package/dist/author.js

@@ -1,72 +0,0 @@
-import { workflowPolicyErrors } from "./admission.js";
-import { randomUUID } from "node:crypto";
-import { mkdir, writeFile } from "node:fs/promises";
-import { resolve, join } from "node:path";
-import { validateWorkflow } from "@parcha/agentrun-dsl";
-import { createPiRunner } from "./runner.js";
-export const AUTHOR_CONTRACT = `You author AgentRun DSL v2 workflows as JSON data.
-Deliver with the submit tool, never prose or a filename. Make the smallest workflow satisfying the request.
-The document has v:2, name, schemas (JSON Schema catalog), optional input:{schemaId}, output:{schemaId,path?}, root.
-Every referenced schema must exist. Each LLM node has node:agent|decide|extract, label, instructions, out (schema id), as (state key), optional requires/tools/sopSection.
-A chain has node:chain, steps:[nodes]. A map has label, itemsPath, body, as, maxConcurrency. A loop has label, body, until, maxIters.
-A judge has label,state,out,as. Judge schemas use flat boolean/enum/integer properties; each property's description states its question. A route has label,state,instructions,branches:{name:{criteria,body}}, optional unsure:{branch,gte} and as.
-LLM instructions are literal text, never interpolated. The node receives workflow state as a JSON user message; refer to its keys in ordinary prose and declare required keys in requires. Use {stateKey} placeholders only in interpolated fields such as judge.state and call.args; whole placeholders preserve their value's type. Output.path selects a final state value validated by output.schemaId.
-Only use tools named by the host. Never invent a tool, capability, evidence source or successful test.
-Keep the full rubric in its original source text. Every LLM node's sopSection must include ALL supplied rubric section names. Merged judgments inherit their parents' union. Do not paraphrase policy into instructions. If rubric sections are supplied, use LLM nodes for judgment; Jev nodes do not carry sopSection and require a separately reviewed question contract.
-Keep independent acceptance checks host-owned; never change their implementation, fixtures or thresholds to pass. A valid candidate is not automatically activated.
-Code, shell/effect calls and artifact delivery require explicit host authorization. Bound loops and parallelism; uncertainty should escalate or select an explicit fallback.
-Example: {"v":2,"name":"summarize","schemas":{"Result":{"type":"object","properties":{"summary":{"type":"string"}},"required":["summary"],"additionalProperties":false}},"output":{"schemaId":"Result","path":"result"},"root":{"node":"extract","label":"summarize","instructions":"Summarize the text field from the JSON input faithfully.","out":"Result","as":"result","requires":["text"]}}`;
-export async function authorWorkflow(options) {
-    if (!options.request.trim())
-        throw new Error("Workflow request is required");
-    const maxCandidates = options.maxCandidates ?? 4;
-    if (!Number.isSafeInteger(maxCandidates) || maxCandidates < 1)
-        throw new Error("maxCandidates must be a positive integer");
-    const directory = resolve(options.outputDir, `candidate-${randomUUID()}`);
-    await mkdir(directory, { recursive: true });
-    await writeFile(join(directory, "request.json"), JSON.stringify({ request: options.request, inputKeys: options.inputKeys ?? [], rubricSections: options.rubricSections ?? {}, allowExecutableCandidates: options.allowExecutableCandidates ?? false, maxCandidates }, null, 2), { flag: "wx" });
-    let candidates = 0;
-    let acceptedPath;
-    const runner = createPiRunner({ ...options.pi, tools: [], maxSubmissions: maxCandidates });
-    try {
-        const workflow = await runner({
-            kind: "agent", label: "author-workflow", tools: [],
-            system: [AUTHOR_CONTRACT, `Executable candidates authorized: ${options.allowExecutableCandidates ?? false}.`, `All supplied rubric sections (authoritative): ${JSON.stringify(options.rubricSections ?? {})}`],
-            user: `${options.request}\nAvailable input keys: ${JSON.stringify(options.inputKeys ?? [])}`,
-            schema: { type: "object", additionalProperties: true },
-            async review(value) {
-                candidates++;
-                if (candidates > maxCandidates)
-                    throw new Error("Candidate limit exceeded");
-                const candidatePath = join(directory, `${String(candidates).padStart(3, "0")}.json`);
-                await writeFile(candidatePath, JSON.stringify(value, null, 2), { flag: "wx" });
-                const errors = workflowPolicyErrors(value, options);
-                if (!errors.length) {
-                    try {
-                        const verdict = validateWorkflow(value, { inputKeys: options.inputKeys });
-                        if (!verdict.ok)
-                            errors.push(...verdict.errors);
-                    }
-                    catch (error) {
-                        errors.push(`Invalid workflow: ${error instanceof Error ? error.message : String(error)}`);
-                    }
-                }
-                if (!errors.length && options.acceptance)
-                    errors.push(...await options.acceptance(structuredClone(value)));
-                await writeFile(join(directory, `${String(candidates).padStart(3, "0")}.review.json`), JSON.stringify({ accepted: !errors.length, checks: options.acceptance ? "structural-and-host" : "structural", errors }, null, 2), { flag: "wx" });
-                if (errors.length)
-                    return { accepted: false, message: errors.join("\n") };
-                acceptedPath = candidatePath;
-                return { accepted: true };
-            },
-        });
-        if (!acceptedPath)
-            throw new Error("Author returned without a retained accepted candidate");
-        await writeFile(join(directory, "result.json"), JSON.stringify({ status: "candidate", path: acceptedPath, candidates }, null, 2), { flag: "wx" });
-        return { workflow, path: acceptedPath, directory, candidates, checks: options.acceptance ? "structural-and-host" : "structural" };
-    }
-    catch (error) {
-        await writeFile(join(directory, "result.json"), JSON.stringify({ status: "failed", candidates, message: error instanceof Error ? error.message : String(error) }, null, 2), { flag: "wx" });
-        throw error;
-    }
-}

package/dist/benchmark-contract.d.ts

@@ -1,88 +0,0 @@
-export type BenchmarkSplit = 'development' | 'heldout';
-export type BenchmarkFinding = {
-    topicId: string;
-    status: 'supported' | 'conflicting' | 'insufficient';
-    summary: string;
-    sourceIds: string[];
-};
-export type BenchmarkCase = {
-    id: string;
-    split: BenchmarkSplit;
-    question: string;
-    topics: {
-        id: string;
-        question: string;
-    }[];
-    sources: {
-        id: string;
-        text: string;
-    }[];
-    expected: Omit<BenchmarkFinding, 'summary'>[];
-};
-export declare const benchmarkHash: (value: unknown) => string;
-export declare const benchmarkInstructions = "Answer using only the fictional source corpus. Produce exactly one finding for each supplied topic. Status supported means concrete evidence establishes a finding, including a negative or qualified answer. Conflicting means incompatible accounts. Insufficient means no concrete information answers the question. Preserve conditions and contrary evidence; cite every material source by ID. Do not invent facts or citations. Submit the final structured report.";
-export declare const benchmarkReportSchema: {
-    type: string;
-    additionalProperties: boolean;
-    required: string[];
-    properties: {
-        findings: {
-            type: string;
-            minItems: number;
-            items: {
-                type: string;
-                additionalProperties: boolean;
-                required: string[];
-                properties: {
-                    topicId: {
-                        type: string;
-                        minLength: number;
-                    };
-                    status: {
-                        enum: string[];
-                    };
-                    summary: {
-                        type: string;
-                        minLength: number;
-                    };
-                    sourceIds: {
-                        type: string;
-                        uniqueItems: boolean;
-                        items: {
-                            type: string;
-                            minLength: number;
-                        };
-                    };
-                };
-            };
-        };
-    };
-};
-export declare const benchmarkEvidenceSchema: {
-    type: string;
-    additionalProperties: boolean;
-    required: string[];
-    properties: {
-        relevant: {
-            type: string;
-            description: string;
-            criteria: {
-                true: string;
-                false: string;
-            };
-        };
-    };
-};
-export declare function benchmarkCases(split: BenchmarkSplit): BenchmarkCase[];
-/** Frozen independent checks. Never return these results to an authoring agent. */
-export declare function evaluatePiBenchmark(caseId: string, output: unknown): {
-    passed: boolean;
-    issues: string[];
-};
-export declare function benchmarkContract(): {
-    version: number;
-    fixturesSha256: string;
-    verifierSha256: string;
-    outputSchemaSha256: string;
-    instructionsSha256: string;
-};

package/dist/benchmark-contract.js

@@ -1,50 +0,0 @@
-import { createHash } from 'node:crypto';
-import { readFileSync } from 'node:fs';
-import { Compile } from 'typebox/compile';
-const cases = [
-    { id: 'docs-editing', split: 'development', question: 'Can contributors edit our docs in the browser?', topics: [{ id: 'editing', question: 'Did contributors edit a page in the browser during our trial?' }], sources: [{ id: 'd1', text: 'All three contributors edited a page in the browser during the trial.' }, { id: 'd2', text: 'The documentation product has a new logo.' }], expected: [{ topicId: 'editing', status: 'supported', sourceIds: ['d1'] }] },
-    { id: 'migration', split: 'development', question: 'What changes with a repository-based wiki?', topics: [{ id: 'review', question: 'How are changes reviewed?' }, { id: 'editing', question: 'Could non-developers edit documentation?' }, { id: 'search', question: 'What did the search trial find?' }], sources: [{ id: 'm1', text: 'Documentation changes require an assigned reviewer to approve the pull request.' }, { id: 'm2', text: 'Non-developers edited in the browser but two needed help with merge conflicts.' }, { id: 'm3', text: 'Search found exact words but missed synonyms during our trial.' }, { id: 'm4', text: 'A revolutionary documentation experience.' }], expected: [{ topicId: 'review', status: 'supported', sourceIds: ['m1'] }, { topicId: 'editing', status: 'supported', sourceIds: ['m2'] }, { topicId: 'search', status: 'supported', sourceIds: ['m3'] }] },
-    { id: 'missing-conflicting', split: 'development', question: 'Can we rely on restricted editing and synonym search?', topics: [{ id: 'permissions', question: 'Who can edit restricted pages?' }, { id: 'search', question: 'Did synonym search work in this trial?' }], sources: [{ id: 'c1', text: 'Restricted-page permissions: details pending.' }, { id: 'c2', text: 'The trial report says all ten synonym searches found their target pages.' }, { id: 'c3', text: 'An audit of the same trial says none of those ten searches found their target pages.' }], expected: [{ topicId: 'permissions', status: 'insufficient', sourceIds: [] }, { topicId: 'search', status: 'conflicting', sourceIds: ['c2', 'c3'] }] },
-    { id: 'backup', split: 'heldout', question: 'Did the restore drill recover the archive?', topics: [{ id: 'restore', question: 'Was the archive recovered during the drill?' }], sources: [{ id: 'h1', text: 'The drill restored the archive and every file matched its recorded checksum.' }, { id: 'h2', text: 'Backups that give you peace of mind.' }], expected: [{ topicId: 'restore', status: 'supported', sourceIds: ['h1'] }] },
-    { id: 'queue-adoption', split: 'heldout', question: 'What should we know before adopting this task queue?', topics: [{ id: 'retries', question: 'What happens to jobs after worker failure?' }, { id: 'ordering', question: 'Is task order guaranteed?' }, { id: 'visibility', question: 'Can operators inspect stuck jobs?' }], sources: [{ id: 'q1', text: 'In the failure test, an unfinished job returned to the queue when its lease expired.' }, { id: 'q2', text: 'Tasks from different workers completed out of submission order in the trial.' }, { id: 'q3', text: 'Operators used the dashboard to inspect job age and the last worker error.' }, { id: 'q4', text: 'Supercharge your asynchronous future.' }], expected: [{ topicId: 'retries', status: 'supported', sourceIds: ['q1'] }, { topicId: 'ordering', status: 'supported', sourceIds: ['q2'] }, { topicId: 'visibility', status: 'supported', sourceIds: ['q3'] }] },
-    { id: 'cache-evidence', split: 'heldout', question: 'What do the cache trials tell us about deletion and encryption?', topics: [{ id: 'deletion', question: 'Did deleted entries disappear immediately in the trial?' }, { id: 'encryption', question: 'Which encryption protects cached entries?' }], sources: [{ id: 'k1', text: 'The trial log says every deleted entry disappeared immediately.' }, { id: 'k2', text: 'A second observer of that same trial recorded deleted entries remaining readable for a full minute.' }, { id: 'k3', text: 'Encryption: documentation forthcoming.' }], expected: [{ topicId: 'deletion', status: 'conflicting', sourceIds: ['k1', 'k2'] }, { topicId: 'encryption', status: 'insufficient', sourceIds: [] }] },
-];
-export const benchmarkHash = (value) => createHash('sha256').update(JSON.stringify(value)).digest('hex');
-export const benchmarkInstructions = 'Answer using only the fictional source corpus. Produce exactly one finding for each supplied topic. Status supported means concrete evidence establishes a finding, including a negative or qualified answer. Conflicting means incompatible accounts. Insufficient means no concrete information answers the question. Preserve conditions and contrary evidence; cite every material source by ID. Do not invent facts or citations. Submit the final structured report.';
-const str = { type: 'string', minLength: 1 };
-export const benchmarkReportSchema = { type: 'object', additionalProperties: false, required: ['findings'], properties: { findings: { type: 'array', minItems: 1, items: { type: 'object', additionalProperties: false, required: ['topicId', 'status', 'summary', 'sourceIds'], properties: { topicId: str, status: { enum: ['supported', 'conflicting', 'insufficient'] }, summary: str, sourceIds: { type: 'array', uniqueItems: true, items: str } } } } } };
-export const benchmarkEvidenceSchema = { type: 'object', additionalProperties: false, required: ['relevant'], properties: { relevant: { type: 'boolean', description: 'Does the passage contain a concrete fact, condition, limitation or contrary evidence bearing on the question?', criteria: { true: 'Concrete evidence relevant to the question, including qualified or contradictory answers.', false: 'Only topic mentions, marketing promises, missing details, or unrelated information.' } } } };
-const validator = Compile(benchmarkReportSchema);
-export function benchmarkCases(split) {
-    if (split !== 'development' && split !== 'heldout')
-        throw new Error('Choose development or heldout explicitly');
-    return structuredClone(cases.filter(row => row.split === split));
-}
-/** Frozen independent checks. Never return these results to an authoring agent. */
-export function evaluatePiBenchmark(caseId, output) {
-    const item = cases.find(row => row.id === caseId);
-    if (!item)
-        throw new Error('Unknown benchmark case');
-    if (!validator.Check(output))
-        return { passed: false, issues: ['schema'] };
-    const findings = output.findings;
-    const issues = [];
-    if (findings.length !== item.expected.length || new Set(findings.map(row => row.topicId)).size !== findings.length)
-        issues.push('coverage');
-    for (const expected of item.expected) {
-        const actual = findings.find(row => row.topicId === expected.topicId);
-        if (actual?.status !== expected.status)
-            issues.push(`${expected.topicId}:status`);
-        if (!actual || JSON.stringify([...actual.sourceIds].sort()) !== JSON.stringify([...expected.sourceIds].sort()))
-            issues.push(`${expected.topicId}:citations`);
-    }
-    return { passed: issues.length === 0, issues };
-}
-export function benchmarkContract() {
-    return {
-        version: 1, fixturesSha256: benchmarkHash(cases),
-        verifierSha256: createHash('sha256').update(readFileSync(new URL('./benchmark-contract.js', import.meta.url))).digest('hex'),
-        outputSchemaSha256: benchmarkHash(benchmarkReportSchema),
-        instructionsSha256: benchmarkHash(benchmarkInstructions),
-    };
-}

package/dist/benchmark-freeze.d.ts

@@ -1,9 +0,0 @@
-/** Frozen before any live sample. Changing these pins requires a new benchmark
- * version and a fresh baseline; never update them to improve a skill's score. */
-export declare const frozenBenchmarkContract: {
-    readonly version: 1;
-    readonly fixturesSha256: "1a8f9ff022d5cdaf2455e327e59ae4593aeffcc8ae4c510a870b36ea0d9e7de2";
-    readonly verifierSha256: "f21d0c8b790186f571003907a5a6c7a6360423d77d1125c4398994cb1ae11f9a";
-    readonly outputSchemaSha256: "38a0d7ccd8590ad8a486a5b7b7edd36c0b8bd32e642ead5345a3f93d12f3e574";
-    readonly instructionsSha256: "d8d3fc6721ed99bb163ff3dbd93dffc15a1a92ab400f11f781bb25e9f8a61535";
-};

package/dist/benchmark-freeze.js

@@ -1,9 +0,0 @@
-/** Frozen before any live sample. Changing these pins requires a new benchmark
- * version and a fresh baseline; never update them to improve a skill's score. */
-export const frozenBenchmarkContract = {
-    version: 1,
-    fixturesSha256: '1a8f9ff022d5cdaf2455e327e59ae4593aeffcc8ae4c510a870b36ea0d9e7de2',
-    verifierSha256: 'f21d0c8b790186f571003907a5a6c7a6360423d77d1125c4398994cb1ae11f9a',
-    outputSchemaSha256: '38a0d7ccd8590ad8a486a5b7b7edd36c0b8bd32e642ead5345a3f93d12f3e574',
-    instructionsSha256: 'd8d3fc6721ed99bb163ff3dbd93dffc15a1a92ab400f11f781bb25e9f8a61535',
-};

package/dist/benchmark.d.ts

@@ -1,127 +0,0 @@
-import { type WorkflowDeps } from '@parcha/agentrun-dsl';
-import type { PiHostContext } from './types.js';
-import { evaluatePiBenchmark, type BenchmarkSplit } from './benchmark-contract.js';
-export interface PiBenchmarkBudgets {
-    timeoutMs: number;
-    maxModelRequests: number;
-    maxToolCalls: number;
-    maxTurns: number;
-    maxSubmissions: number;
-}
-export interface PiSkillBenchmarkOptions {
-    host: PiHostContext;
-    runJudge: NonNullable<WorkflowDeps['runJudge']>;
-    /** Exact native skill plus its loaded references, supplied by the caller. No hidden authoring contract is added. */
-    skillText: string;
-    split: BenchmarkSplit;
-    mode: 'live' | 'scripted-transport';
-    signal?: AbortSignal;
-    budgets?: Partial<PiBenchmarkBudgets>;
-    onProgress?: (event: {
-        caseId: string;
-        arm: 'plain-pi' | 'skill-dsl';
-        stage: 'arm' | 'outer' | 'workflow';
-        event: 'start' | 'end';
-        status?: string;
-    }) => unknown;
-}
-type Metrics = {
-    pi: number;
-    jev: number;
-    tools: number;
-    submissions: number;
-    tokens: number;
-    usageSamples: number;
-    knownCostUsd: number;
-    costSamples: number;
-};
-type Arm = 'plain-pi' | 'skill-dsl';
-export interface PiBenchmarkArm {
-    arm: Arm;
-    status: 'passed' | 'rejected' | 'failed' | 'cancelled';
-    elapsedMs: number;
-    total: Metrics;
-    stages: {
-        outer: Metrics;
-        workflow: Metrics;
-    };
-    costComplete: boolean;
-    usageComplete: boolean;
-    candidates: {
-        digest?: string;
-        accepted: boolean;
-        workflow?: unknown;
-    }[];
-    workflowRuns: {
-        status: string;
-        digest: string;
-        output?: unknown;
-    }[];
-    output?: unknown;
-    evaluation?: ReturnType<typeof evaluatePiBenchmark>;
-    failure?: string;
-}
-/** End-to-end comparison: DSL authoring, repair, execution and delivery all count.
- * Uses the selected host model, never a model selector or alternate provider. */
-export declare function runPiSkillBenchmark(options: PiSkillBenchmarkOptions): Promise<{
-    mode: "live" | "scripted-transport";
-    split: BenchmarkSplit;
-    status: "complete" | "cancelled";
-    passed: boolean;
-    contract: {
-        version: number;
-        fixturesSha256: string;
-        verifierSha256: string;
-        outputSchemaSha256: string;
-        instructionsSha256: string;
-    };
-    skillSha256: string;
-    budgets: {
-        timeoutMs: number;
-        maxModelRequests: number;
-        maxToolCalls: number;
-        maxTurns: number;
-        maxSubmissions: number;
-    };
-    runtime: {
-        model: string;
-        provider: string;
-        api: string;
-        thinking: import("./types.js").PiThinkingLevel;
-    };
-    runs: {
-        caseId: string;
-        inputSha256: string;
-        order: Arm[];
-        arms: PiBenchmarkArm[];
-    }[];
-    limitations: string[];
-}>;
-export type PiSkillBenchmarkReport = Awaited<ReturnType<typeof runPiSkillBenchmark>>;
-/** A development comparison receipt, not automatic promotion. Heldout is a final
- * confirmation set, never the optimization target for this helper. */
-export declare function comparePiSkillBenchmarks(baseline: PiSkillBenchmarkReport, candidate: PiSkillBenchmarkReport): {
-    mode: "live" | "scripted-transport";
-    baselineSkillSha256: string;
-    candidateSkillSha256: string;
-    regressions: string[];
-    previous: {
-        passed: number;
-        requests: number;
-        tokens: number;
-        usageComplete: boolean;
-        knownCostUsd: number;
-        costComplete: boolean;
-    };
-    current: {
-        passed: number;
-        requests: number;
-        tokens: number;
-        usageComplete: boolean;
-        knownCostUsd: number;
-        costComplete: boolean;
-    };
-    improved: boolean;
-    eligibleForLiveHeldout: boolean;
-};
-export {};