@tangle-network/braid 0.1.0

package/dist/eval/campaigns.d.ts

@@ -0,0 +1,30 @@
+import type { CostLedger } from '@tangle-network/agent-eval';
+import { type CampaignResult, type ChatClient } from '@tangle-network/agent-eval/contract';
+import type { PilotInspection } from './records.js';
+import type { RecordedJudgeCall, SemanticCaseEvidence, SemanticEvalArtifact, SemanticEvalCase, SemanticEvalScenario } from './types.js';
+export interface CampaignExecution {
+    readonly scenarios: readonly SemanticEvalScenario[];
+    readonly productFailures: readonly {
+        readonly fixtureId: string;
+        readonly reason: string;
+    }[];
+    readonly campaign: CampaignResult<SemanticEvalArtifact, SemanticEvalScenario> | null;
+    readonly ledger: CostLedger;
+}
+export declare function runPilotCampaign(input: {
+    readonly definition: SemanticEvalCase;
+    readonly chat: ChatClient;
+    readonly ledger: CostLedger;
+    readonly runDir: string;
+}): Promise<CampaignExecution>;
+export declare function runReleaseCampaign(input: {
+    readonly definition: SemanticEvalCase;
+    readonly chat: ChatClient;
+    readonly ledger: CostLedger;
+    readonly runDir: string;
+}): Promise<CampaignExecution>;
+export declare function inspectPilot(execution: CampaignExecution, rawCalls: readonly RecordedJudgeCall[], threshold: number): PilotInspection;
+export declare function caseEvidence(definition: SemanticEvalCase, execution: CampaignExecution, rawCalls: readonly RecordedJudgeCall[]): SemanticCaseEvidence;
+export declare function unavailableCaseEvidence(definition: SemanticEvalCase, reason: string): SemanticCaseEvidence;
+export declare function assertHeldOutInputs(definitions: readonly SemanticEvalCase[]): void;
+//# sourceMappingURL=campaigns.d.ts.map

package/dist/eval/campaigns.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"campaigns.d.ts","sourceRoot":"","sources":["../../src/eval/campaigns.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,UAAU,EAEhB,MAAM,qCAAqC,CAAA;AAG5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAEnD,OAAO,KAAK,EACV,iBAAiB,EACjB,oBAAoB,EACpB,oBAAoB,EACpB,gBAAgB,EAChB,oBAAoB,EACrB,MAAM,YAAY,CAAA;AAEnB,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,SAAS,EAAE,SAAS,oBAAoB,EAAE,CAAA;IACnD,QAAQ,CAAC,eAAe,EAAE,SAAS;QAAE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;IAC5F,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC,oBAAoB,EAAE,oBAAoB,CAAC,GAAG,IAAI,CAAA;IACpF,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAA;CAC5B;AAwDD,wBAAsB,gBAAgB,CAAC,KAAK,EAAE;IAC5C,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAA;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACxB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAoB7B;AAED,wBAAsB,kBAAkB,CAAC,KAAK,EAAE;IAC9C,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAA;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACxB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAoB7B;AAqBD,wBAAgB,YAAY,CAC1B,SAAS,EAAE,iBAAiB,EAC5B,QAAQ,EAAE,SAAS,iBAAiB,EAAE,EACtC,SAAS,EAAE,MAAM,GAChB,eAAe,CAoEjB;AAED,wBAAgB,YAAY,CAC1B,UAAU,EAAE,gBAAgB,EAC5B,SAAS,EAAE,iBAAiB,EAC5B,QAAQ,EAAE,SAAS,iBAAiB,EAAE,GACrC,oBAAoB,CAsCtB;AAED,wBAAgB,uBAAuB,CACrC,UAAU,EAAE,gBAAgB,EAC5B,MAAM,EAAE,MAAM,GACb,oBAAoB,CAkBtB;AAED,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,SAAS,gBAAgB,EAAE,GAAG,IAAI,CAelF"}

package/dist/eval/campaigns.js

@@ -0,0 +1,194 @@
+import { runCampaign, } from '@tangle-network/agent-eval/contract';
+import { semanticJudge } from './calibration.js';
+import { artifactForScenario, scenariosForRelease } from './cases.js';
+import { cellCostEvidence, cellCostReceipts, cellEvidence, evalSha256 } from './records.js';
+function productFailures(scenarios) {
+    return scenarios.flatMap((scenario) => {
+        const product = scenario.productPath;
+        return product?.available === false
+            ? [
+                {
+                    fixtureId: scenario.fixtureId,
+                    reason: product.missingReason ?? 'product presenter unavailable',
+                },
+            ]
+            : [];
+    });
+}
+function validProductScenarios(scenarios) {
+    return (productFailures(scenarios).length === 0 &&
+        scenarios.every((scenario) => scenario.productPath?.available === true));
+}
+function campaignOptions(definition, chat, ledger, scenarios, runDir, dispatchRef, costPhase, costTags, abortOnCellError) {
+    return {
+        scenarios: [...scenarios],
+        dispatch: async (scenario) => artifactForScenario(scenario),
+        dispatchRef,
+        judges: [semanticJudge(definition, chat)],
+        seed: 42,
+        reps: 1,
+        resumable: false,
+        costLedger: ledger,
+        costPhase,
+        costTags,
+        maxConcurrency: 1,
+        abortOnCellError,
+        dispatchTimeoutMs: 120_000,
+        runDir,
+        tracing: 'on',
+        // The product presenter is deterministic. Paid work happens only in the judge,
+        // whose receipt is independently required and recorded by agent-eval.
+        expectUsage: 'off',
+    };
+}
+export async function runPilotCampaign(input) {
+    const scenarios = scenariosForRelease(input.definition).slice(0, 1);
+    const failures = productFailures(scenarios);
+    if (!validProductScenarios(scenarios)) {
+        return { scenarios, productFailures: failures, campaign: null, ledger: input.ledger };
+    }
+    const campaign = await runCampaign(campaignOptions(input.definition, input.chat, input.ledger, scenarios, input.runDir, 'braid-semantic-pilot-v2-EVAL-01', 'semantic-pilot-EVAL-01', { braidEvalCase: 'EVAL-01', phase: 'pilot' }, true));
+    return { scenarios, productFailures: failures, campaign, ledger: input.ledger };
+}
+export async function runReleaseCampaign(input) {
+    const scenarios = scenariosForRelease(input.definition);
+    const failures = productFailures(scenarios);
+    if (!validProductScenarios(scenarios)) {
+        return { scenarios, productFailures: failures, campaign: null, ledger: input.ledger };
+    }
+    const campaign = await runCampaign(campaignOptions(input.definition, input.chat, input.ledger, scenarios, input.runDir, `braid-semantic-release-v2-${input.definition.id}`, `semantic-release-${input.definition.id}`, { braidEvalCase: input.definition.id, phase: 'release' }, false));
+    return { scenarios, productFailures: failures, campaign, ledger: input.ledger };
+}
+function scoreIsParseable(score) {
+    if (score === null || typeof score !== 'object')
+        return false;
+    const value = score;
+    if (value.failed === true ||
+        typeof value.composite !== 'number' ||
+        !Number.isFinite(value.composite))
+        return false;
+    if (value.dimensions === null || typeof value.dimensions !== 'object')
+        return false;
+    return Object.values(value.dimensions).every((dimension) => typeof dimension === 'number' && Number.isFinite(dimension));
+}
+export function inspectPilot(execution, rawCalls, threshold) {
+    const failures = [];
+    const cell = execution.campaign?.cells.length === 1 ? execution.campaign.cells[0] : undefined;
+    const scenario = execution.scenarios[0];
+    const productAvailable = scenario?.productPath?.available === true && execution.productFailures.length === 0;
+    const scores = cell === undefined ? [] : Object.values(cell.judgeScores);
+    const parseableJudgeScore = scores.length === 1 && scores.every(scoreIsParseable);
+    const rawJudgeCallCaptured = cell !== undefined &&
+        (cell.costCallIds ?? []).some((callId) => rawCalls.some((call) => call.callId === callId && call.response !== null && call.error === null));
+    const judgeReceipts = cell === undefined
+        ? []
+        : cellCostReceipts(cell, execution.ledger).filter((receipt) => receipt.channel === 'judge');
+    const usageCaptured = judgeReceipts.length > 0 &&
+        judgeReceipts.every((receipt) => receipt.usageUnknown !== true &&
+            Number.isFinite(receipt.inputTokens) &&
+            receipt.inputTokens >= 0 &&
+            Number.isFinite(receipt.outputTokens) &&
+            receipt.outputTokens >= 0 &&
+            (receipt.reasoningTokens === undefined ||
+                (Number.isFinite(receipt.reasoningTokens) && receipt.reasoningTokens >= 0))) &&
+        judgeReceipts.some((receipt) => receipt.inputTokens + receipt.outputTokens > 0);
+    const costCaptured = judgeReceipts.length > 0 &&
+        judgeReceipts.every((receipt) => Number.isFinite(receipt.costUsd) && receipt.costUsd >= 0 && receipt.costUnknown === false);
+    const costEvidence = cell === undefined ? null : cellCostEvidence(cell, rawCalls, execution.ledger);
+    const wallTimeCaptured = costEvidence !== null && Number.isFinite(costEvidence.wallTimeMs) && costEvidence.wallTimeMs > 0;
+    if (!productAvailable)
+        failures.push(...execution.productFailures.map((failure) => `${failure.fixtureId}: ${failure.reason}`));
+    if (execution.campaign === null)
+        failures.push('pilot produced no campaign because the product path was unavailable');
+    if (cell === undefined)
+        failures.push('pilot did not produce exactly one campaign cell');
+    if (!parseableJudgeScore)
+        failures.push('pilot did not produce one parseable judge score');
+    if (!rawJudgeCallCaptured)
+        failures.push('pilot did not capture the successful raw judge call');
+    if (!usageCaptured)
+        failures.push('pilot did not capture input/output token usage');
+    if (!costCaptured)
+        failures.push('pilot did not capture cost provenance, including zero');
+    if (!wallTimeCaptured)
+        failures.push('pilot did not capture positive wall time');
+    if (cell?.error !== undefined)
+        failures.push(`pilot cell failed: ${cell.error}`);
+    const composite = scores[0]?.composite;
+    if (typeof composite !== 'number' || composite < threshold)
+        failures.push(`pilot score is below ${threshold.toFixed(2)}`);
+    return {
+        passed: failures.length === 0,
+        productAvailable,
+        parseableJudgeScore,
+        rawJudgeCallCaptured,
+        usageCaptured,
+        costCaptured,
+        wallTimeCaptured,
+        failures,
+    };
+}
+export function caseEvidence(definition, execution, rawCalls) {
+    const artifacts = execution.scenarios.map(artifactForScenario);
+    const cells = execution.campaign === null
+        ? []
+        : execution.campaign.cells.map((cell) => cellEvidence(cell, rawCalls, definition.criteria.passThreshold, execution.ledger));
+    const disagreements = [
+        ...execution.productFailures.map((failure) => `${failure.fixtureId}: ${failure.reason}`),
+        ...cells.flatMap((cell) => cell.pass
+            ? []
+            : [`${cell.scenarioId} scored below ${definition.criteria.passThreshold.toFixed(2)}`]),
+    ];
+    return {
+        id: definition.id,
+        category: 'eval',
+        command: 'pnpm test:eval',
+        question: definition.question,
+        referenceCriteria: definition.criteria,
+        fixtureCount: execution.scenarios.length,
+        passedFixtures: cells.filter((cell) => cell.pass).length,
+        failedFixtures: execution.scenarios.length - cells.filter((cell) => cell.pass).length,
+        artifacts,
+        productFailures: execution.productFailures,
+        result: execution.campaign !== null &&
+            execution.productFailures.length === 0 &&
+            cells.length === execution.scenarios.length &&
+            cells.every((cell) => cell.pass)
+            ? 'passed'
+            : 'failed',
+        campaign: execution.campaign,
+        cells,
+        disagreements,
+    };
+}
+export function unavailableCaseEvidence(definition, reason) {
+    const scenarios = scenariosForRelease(definition);
+    return {
+        id: definition.id,
+        category: 'eval',
+        command: 'pnpm test:eval',
+        question: definition.question,
+        referenceCriteria: definition.criteria,
+        fixtureCount: scenarios.length,
+        passedFixtures: 0,
+        failedFixtures: scenarios.length,
+        artifacts: scenarios.map(artifactForScenario),
+        productFailures: scenarios.map((scenario) => ({ fixtureId: scenario.fixtureId, reason })),
+        result: 'unavailable',
+        campaign: null,
+        cells: [],
+        disagreements: [reason],
+    };
+}
+export function assertHeldOutInputs(definitions) {
+    const calibrationDigests = new Set(definitions.flatMap((definition) => definition.calibrationFixtures.map((fixture) => evalSha256(fixture.semanticOutput))));
+    const collisions = definitions.flatMap((definition) => definition.releaseFixtures.flatMap((fixture) => {
+        const hash = evalSha256(fixture.semanticOutput);
+        return calibrationDigests.has(hash)
+            ? [`${definition.id}/${fixture.id} collides with calibration input ${hash}`]
+            : [];
+    }));
+    if (collisions.length > 0)
+        throw new Error(`Held-out fixture collision: ${collisions.join('; ')}`);
+}
+//# sourceMappingURL=campaigns.js.map

package/dist/eval/campaigns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"campaigns.js","sourceRoot":"","sources":["../../src/eval/campaigns.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,WAAW,GACZ,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAChD,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAErE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAgB3F,SAAS,eAAe,CAAC,SAA0C;IACjE,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QACpC,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,CAAA;QACpC,OAAO,OAAO,EAAE,SAAS,KAAK,KAAK;YACjC,CAAC,CAAC;gBACE;oBACE,SAAS,EAAE,QAAQ,CAAC,SAAS;oBAC7B,MAAM,EAAE,OAAO,CAAC,aAAa,IAAI,+BAA+B;iBACjE;aACF;YACH,CAAC,CAAC,EAAE,CAAA;IACR,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,SAA0C;IACvE,OAAO,CACL,eAAe,CAAC,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC;QACvC,SAAS,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,SAAS,KAAK,IAAI,CAAC,CACxE,CAAA;AACH,CAAC;AAED,SAAS,eAAe,CACtB,UAA4B,EAC5B,IAAgB,EAChB,MAAkB,EAClB,SAA0C,EAC1C,MAAc,EACd,WAAmB,EACnB,SAAiB,EACjB,QAAgC,EAChC,gBAAyB;IAEzB,OAAO;QACL,SAAS,EAAE,CAAC,GAAG,SAAS,CAAC;QACzB,QAAQ,EAAE,KAAK,EAAE,QAA8B,EAAE,EAAE,CAAC,mBAAmB,CAAC,QAAQ,CAAC;QACjF,WAAW;QACX,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACzC,IAAI,EAAE,EAAE;QACR,IAAI,EAAE,CAAC;QACP,SAAS,EAAE,KAAK;QAChB,UAAU,EAAE,MAAM;QAClB,SAAS;QACT,QAAQ;QACR,cAAc,EAAE,CAAC;QACjB,gBAAgB;QAChB,iBAAiB,EAAE,OAAO;QAC1B,MAAM;QACN,OAAO,EAAE,IAAa;QACtB,+EAA+E;QAC/E,sEAAsE;QACtE,WAAW,EAAE,KAAc;KAC5B,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAKtC;IACC,MAAM,SAAS,GAAG,mBAAmB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IACnE,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,CAAC,CAAA;IAC3C,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAA;IACvF,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAChC,eAAe,CACb,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,MAAM,EACZ,SAAS,EACT,KAAK,CAAC,MAAM,EACZ,iCAAiC,EACjC,wBAAwB,EACxB,EAAE,aAAa,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAC5C,IAAI,CACL,CACF,CAAA;IACD,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAA;AACjF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAKxC;IACC,MAAM,SAAS,GAAG,mBAAmB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAA;IACvD,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,CAAC,CAAA;IAC3C,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAA;IACvF,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAChC,eAAe,CACb,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,MAAM,EACZ,SAAS,EACT,KAAK,CAAC,MAAM,EACZ,6BAA6B,KAAK,CAAC,UAAU,CAAC,EAAE,EAAE,EAClD,oBAAoB,KAAK,CAAC,UAAU,CAAC,EAAE,EAAE,EACzC,EAAE,aAAa,EAAE,KAAK,CAAC,UAAU,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,EACxD,KAAK,CACN,CACF,CAAA;IACD,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAA;AACjF,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC7D,MAAM,KAAK,GAAG,KAIb,CAAA;IACD,IACE,KAAK,CAAC,MAAM,KAAK,IAAI;QACrB,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC;QAEjC,OAAO,KAAK,CAAA;IACd,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IACnF,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,UAAqC,CAAC,CAAC,KAAK,CACrE,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAC3E,CAAA;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,SAA4B,EAC5B,QAAsC,EACtC,SAAiB;IAEjB,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAC7F,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;IACvC,MAAM,gBAAgB,GACpB,QAAQ,EAAE,WAAW,EAAE,SAAS,KAAK,IAAI,IAAI,SAAS,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,CAAA;IACrF,MAAM,MAAM,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IACxE,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAA;IACjF,MAAM,oBAAoB,GACxB,IAAI,KAAK,SAAS;QAClB,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CACvC,QAAQ,CAAC,IAAI,CACX,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAClF,CACF,CAAA;IACH,MAAM,aAAa,GACjB,IAAI,KAAK,SAAS;QAChB,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,gBAAgB,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC,CAAA;IAC/F,MAAM,aAAa,GACjB,aAAa,CAAC,MAAM,GAAG,CAAC;QACxB,aAAa,CAAC,KAAK,CACjB,CAAC,OAAO,EAAE,EAAE,CACV,OAAO,CAAC,YAAY,KAAK,IAAI;YAC7B,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC;YACpC,OAAO,CAAC,WAAW,IAAI,CAAC;YACxB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC;YACrC,OAAO,CAAC,YAAY,IAAI,CAAC;YACzB,CAAC,OAAO,CAAC,eAAe,KAAK,SAAS;gBACpC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,OAAO,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,CAChF;QACD,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,YAAY,GAAG,CAAC,CAAC,CAAA;IACjF,MAAM,YAAY,GAChB,aAAa,CAAC,MAAM,GAAG,CAAC;QACxB,aAAa,CAAC,KAAK,CACjB,CAAC,OAAO,EAAE,EAAE,CACV,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,CAC5F,CAAA;IACH,MAAM,YAAY,GAChB,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,CAAA;IAChF,MAAM,gBAAgB,GACpB,YAAY,KAAK,IAAI,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,YAAY,CAAC,UAAU,GAAG,CAAC,CAAA;IAClG,IAAI,CAAC,gBAAgB;QACnB,QAAQ,CAAC,IAAI,CACX,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,SAAS,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CACzF,CAAA;IACH,IAAI,SAAS,CAAC,QAAQ,KAAK,IAAI;QAC7B,QAAQ,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAA;IACtF,IAAI,IAAI,KAAK,SAAS;QAAE,QAAQ,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAA;IACxF,IAAI,CAAC,mBAAmB;QAAE,QAAQ,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAA;IAC1F,IAAI,CAAC,oBAAoB;QAAE,QAAQ,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAA;IAC/F,IAAI,CAAC,aAAa;QAAE,QAAQ,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAA;IACnF,IAAI,CAAC,YAAY;QAAE,QAAQ,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAA;IACzF,IAAI,CAAC,gBAAgB;QAAE,QAAQ,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAA;IAChF,IAAI,IAAI,EAAE,KAAK,KAAK,SAAS;QAAE,QAAQ,CAAC,IAAI,CAAC,sBAAsB,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;IAChF,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,SAAS,CAAA;IACtC,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,SAAS;QACxD,QAAQ,CAAC,IAAI,CAAC,wBAAwB,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC/D,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;QAC7B,gBAAgB;QAChB,mBAAmB;QACnB,oBAAoB;QACpB,aAAa;QACb,YAAY;QACZ,gBAAgB;QAChB,QAAQ;KACT,CAAA;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,UAA4B,EAC5B,SAA4B,EAC5B,QAAsC;IAEtC,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAA;IAC9D,MAAM,KAAK,GACT,SAAS,CAAC,QAAQ,KAAK,IAAI;QACzB,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACpC,YAAY,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAClF,CAAA;IACP,MAAM,aAAa,GAAG;QACpB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,SAAS,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;QACxF,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACxB,IAAI,CAAC,IAAI;YACP,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,iBAAiB,UAAU,CAAC,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CACxF;KACF,CAAA;IACD,OAAO;QACL,EAAE,EAAE,UAAU,CAAC,EAAE;QACjB,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,gBAAgB;QACzB,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,iBAAiB,EAAE,UAAU,CAAC,QAAQ;QACtC,YAAY,EAAE,SAAS,CAAC,SAAS,CAAC,MAAM;QACxC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM;QACxD,cAAc,EAAE,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM;QACrF,SAAS;QACT,eAAe,EAAE,SAAS,CAAC,eAAe;QAC1C,MAAM,EACJ,SAAS,CAAC,QAAQ,KAAK,IAAI;YAC3B,SAAS,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC;YACtC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,CAAC,MAAM;YAC3C,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;YAC9B,CAAC,CAAC,QAAQ;YACV,CAAC,CAAC,QAAQ;QACd,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,KAAK;QACL,aAAa;KACd,CAAA;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,UAA4B,EAC5B,MAAc;IAEd,MAAM,SAAS,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAA;IACjD,OAAO;QACL,EAAE,EAAE,UAAU,CAAC,EAAE;QACjB,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,gBAAgB;QACzB,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,iBAAiB,EAAE,UAAU,CAAC,QAAQ;QACtC,YAAY,EAAE,SAAS,CAAC,MAAM;QAC9B,cAAc,EAAE,CAAC;QACjB,cAAc,EAAE,SAAS,CAAC,MAAM;QAChC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,mBAAmB,CAAC;QAC7C,eAAe,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzF,MAAM,EAAE,aAAa;QACrB,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,EAAE;QACT,aAAa,EAAE,CAAC,MAAM,CAAC;KACxB,CAAA;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,WAAwC;IAC1E,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAChC,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CACjC,UAAU,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CACpF,CACF,CAAA;IACD,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CACpD,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;QAC/C,OAAO,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC;YACjC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,EAAE,IAAI,OAAO,CAAC,EAAE,oCAAoC,IAAI,EAAE,CAAC;YAC5E,CAAC,CAAC,EAAE,CAAA;IACR,CAAC,CAAC,CACH,CAAA;IACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AACpG,CAAC"}

package/dist/eval/cases.d.ts

@@ -0,0 +1,9 @@
+import type { SemanticEvalArtifact, SemanticEvalCase, SemanticEvalScenario } from './types.js';
+import { type CalibrationLabel, type SemanticEvalCaseId } from './types.js';
+export declare const SEMANTIC_PASS_THRESHOLD = 0.7;
+export declare const SEMANTIC_CASES: readonly SemanticEvalCase[];
+export declare function scenariosForCalibration(definition: SemanticEvalCase, label: CalibrationLabel): SemanticEvalScenario[];
+export declare function scenariosForRelease(definition: SemanticEvalCase): SemanticEvalScenario[];
+export declare function artifactForScenario(scenario: SemanticEvalScenario): SemanticEvalArtifact;
+export declare function definitionForCase(id: SemanticEvalCaseId): SemanticEvalCase;
+//# sourceMappingURL=cases.d.ts.map

package/dist/eval/cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cases.d.ts","sourceRoot":"","sources":["../../src/eval/cases.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAE9F,OAAO,EACL,KAAK,gBAAgB,EAErB,KAAK,kBAAkB,EACxB,MAAM,YAAY,CAAA;AAEnB,eAAO,MAAM,uBAAuB,MAAM,CAAA;AA2P1C,eAAO,MAAM,cAAc,EAAE,SAAS,gBAAgB,EAOrD,CAAA;AAED,wBAAgB,uBAAuB,CACrC,UAAU,EAAE,gBAAgB,EAC5B,KAAK,EAAE,gBAAgB,GACtB,oBAAoB,EAAE,CAgBxB;AAED,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,gBAAgB,GAAG,oBAAoB,EAAE,CAWxF;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,oBAAoB,GAAG,oBAAoB,CAQxF;AAED,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,kBAAkB,GAAG,gBAAgB,CAI1E"}

package/dist/eval/cases.js

@@ -0,0 +1,289 @@
+import { SEMANTIC_CALIBRATION_FIXTURES, SEMANTIC_RELEASE_FIXTURES } from './fixtures.js';
+import { SEMANTIC_EVAL_CATEGORIES, } from './types.js';
+export const SEMANTIC_PASS_THRESHOLD = 0.7;
+const CASE_PROMPTS = {
+    'EVAL-01': `You are judging a Braid fork explanation for a person deciding whether to continue work.
+Use only the Braid fork-plan object in the semantic output and the answer shown to the user.
+Reward explicit names or values for copied conversation context, provider-session behavior, environment behavior, checkpoint behavior, and omitted state.
+Penalize claims that a new provider session resumes the source session, that an unavailable environment was created, or that omitted provider-owned state was copied.
+The answer must make the allowed or unavailable decision and a useful next choice clear.`,
+    'EVAL-02': `You are judging a Braid permission explanation before a command can run.
+Reward the exact subject, concrete consequence, offered permission scopes, and concise safe choices.
+The explanation must preserve the pause and must not recommend a broader scope than the InteractionRequest offers.
+Penalize vague approval language, hidden destructive or network consequences, persistent approval when it is not offered, and burying the decision.`,
+    'EVAL-03': `You are judging a Braid /ask answer over a frozen analysis record.
+Reward actionable findings visibly supported by the serialized citation, accurate uncertainty or confidence, and a next diagnostic action.
+Penalize any citation that does not resolve to the shown frozen source, any invented citation, and any root-cause claim not supported by the source event.
+The answer must respect failed or incomplete source status and must not promote speculation into fact.`,
+    'EVAL-04': `You are judging a Braid paired-comparison explanation.
+Before any verdict, require every field captured on either arm, explicit missing fields, cost provenance, terminal outcome, and sample-size limitations.
+Missing is not zero, a failed arm is not successful, and one pair is descriptive rather than a reliable general conclusion.
+Penalize any verdict that hides an asymmetry or invents a value.`,
+    'EVAL-05': `You are judging Braid reconnect status language.
+Reward distinct plain-language meanings and actions for detached, reconnecting, cancelled, failed, expired, and unknown.
+Detached and reconnecting describe connection or recovery state; cancelled, failed, and expired are different terminal outcomes; unknown means evidence is insufficient.
+Penalize collapsing these states, claiming success without evidence, or recommending reconnect for a terminal state.`,
+    'EVAL-06': `You are judging a Braid profile incompatibility explanation.
+Identify the exact unsupported profile field and value, preserve the authored profile, and name valid explicit choices that resolve the mismatch.
+Reward the distinction between changing the harness and changing the model.
+Penalize silently dropping, replacing, or weakening a profile field, and penalize claims that compatibility is automatic.`,
+};
+const CASE_METADATA = {
+    'EVAL-01': {
+        question: 'Can a user tell what a fork copied?',
+        category: SEMANTIC_EVAL_CATEGORIES['EVAL-01'],
+        requiredSignals: [
+            'conversation context',
+            'provider session',
+            'environment',
+            'checkpoint',
+            'omitted state',
+        ],
+        forbiddenSignals: [
+            'false session resume',
+            'fabricated environment',
+            'fabricated checkpoint',
+            'omitted state claimed copied',
+        ],
+        dimensions: [
+            {
+                key: 'copied_state',
+                description: 'Names the conversation and context that the plan actually copies.',
+            },
+            {
+                key: 'runtime_boundaries',
+                description: 'Correctly distinguishes provider session, environment, and checkpoint behavior.',
+            },
+            {
+                key: 'omissions',
+                description: 'Names omitted or unavailable state without inventing a copy.',
+            },
+            {
+                key: 'decision_clarity',
+                description: 'Makes the allowed or unavailable decision and next choice clear.',
+            },
+        ],
+    },
+    'EVAL-02': {
+        question: 'Is a permission understandable?',
+        category: SEMANTIC_EVAL_CATEGORIES['EVAL-02'],
+        requiredSignals: ['exact subject', 'consequence', 'offered scope', 'safe choices'],
+        forbiddenSignals: [
+            'vague approval',
+            'unoffered scope',
+            'hidden consequence',
+            'buried decision',
+        ],
+        dimensions: [
+            {
+                key: 'subject_consequence',
+                description: 'Identifies the exact subject and what it will do.',
+            },
+            {
+                key: 'scope_accuracy',
+                description: 'Describes only the permission scopes offered by the answer specification.',
+            },
+            {
+                key: 'safe_choices',
+                description: 'Names clear deny or allow choices without broadening the request.',
+            },
+            {
+                key: 'decision_clarity',
+                description: 'Keeps the paused decision concise and understandable.',
+            },
+        ],
+    },
+    'EVAL-03': {
+        question: 'Is /ask useful?',
+        category: SEMANTIC_EVAL_CATEGORIES['EVAL-03'],
+        requiredSignals: [
+            'actionable finding',
+            'resolvable citation',
+            'uncertainty',
+            'supported claim',
+        ],
+        forbiddenSignals: [
+            'dangling citation',
+            'invented citation',
+            'unsupported root cause',
+            'certainty beyond source',
+        ],
+        dimensions: [
+            { key: 'actionability', description: 'Gives a concrete next action tied to the finding.' },
+            {
+                key: 'citation_integrity',
+                description: 'Uses the shown citation and does not invent or misresolve evidence.',
+            },
+            {
+                key: 'uncertainty',
+                description: 'Communicates confidence and source completeness accurately.',
+            },
+            {
+                key: 'causal_discipline',
+                description: 'Does not assert a root cause absent from the frozen source.',
+            },
+        ],
+    },
+    'EVAL-04': {
+        question: 'Is a comparison honest?',
+        category: SEMANTIC_EVAL_CATEGORIES['EVAL-04'],
+        requiredSignals: [
+            'every captured field',
+            'missing field',
+            'cost',
+            'outcome',
+            'sample limitation',
+        ],
+        forbiddenSignals: [
+            'missing treated as zero',
+            'failed treated as success',
+            'hidden asymmetry',
+            'overconfident verdict',
+        ],
+        dimensions: [
+            {
+                key: 'asymmetry_completeness',
+                description: 'Shows every captured field present on either comparison arm.',
+            },
+            {
+                key: 'missingness',
+                description: 'Labels missing values as missing and never substitutes zero.',
+            },
+            {
+                key: 'cost_outcome',
+                description: 'Reports cost provenance and terminal outcome before judging.',
+            },
+            {
+                key: 'verdict_discipline',
+                description: 'Limits the verdict to what the pair count and evidence support.',
+            },
+        ],
+    },
+    'EVAL-05': {
+        question: 'Is reconnect status clear?',
+        category: SEMANTIC_EVAL_CATEGORIES['EVAL-05'],
+        requiredSignals: ['detached', 'reconnecting', 'cancelled', 'failed', 'expired', 'unknown'],
+        forbiddenSignals: [
+            'collapsed statuses',
+            'unknown called success',
+            'terminal reconnect',
+            'unproven continuation',
+        ],
+        dimensions: [
+            {
+                key: 'status_distinction',
+                description: 'Gives each of the six statuses a distinct meaning.',
+            },
+            {
+                key: 'evidence',
+                description: 'States what the available status evidence does and does not prove.',
+            },
+            {
+                key: 'next_step',
+                description: 'Gives an appropriate action for recoverable and terminal states.',
+            },
+            {
+                key: 'uncertainty',
+                description: 'Treats unknown as unresolved evidence rather than success.',
+            },
+        ],
+    },
+    'EVAL-06': {
+        question: 'Is profile incompatibility clear?',
+        category: SEMANTIC_EVAL_CATEGORIES['EVAL-06'],
+        requiredSignals: [
+            'unsupported field',
+            'exact values',
+            'valid harness choice',
+            'valid model choice',
+            'no silent mutation',
+        ],
+        forbiddenSignals: [
+            'silent model replacement',
+            'dropped field',
+            'automatic compatibility',
+            'hidden profile mutation',
+        ],
+        dimensions: [
+            { key: 'exact_field', description: 'Identifies the unsupported field and exact values.' },
+            {
+                key: 'valid_choices',
+                description: 'Names explicit harness and model choices that resolve the mismatch.',
+            },
+            {
+                key: 'profile_integrity',
+                description: 'Makes clear that the authored profile is not silently weakened.',
+            },
+            { key: 'decision_clarity', description: 'Explains the user decision in plain language.' },
+        ],
+    },
+};
+function caseDefinition(id) {
+    const metadata = CASE_METADATA[id];
+    return {
+        id,
+        question: metadata.question,
+        category: metadata.category,
+        prompt: CASE_PROMPTS[id],
+        dimensions: metadata.dimensions,
+        criteria: {
+            requiredSignals: metadata.requiredSignals,
+            forbiddenSignals: metadata.forbiddenSignals,
+            passThreshold: SEMANTIC_PASS_THRESHOLD,
+        },
+        calibrationFixtures: SEMANTIC_CALIBRATION_FIXTURES[id],
+        releaseFixtures: SEMANTIC_RELEASE_FIXTURES[id],
+    };
+}
+export const SEMANTIC_CASES = [
+    caseDefinition('EVAL-01'),
+    caseDefinition('EVAL-02'),
+    caseDefinition('EVAL-03'),
+    caseDefinition('EVAL-04'),
+    caseDefinition('EVAL-05'),
+    caseDefinition('EVAL-06'),
+];
+export function scenariosForCalibration(definition, label) {
+    return definition.calibrationFixtures.map((fixture, index) => ({
+        id: `${definition.id}-calibration-${index + 1}`,
+        kind: `braid-semantic-${definition.category}`,
+        tags: ['braid', definition.id, 'calibration', label],
+        caseId: definition.id,
+        fixtureId: fixture.id,
+        semanticOutput: fixture.semanticOutput,
+        candidateOutput: label === 'good'
+            ? fixture.goodOutput
+            : label === 'bad'
+                ? fixture.badOutput
+                : fixture.trivialOutput,
+        candidateLabel: label,
+    }));
+}
+export function scenariosForRelease(definition) {
+    return definition.releaseFixtures.map((fixture, index) => ({
+        id: `${definition.id}-release-${index + 1}`,
+        kind: `braid-semantic-${definition.category}`,
+        tags: ['braid', definition.id, 'release', fixture.productOutput.path],
+        caseId: definition.id,
+        fixtureId: fixture.id,
+        semanticOutput: fixture.semanticOutput,
+        candidateOutput: fixture.productOutput.text,
+        productPath: fixture.productOutput,
+    }));
+}
+export function artifactForScenario(scenario) {
+    return {
+        caseId: scenario.caseId,
+        fixtureId: scenario.fixtureId,
+        semanticOutput: scenario.semanticOutput,
+        candidateOutput: scenario.candidateOutput,
+        ...(scenario.productPath === undefined ? {} : { productPath: scenario.productPath }),
+    };
+}
+export function definitionForCase(id) {
+    const definition = SEMANTIC_CASES.find((candidate) => candidate.id === id);
+    if (definition === undefined)
+        throw new Error(`Unknown semantic evaluation case ${id}`);
+    return definition;
+}
+//# sourceMappingURL=cases.js.map

package/dist/eval/cases.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cases.js","sourceRoot":"","sources":["../../src/eval/cases.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,6BAA6B,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAA;AACxF,OAAO,EAEL,wBAAwB,GAEzB,MAAM,YAAY,CAAA;AAEnB,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAA;AAE1C,MAAM,YAAY,GAAiD;IACjE,SAAS,EAAE;;;;yFAI4E;IACvF,SAAS,EAAE;;;oJAGuI;IAClJ,SAAS,EAAE;;;uGAG0F;IACrG,SAAS,EAAE;;;iEAGoD;IAC/D,SAAS,EAAE;;;qHAGwG;IACnH,SAAS,EAAE;;;0HAG6G;CACzH,CAAA;AAED,MAAM,aAAa,GAWf;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,qCAAqC;QAC/C,QAAQ,EAAE,wBAAwB,CAAC,SAAS,CAAC;QAC7C,eAAe,EAAE;YACf,sBAAsB;YACtB,kBAAkB;YAClB,aAAa;YACb,YAAY;YACZ,eAAe;SAChB;QACD,gBAAgB,EAAE;YAChB,sBAAsB;YACtB,wBAAwB;YACxB,uBAAuB;YACvB,8BAA8B;SAC/B;QACD,UAAU,EAAE;YACV;gBACE,GAAG,EAAE,cAAc;gBACnB,WAAW,EAAE,mEAAmE;aACjF;YACD;gBACE,GAAG,EAAE,oBAAoB;gBACzB,WAAW,EACT,iFAAiF;aACpF;YACD;gBACE,GAAG,EAAE,WAAW;gBAChB,WAAW,EAAE,8DAA8D;aAC5E;YACD;gBACE,GAAG,EAAE,kBAAkB;gBACvB,WAAW,EAAE,kEAAkE;aAChF;SACF;KACF;IACD,SAAS,EAAE;QACT,QAAQ,EAAE,iCAAiC;QAC3C,QAAQ,EAAE,wBAAwB,CAAC,SAAS,CAAC;QAC7C,eAAe,EAAE,CAAC,eAAe,EAAE,aAAa,EAAE,eAAe,EAAE,cAAc,CAAC;QAClF,gBAAgB,EAAE;YAChB,gBAAgB;YAChB,iBAAiB;YACjB,oBAAoB;YACpB,iBAAiB;SAClB;QACD,UAAU,EAAE;YACV;gBACE,GAAG,EAAE,qBAAqB;gBAC1B,WAAW,EAAE,mDAAmD;aACjE;YACD;gBACE,GAAG,EAAE,gBAAgB;gBACrB,WAAW,EAAE,2EAA2E;aACzF;YACD;gBACE,GAAG,EAAE,cAAc;gBACnB,WAAW,EAAE,mEAAmE;aACjF;YACD;gBACE,GAAG,EAAE,kBAAkB;gBACvB,WAAW,EAAE,uDAAuD;aACrE;SACF;KACF;IACD,SAAS,EAAE;QACT,QAAQ,EAAE,iBAAiB;QAC3B,QAAQ,EAAE,wBAAwB,CAAC,SAAS,CAAC;QAC7C,eAAe,EAAE;YACf,oBAAoB;YACpB,qBAAqB;YACrB,aAAa;YACb,iBAAiB;SAClB;QACD,gBAAgB,EAAE;YAChB,mBAAmB;YACnB,mBAAmB;YACnB,wBAAwB;YACxB,yBAAyB;SAC1B;QACD,UAAU,EAAE;YACV,EAAE,GAAG,EAAE,eAAe,EAAE,WAAW,EAAE,mDAAmD,EAAE;YAC1F;gBACE,GAAG,EAAE,oBAAoB;gBACzB,WAAW,EAAE,qEAAqE;aACnF;YACD;gBACE,GAAG,EAAE,aAAa;gBAClB,WAAW,EAAE,6DAA6D;aAC3E;YACD;gBACE,GAAG,EAAE,mBAAmB;gBACxB,WAAW,EAAE,6DAA6D;aAC3E;SACF;KACF;IACD,SAAS,EAAE;QACT,QAAQ,EAAE,yBAAyB;QACnC,QAAQ,EAAE,wBAAwB,CAAC,SAAS,CAAC;QAC7C,eAAe,EAAE;YACf,sBAAsB;YACtB,eAAe;YACf,MAAM;YACN,SAAS;YACT,mBAAmB;SACpB;QACD,gBAAgB,EAAE;YAChB,yBAAyB;YACzB,2BAA2B;YAC3B,kBAAkB;YAClB,uBAAuB;SACxB;QACD,UAAU,EAAE;YACV;gBACE,GAAG,EAAE,wBAAwB;gBAC7B,WAAW,EAAE,8DAA8D;aAC5E;YACD;gBACE,GAAG,EAAE,aAAa;gBAClB,WAAW,EAAE,8DAA8D;aAC5E;YACD;gBACE,GAAG,EAAE,cAAc;gBACnB,WAAW,EAAE,8DAA8D;aAC5E;YACD;gBACE,GAAG,EAAE,oBAAoB;gBACzB,WAAW,EAAE,iEAAiE;aAC/E;SACF;KACF;IACD,SAAS,EAAE;QACT,QAAQ,EAAE,4BAA4B;QACtC,QAAQ,EAAE,wBAAwB,CAAC,SAAS,CAAC;QAC7C,eAAe,EAAE,CAAC,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;QAC1F,gBAAgB,EAAE;YAChB,oBAAoB;YACpB,wBAAwB;YACxB,oBAAoB;YACpB,uBAAuB;SACxB;QACD,UAAU,EAAE;YACV;gBACE,GAAG,EAAE,oBAAoB;gBACzB,WAAW,EAAE,oDAAoD;aAClE;YACD;gBACE,GAAG,EAAE,UAAU;gBACf,WAAW,EAAE,oEAAoE;aAClF;YACD;gBACE,GAAG,EAAE,WAAW;gBAChB,WAAW,EAAE,kEAAkE;aAChF;YACD;gBACE,GAAG,EAAE,aAAa;gBAClB,WAAW,EAAE,4DAA4D;aAC1E;SACF;KACF;IACD,SAAS,EAAE;QACT,QAAQ,EAAE,mCAAmC;QAC7C,QAAQ,EAAE,wBAAwB,CAAC,SAAS,CAAC;QAC7C,eAAe,EAAE;YACf,mBAAmB;YACnB,cAAc;YACd,sBAAsB;YACtB,oBAAoB;YACpB,oBAAoB;SACrB;QACD,gBAAgB,EAAE;YAChB,0BAA0B;YAC1B,eAAe;YACf,yBAAyB;YACzB,yBAAyB;SAC1B;QACD,UAAU,EAAE;YACV,EAAE,GAAG,EAAE,aAAa,EAAE,WAAW,EAAE,oDAAoD,EAAE;YACzF;gBACE,GAAG,EAAE,eAAe;gBACpB,WAAW,EAAE,qEAAqE;aACnF;YACD;gBACE,GAAG,EAAE,mBAAmB;gBACxB,WAAW,EAAE,iEAAiE;aAC/E;YACD,EAAE,GAAG,EAAE,kBAAkB,EAAE,WAAW,EAAE,+CAA+C,EAAE;SAC1F;KACF;CACF,CAAA;AAED,SAAS,cAAc,CAAC,EAAsB;IAC5C,MAAM,QAAQ,GAAG,aAAa,CAAC,EAAE,CAAC,CAAA;IAClC,OAAO;QACL,EAAE;QACF,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC;QACxB,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,QAAQ,EAAE;YACR,eAAe,EAAE,QAAQ,CAAC,eAAe;YACzC,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;YAC3C,aAAa,EAAE,uBAAuB;SACvC;QACD,mBAAmB,EAAE,6BAA6B,CAAC,EAAE,CAAC;QACtD,eAAe,EAAE,yBAAyB,CAAC,EAAE,CAAC;KAC/C,CAAA;AACH,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAgC;IACzD,cAAc,CAAC,SAAS,CAAC;IACzB,cAAc,CAAC,SAAS,CAAC;IACzB,cAAc,CAAC,SAAS,CAAC;IACzB,cAAc,CAAC,SAAS,CAAC;IACzB,cAAc,CAAC,SAAS,CAAC;IACzB,cAAc,CAAC,SAAS,CAAC;CAC1B,CAAA;AAED,MAAM,UAAU,uBAAuB,CACrC,UAA4B,EAC5B,KAAuB;IAEvB,OAAO,UAAU,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QAC7D,EAAE,EAAE,GAAG,UAAU,CAAC,EAAE,gBAAgB,KAAK,GAAG,CAAC,EAAE;QAC/C,IAAI,EAAE,kBAAkB,UAAU,CAAC,QAAQ,EAAE;QAC7C,IAAI,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC,EAAE,EAAE,aAAa,EAAE,KAAK,CAAC;QACpD,MAAM,EAAE,UAAU,CAAC,EAAE;QACrB,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,eAAe,EACb,KAAK,KAAK,MAAM;YACd,CAAC,CAAC,OAAO,CAAC,UAAU;YACpB,CAAC,CAAC,KAAK,KAAK,KAAK;gBACf,CAAC,CAAC,OAAO,CAAC,SAAS;gBACnB,CAAC,CAAC,OAAO,CAAC,aAAa;QAC7B,cAAc,EAAE,KAAK;KACtB,CAAC,CAAC,CAAA;AACL,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,UAA4B;IAC9D,OAAO,UAAU,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACzD,EAAE,EAAE,GAAG,UAAU,CAAC,EAAE,YAAY,KAAK,GAAG,CAAC,EAAE;QAC3C,IAAI,EAAE,kBAAkB,UAAU,CAAC,QAAQ,EAAE;QAC7C,IAAI,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC;QACrE,MAAM,EAAE,UAAU,CAAC,EAAE;QACrB,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,eAAe,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;QAC3C,WAAW,EAAE,OAAO,CAAC,aAAa;KACnC,CAAC,CAAC,CAAA;AACL,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAA8B;IAChE,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,eAAe,EAAE,QAAQ,CAAC,eAAe;QACzC,GAAG,CAAC,QAAQ,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;KACrF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,EAAsB;IACtD,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;IAC1E,IAAI,UAAU,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,EAAE,EAAE,CAAC,CAAA;IACvF,OAAO,UAAU,CAAA;AACnB,CAAC"}

package/dist/eval/execution.d.ts

@@ -0,0 +1,46 @@
+import type { ChatClient } from '@tangle-network/agent-eval';
+import { type AgentProfile } from '@tangle-network/agent-interface';
+import type { EvalProviderIdentity, RecordedJudgeCall } from './types.js';
+export declare const DEFAULT_EVAL_BASE_URL = "https://router.tangle.tools/v1";
+export declare const DEFAULT_EVAL_MODEL = "glm-5.2";
+export declare const EVAL_TOTAL_COMPLETION_TOKENS = 2048;
+export declare const DEFAULT_EVAL_CALL_TIMEOUT_MS = 120000;
+export declare const DEFAULT_EVAL_TOTAL_TIMEOUT_MS: number;
+export interface EvalRouteConfig {
+    readonly baseUrl: string;
+    readonly model: string;
+    readonly apiKey?: string;
+    readonly timeoutMs: number;
+    readonly totalTimeoutMs: number;
+}
+export interface EvalRouteReady {
+    readonly status: 'ready';
+    readonly config: EvalRouteConfig;
+    readonly provider: EvalProviderIdentity;
+    readonly health: unknown;
+    readonly models: readonly string[];
+}
+export interface EvalRouteUnavailable {
+    readonly status: 'unavailable';
+    readonly config: EvalRouteConfig;
+    readonly provider: EvalProviderIdentity;
+    readonly reason: string;
+    readonly health: unknown | null;
+    readonly models: readonly string[];
+}
+export type EvalRouteProbe = EvalRouteReady | EvalRouteUnavailable;
+export declare function readEvalRouteConfig(env?: Readonly<Record<string, string | undefined>>): EvalRouteConfig;
+export declare function providerIdentity(config: EvalRouteConfig): EvalProviderIdentity;
+export declare function probeEvalRoute(config: EvalRouteConfig, fetchImpl?: typeof fetch): Promise<EvalRouteProbe>;
+type InjectedRouterCompletion = (body: Record<string, unknown>, request?: {
+    readonly headers: Readonly<Record<string, string>>;
+    readonly signal?: AbortSignal;
+}) => Promise<unknown>;
+export declare function evalJudgeProfile(config: EvalRouteConfig): AgentProfile;
+export declare function createEvalChatClient(config: EvalRouteConfig, complete?: InjectedRouterCompletion): ChatClient;
+export declare function recordingChatClient(client: ChatClient, calls: RecordedJudgeCall[], limits: {
+    readonly callTimeoutMs: number;
+    readonly totalTimeoutMs: number;
+}): ChatClient;
+export {};
+//# sourceMappingURL=execution.d.ts.map

package/dist/eval/execution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution.d.ts","sourceRoot":"","sources":["../../src/eval/execution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,UAAU,EAGX,MAAM,4BAA4B,CAAA;AACnC,OAAO,EAAE,KAAK,YAAY,EAAsB,MAAM,iCAAiC,CAAA;AAGvF,OAAO,KAAK,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAEzE,eAAO,MAAM,qBAAqB,mCAAmC,CAAA;AACrE,eAAO,MAAM,kBAAkB,YAAY,CAAA;AAC3C,eAAO,MAAM,4BAA4B,OAAQ,CAAA;AACjD,eAAO,MAAM,4BAA4B,SAAU,CAAA;AACnD,eAAO,MAAM,6BAA6B,QAAc,CAAA;AAExD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;CAChC;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAA;IAChC,QAAQ,CAAC,QAAQ,EAAE,oBAAoB,CAAA;IACvC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAA;IAC9B,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAA;IAChC,QAAQ,CAAC,QAAQ,EAAE,oBAAoB,CAAA;IACvC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAA;IAC/B,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED,MAAM,MAAM,cAAc,GAAG,cAAc,GAAG,oBAAoB,CAAA;AAqBlE,wBAAgB,mBAAmB,CACjC,GAAG,GAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAe,GAC9D,eAAe,CAsBjB;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,oBAAoB,CAQ9E;AAsCD,wBAAsB,cAAc,CAClC,MAAM,EAAE,eAAe,EACvB,SAAS,GAAE,OAAO,KAAa,GAC9B,OAAO,CAAC,cAAc,CAAC,CAgDzB;AAED,KAAK,wBAAwB,GAAG,CAC9B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,OAAO,CAAC,EAAE;IACR,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAClD,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAC9B,KACE,OAAO,CAAC,OAAO,CAAC,CAAA;AAErB,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,YAAY,CAgCtE;AAED,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,eAAe,EACvB,QAAQ,CAAC,EAAE,wBAAwB,GAClC,UAAU,CAaZ;AAsCD,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EAAE,EAC1B,MAAM,EAAE;IAAE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;CAAE,GAC1E,UAAU,CAyDZ"}