@tangle-network/braid 0.1.0

package/dist/eval/records.js

@@ -0,0 +1,253 @@
+import { createHash } from 'node:crypto';
+import { writeFile } from 'node:fs/promises';
+import { redactStructuredValueWithNumericTelemetry, } from '../domain/bounded-structured.js';
+import { canonicalJson } from '../domain/canonical.js';
+import { SEMANTIC_EVAL_CASE_IDS } from './types.js';
+/** Deliberate values used by tests and live-record redaction checks. */
+export const EVAL_SECRET_CANARIES = Object.freeze([
+    'BRAID_EVAL_SECRET_CANARY',
+    'braid-eval-secret-canary',
+    'BRAID_EVAL_SECRET',
+    'CANARY',
+]);
+export const SEMANTIC_EVAL_RECORD_SCHEMA_VERSION = 2;
+const EVAL_ARTIFACT_REDACTION_LIMITS = Object.freeze({
+    maxDepth: 24,
+    maxItems: 100_000,
+    maxBytes: 4 * 1024 * 1024,
+});
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
+}
+function replaceEvalCanaries(value) {
+    let output = value;
+    for (const canary of EVAL_SECRET_CANARIES) {
+        output = output.replace(new RegExp(escapeRegExp(canary), 'giu'), '[REDACTED_SECRET]');
+    }
+    return output;
+}
+export function containsEvalSecretCanary(value) {
+    const serialized = typeof value === 'string' ? value : (JSON.stringify(value) ?? '');
+    return EVAL_SECRET_CANARIES.some((canary) => serialized.toLocaleLowerCase().includes(canary.toLocaleLowerCase()));
+}
+export function redactEvalValue(value, _key, limits = {}) {
+    return redactStructuredValueWithNumericTelemetry(value, _key, {
+        ...limits,
+        redactionMarker: '[REDACTED]',
+        stringTransform: replaceEvalCanaries,
+    });
+}
+export function redactEvalArtifactValue(value, key) {
+    return redactEvalValue(value, key, EVAL_ARTIFACT_REDACTION_LIMITS);
+}
+export function evalCanonicalJson(value, limits = {}) {
+    return canonicalJson(redactEvalValue(value, undefined, limits));
+}
+export function evalSha256(value, limits = {}) {
+    return `sha256:${createHash('sha256').update(evalCanonicalJson(value, limits)).digest('hex')}`;
+}
+export function hashText(value) {
+    return `sha256:${createHash('sha256').update(value).digest('hex')}`;
+}
+export function hashArtifact(value) {
+    return evalSha256(value);
+}
+export function receiptView(receipt) {
+    return redactEvalValue({
+        callId: receipt.callId,
+        channel: receipt.channel,
+        phase: receipt.phase,
+        actor: receipt.actor,
+        model: receipt.model,
+        maximumCostUsd: receipt.maximumCostUsd ?? null,
+        tags: receipt.tags ?? null,
+        timestamp: receipt.timestamp,
+        status: receipt.status,
+        inputTokens: receipt.inputTokens,
+        outputTokens: receipt.outputTokens,
+        reasoningTokens: receipt.reasoningTokens ?? null,
+        cachedTokens: receipt.cachedTokens ?? 0,
+        cacheWriteTokens: receipt.cacheWriteTokens ?? 0,
+        costUsd: receipt.costUsd,
+        costUnknown: receipt.costUnknown,
+        usageUnknown: receipt.usageUnknown ?? false,
+        actualCostUsd: receipt.actualCostUsd ?? null,
+        estimatedCostUsd: receipt.estimatedCostUsd ?? null,
+        pricing: receipt.pricing ?? null,
+        error: receipt.error ?? null,
+    });
+}
+export function receiptsView(ledger) {
+    return ledger.list().map(receiptView);
+}
+export function cellEvidence(cell, rawCalls, criteriaThreshold, ledger) {
+    const callIds = cell.costCallIds ?? [];
+    const judgeScores = Object.fromEntries(Object.entries(cell.judgeScores).map(([key, score]) => [
+        key,
+        redactEvalValue(score),
+    ]));
+    const compositeValues = Object.values(cell.judgeScores)
+        .filter((score) => score.failed !== true)
+        .map((score) => score.composite);
+    const composite = compositeValues.length === 0 ? 0 : Math.min(...compositeValues);
+    const calls = rawCalls.filter((call) => call.callId !== null && callIds.includes(call.callId));
+    const productPath = cell.artifact.productPath === undefined
+        ? null
+        : redactEvalValue(cell.artifact.productPath);
+    return {
+        cellId: cell.cellId,
+        scenarioId: cell.scenarioId,
+        fixtureId: cell.artifact.fixtureId,
+        semanticOutput: redactEvalValue(cell.artifact.semanticOutput),
+        candidateOutput: redactEvalValue(cell.artifact.candidateOutput),
+        productPath,
+        judgeScores,
+        pass: cell.error === undefined &&
+            productPath?.available !== false &&
+            compositeValues.length > 0 &&
+            composite >= criteriaThreshold,
+        rawJudgeCallIds: calls.flatMap((call) => (call.callId === null ? [] : [call.callId])),
+        cost: cellCostEvidence(cell, rawCalls, ledger),
+    };
+}
+export function cellCostReceipts(cell, ledger) {
+    const callIds = new Set(cell.costCallIds ?? []);
+    return ledger.list().filter((receipt) => callIds.has(receipt.callId));
+}
+export function cellCostEvidence(cell, rawCalls, ledger) {
+    const callIds = new Set(cell.costCallIds ?? []);
+    const receipts = cellCostReceipts(cell, ledger);
+    const calls = rawCalls.filter((call) => call.callId !== null && callIds.has(call.callId));
+    const reasoning = receipts.flatMap((receipt) => receipt.reasoningTokens === undefined ? [] : [receipt.reasoningTokens]);
+    return {
+        usd: receipts.reduce((sum, receipt) => sum + receipt.costUsd, 0),
+        provenance: receipts.length === 0 ? null : receipts.map(receiptView),
+        inputTokens: receipts.reduce((sum, receipt) => sum + receipt.inputTokens, 0),
+        outputTokens: receipts.reduce((sum, receipt) => sum + receipt.outputTokens, 0),
+        reasoningTokens: reasoning.length === 0 ? null : reasoning.reduce((sum, value) => sum + value, 0),
+        cachedTokens: receipts.reduce((sum, receipt) => sum + (receipt.cachedTokens ?? 0), 0),
+        wallTimeMs: Math.max(cell.durationMs, ...calls.map((call) => call.wallTimeMs)),
+    };
+}
+export function recordHashMaterial(record) {
+    return redactEvalArtifactValue(record);
+}
+export function withArtifactHash(record) {
+    return {
+        ...record,
+        artifactHash: evalSha256(recordHashMaterial(record), EVAL_ARTIFACT_REDACTION_LIMITS),
+    };
+}
+function requiredNumber(value, label, errors) {
+    if (typeof value !== 'number' || !Number.isFinite(value) || value < 0)
+        errors.push(`${label} must be a finite non-negative number`);
+}
+export function validateSemanticEvalRecord(record) {
+    const errors = [];
+    if (record === null || typeof record !== 'object')
+        return ['record must be an object'];
+    const candidate = record;
+    if (candidate.schemaVersion !== SEMANTIC_EVAL_RECORD_SCHEMA_VERSION)
+        errors.push(`schemaVersion must be ${SEMANTIC_EVAL_RECORD_SCHEMA_VERSION}`);
+    if (typeof candidate.recordId !== 'string' || candidate.recordId.length === 0)
+        errors.push('recordId is required');
+    if (!['passed', 'failed', 'unavailable'].includes(candidate.status ?? ''))
+        errors.push('status is invalid');
+    if (typeof candidate.startedAt !== 'string' || typeof candidate.finishedAt !== 'string')
+        errors.push('timestamps are required');
+    requiredNumber(candidate.durationMs, 'durationMs', errors);
+    if (candidate.cases === undefined || !Array.isArray(candidate.cases))
+        errors.push('cases are required');
+    else {
+        const ids = candidate.cases.map((item) => item.id);
+        for (const id of SEMANTIC_EVAL_CASE_IDS) {
+            if (ids.filter((value) => value === id).length !== 1)
+                errors.push(`${id} must occur exactly once`);
+        }
+        if (candidate.status === 'passed') {
+            for (const evidence of candidate.cases) {
+                if (evidence.result !== 'passed')
+                    errors.push(`${evidence.id} must pass in a passed record`);
+                if (evidence.fixtureCount < 3)
+                    errors.push(`${evidence.id} requires at least three release fixtures`);
+                if (evidence.passedFixtures !== evidence.fixtureCount ||
+                    evidence.failedFixtures !== 0 ||
+                    evidence.cells.length !== evidence.fixtureCount) {
+                    errors.push(`${evidence.id} fixture results are incomplete`);
+                }
+                if (evidence.productFailures.length > 0 || evidence.campaign === null) {
+                    errors.push(`${evidence.id} product or campaign evidence is incomplete`);
+                }
+            }
+        }
+    }
+    if (candidate.calibration === undefined)
+        errors.push('calibration is required');
+    else {
+        if (!Array.isArray(candidate.calibration.pairOutcomes))
+            errors.push('calibration pair outcomes are required');
+        if (!Array.isArray(candidate.calibration.perCategory) ||
+            candidate.calibration.perCategory.length !== 6)
+            errors.push('calibration must record six category outcomes');
+        if (candidate.status === 'passed' && candidate.calibration.passed !== true)
+            errors.push('passed record requires passed calibration');
+    }
+    if (candidate.pilot === undefined || candidate.pilot.caseId !== 'EVAL-01')
+        errors.push('pilot must be EVAL-01');
+    else if (candidate.status === 'passed' &&
+        (candidate.pilot.status !== 'passed' || candidate.pilot.inspection.passed !== true)) {
+        errors.push('passed record requires a passed pilot');
+    }
+    if (candidate.rawJudgeCalls === undefined || !Array.isArray(candidate.rawJudgeCalls))
+        errors.push('raw judge calls are required');
+    else {
+        for (const call of candidate.rawJudgeCalls) {
+            if (call.response === null && call.error === null)
+                errors.push('judge call must have response or error');
+            if (call.response !== null && call.error !== null)
+                errors.push('judge call cannot have response and error');
+            requiredNumber(call.wallTimeMs, 'judge call wall time', errors);
+        }
+    }
+    if (candidate.streams?.stdout.captured !== false || candidate.streams.stderr.captured !== false)
+        errors.push('stream capture posture must be explicit');
+    if (candidate.status === 'passed') {
+        if (candidate.releaseAdmissible !== true)
+            errors.push('passed record must be release-admissible');
+        if (candidate.packageProvenance?.status !== 'ready') {
+            errors.push('passed record requires ready installed-package provenance');
+        }
+        if (!Array.isArray(candidate.releaseFailureReasons) ||
+            candidate.releaseFailureReasons.length > 0) {
+            errors.push('passed record cannot contain release failure reasons');
+        }
+        if (candidate.unavailableReason !== null)
+            errors.push('passed record cannot be unavailable');
+    }
+    else if (candidate.releaseAdmissible === true) {
+        errors.push('only a passed record can be release-admissible');
+    }
+    if (candidate.artifactHash !== undefined) {
+        const material = { ...candidate };
+        delete material.artifactHash;
+        const expected = evalSha256(material, EVAL_ARTIFACT_REDACTION_LIMITS);
+        if (candidate.artifactHash !== expected)
+            errors.push('artifactHash does not match canonical record');
+    }
+    else
+        errors.push('artifactHash is required');
+    if (containsEvalSecretCanary(record))
+        errors.push('record contains an unredacted seeded secret canary');
+    return errors;
+}
+export function assertSemanticEvalRecord(record) {
+    const errors = validateSemanticEvalRecord(record);
+    if (errors.length > 0)
+        throw new Error(`Invalid semantic evaluation record: ${errors.join('; ')}`);
+}
+export async function writeJsonArtifact(path, artifactId, value) {
+    const content = `${evalCanonicalJson(value, EVAL_ARTIFACT_REDACTION_LIMITS)}\n`;
+    await writeFile(path, content, { encoding: 'utf8', mode: 0o600 });
+    return { artifactId, sha256: hashText(content), path };
+}
+//# sourceMappingURL=records.js.map

package/dist/eval/records.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"records.js","sourceRoot":"","sources":["../../src/eval/records.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAG5C,OAAO,EACL,yCAAyC,GAE1C,MAAM,iCAAiC,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAA;AAWtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAA;AAEnD,wEAAwE;AACxE,MAAM,CAAC,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,CAAC;IAChD,0BAA0B;IAC1B,0BAA0B;IAC1B,mBAAmB;IACnB,QAAQ;CACT,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAU,CAAA;AAE7D,MAAM,8BAA8B,GAAG,MAAM,CAAC,MAAM,CAAC;IACnD,QAAQ,EAAE,EAAE;IACZ,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;CAC1B,CAAC,CAAA;AAyEF,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,KAAK,CAAC,OAAO,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAA;AACtD,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACxC,IAAI,MAAM,GAAG,KAAK,CAAA;IAClB,KAAK,MAAM,MAAM,IAAI,oBAAoB,EAAE,CAAC;QAC1C,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,EAAE,mBAAmB,CAAC,CAAA;IACvF,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,KAAc;IACrD,MAAM,UAAU,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IACpF,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAC1C,UAAU,CAAC,iBAAiB,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC,CACpE,CAAA;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,KAAc,EACd,IAAa,EACb,MAAM,GAA8B,EAAE;IAEtC,OAAO,yCAAyC,CAAC,KAAK,EAAE,IAAI,EAAE;QAC5D,GAAG,MAAM;QACT,eAAe,EAAE,YAAY;QAC7B,eAAe,EAAE,mBAAmB;KACrC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,KAAc,EAAE,GAAY;IAClE,OAAO,eAAe,CAAC,KAAK,EAAE,GAAG,EAAE,8BAA8B,CAAC,CAAA;AACpE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAc,EAAE,MAAM,GAA8B,EAAE;IACtF,OAAO,aAAa,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAA;AACjE,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAc,EAAE,MAAM,GAA8B,EAAE;IAC/E,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;AAChG,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAa;IACpC,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;AACrE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAc;IACzC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAA;AAC1B,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAoB;IAC9C,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI;QAC9C,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,IAAI;QAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI;QAChD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,CAAC;QACvC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,CAAC;QAC/C,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,KAAK;QAC3C,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI;QAC5C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,IAAI;QAClD,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI;KAC7B,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAkB;IAC7C,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,IAAiF,EACjF,QAAsC,EACtC,iBAAyB,EACzB,MAAkB;IAElB,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAA;IACtC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CACpC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;QACrD,GAAG;QACH,eAAe,CAAC,KAAK,CAAiB;KACvC,CAAC,CACH,CAAA;IACD,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;SACpD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC;SACxC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAClC,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAA;IACjF,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAC9F,MAAM,WAAW,GACf,IAAI,CAAC,QAAQ,CAAC,WAAW,KAAK,SAAS;QACrC,CAAC,CAAC,IAAI;QACN,CAAC,CAAE,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAmB,CAAA;IACnE,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;QAClC,cAAc,EAAE,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;QAC7D,eAAe,EAAE,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAW;QACzE,WAAW;QACX,WAAW;QACX,IAAI,EACF,IAAI,CAAC,KAAK,KAAK,SAAS;YACxB,WAAW,EAAE,SAAS,KAAK,KAAK;YAChC,eAAe,CAAC,MAAM,GAAG,CAAC;YAC1B,SAAS,IAAI,iBAAiB;QAChC,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACrF,IAAI,EAAE,gBAAgB,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC;KAC/C,CAAA;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,IAAiF,EACjF,MAAkB;IAElB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;IAC/C,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;AACvE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,IAAiF,EACjF,QAAsC,EACtC,MAAkB;IAElB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;IAC/C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAC/C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IACzF,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAC7C,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,CACvE,CAAA;IACD,OAAO;QACL,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAChE,UAAU,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC;QACpE,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5E,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;QAC9E,eAAe,EACb,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC;QAClF,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACrF,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;KAC/E,CAAA;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,MAAgD;IACjF,OAAO,uBAAuB,CAAC,MAAM,CAAC,CAAA;AACxC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,MAAgD;IAEhD,OAAO;QACL,GAAG,MAAM;QACT,YAAY,EAAE,UAAU,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,8BAA8B,CAAC;KACrF,CAAA;AACH,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,KAAa,EAAE,MAAgB;IACrE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QACnE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,uCAAuC,CAAC,CAAA;AAChE,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,MAAe;IACxD,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,CAAC,0BAA0B,CAAC,CAAA;IACtF,MAAM,SAAS,GAAG,MAAqC,CAAA;IACvD,IAAI,SAAS,CAAC,aAAa,KAAK,mCAAmC;QACjE,MAAM,CAAC,IAAI,CAAC,yBAAyB,mCAAmC,EAAE,CAAC,CAAA;IAC7E,IAAI,OAAO,SAAS,CAAC,QAAQ,KAAK,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAC3E,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAA;IACrC,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC;QACvE,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;IAClC,IAAI,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ,IAAI,OAAO,SAAS,CAAC,UAAU,KAAK,QAAQ;QACrF,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA;IACxC,cAAc,CAAC,SAAS,CAAC,UAAU,EAAE,YAAY,EAAE,MAAM,CAAC,CAAA;IAC1D,IAAI,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC;QAClE,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;SAC9B,CAAC;QACJ,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAClD,KAAK,MAAM,EAAE,IAAI,sBAAsB,EAAE,CAAC;YACxC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAA;QAChD,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClC,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;gBACvC,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ;oBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,+BAA+B,CAAC,CAAA;gBAC5F,IAAI,QAAQ,CAAC,YAAY,GAAG,CAAC;oBAC3B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,2CAA2C,CAAC,CAAA;gBACxE,IACE,QAAQ,CAAC,cAAc,KAAK,QAAQ,CAAC,YAAY;oBACjD,QAAQ,CAAC,cAAc,KAAK,CAAC;oBAC7B,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,YAAY,EAC/C,CAAC;oBACD,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,iCAAiC,CAAC,CAAA;gBAC9D,CAAC;gBACD,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;oBACtE,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,6CAA6C,CAAC,CAAA;gBAC1E,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,SAAS,CAAC,WAAW,KAAK,SAAS;QAAE,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA;SAC1E,CAAC;QACJ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAA;QACvD,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC,WAAW,CAAC;YACjD,SAAS,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;YAE9C,MAAM,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAA;QAC9D,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,SAAS,CAAC,WAAW,CAAC,MAAM,KAAK,IAAI;YACxE,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAA;IAC5D,CAAC;IACD,IAAI,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS;QACvE,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAA;SACjC,IACH,SAAS,CAAC,MAAM,KAAK,QAAQ;QAC7B,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,KAAK,IAAI,CAAC,EACnF,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAA;IACtD,CAAC;IACD,IAAI,SAAS,CAAC,aAAa,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC;QAClF,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAA;SACxC,CAAC;QACJ,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;YAC3C,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;gBAC/C,MAAM,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAA;YACvD,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;gBAC/C,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAA;YAC1D,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,EAAE,MAAM,CAAC,CAAA;QACjE,CAAC;IACH,CAAC;IACD,IAAI,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,KAAK,KAAK,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,KAAK;QAC7F,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAA;IACxD,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAClC,IAAI,SAAS,CAAC,iBAAiB,KAAK,IAAI;YACtC,MAAM,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAA;QACzD,IAAI,SAAS,CAAC,iBAAiB,EAAE,MAAM,KAAK,OAAO,EAAE,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAA;QAC1E,CAAC;QACD,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,qBAAqB,CAAC;YAC/C,SAAS,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,EAC1C,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAA;QACrE,CAAC;QACD,IAAI,SAAS,CAAC,iBAAiB,KAAK,IAAI;YAAE,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAA;IAC9F,CAAC;SAAM,IAAI,SAAS,CAAC,iBAAiB,KAAK,IAAI,EAAE,CAAC;QAChD,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAA;IAC/D,CAAC;IACD,IAAI,SAAS,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAG,EAAE,GAAG,SAAS,EAAE,CAAA;QACjC,OAAQ,QAAsC,CAAC,YAAY,CAAA;QAC3D,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,8BAA8B,CAAC,CAAA;QACrE,IAAI,SAAS,CAAC,YAAY,KAAK,QAAQ;YACrC,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAA;IAC/D,CAAC;;QAAM,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAA;IAC9C,IAAI,wBAAwB,CAAC,MAAM,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;IACnE,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,MAAe;IACtD,MAAM,MAAM,GAAG,0BAA0B,CAAC,MAAM,CAAC,CAAA;IACjD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AACpG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,UAAkB,EAClB,KAAc;IAEd,MAAM,OAAO,GAAG,GAAG,iBAAiB,CAAC,KAAK,EAAE,8BAA8B,CAAC,IAAI,CAAA;IAC/E,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;IACjE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAA;AACxD,CAAC"}

package/dist/eval/release-decision.d.ts

@@ -0,0 +1,17 @@
+import type { SemanticEvalRecord } from './records.js';
+import type { RecordedJudgeCall, SemanticCaseEvidence } from './types.js';
+export interface SemanticReleaseDecisionInput {
+    readonly packageReady: boolean;
+    readonly calibration: SemanticEvalRecord['calibration'];
+    readonly pilot: SemanticEvalRecord['pilot'];
+    readonly cases: readonly SemanticCaseEvidence[];
+    readonly rawJudgeCalls: readonly RecordedJudgeCall[];
+}
+export interface SemanticReleaseDecision {
+    readonly status: 'passed' | 'failed';
+    readonly releaseAdmissible: boolean;
+    readonly reasons: readonly string[];
+}
+/** Decide semantic release eligibility from complete recorded evidence only. */
+export declare function semanticReleaseDecision(input: SemanticReleaseDecisionInput): SemanticReleaseDecision;
+//# sourceMappingURL=release-decision.d.ts.map

package/dist/eval/release-decision.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"release-decision.d.ts","sourceRoot":"","sources":["../../src/eval/release-decision.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AACtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAGzE,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAA;IAC9B,QAAQ,CAAC,WAAW,EAAE,kBAAkB,CAAC,aAAa,CAAC,CAAA;IACvD,QAAQ,CAAC,KAAK,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAA;IAC3C,QAAQ,CAAC,KAAK,EAAE,SAAS,oBAAoB,EAAE,CAAA;IAC/C,QAAQ,CAAC,aAAa,EAAE,SAAS,iBAAiB,EAAE,CAAA;CACrD;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACpC,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;CACpC;AAiED,gFAAgF;AAChF,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,4BAA4B,GAClC,uBAAuB,CA6BzB"}

package/dist/eval/release-decision.js

@@ -0,0 +1,79 @@
+import { SEMANTIC_EVAL_CASE_IDS } from './types.js';
+function finiteNonNegative(value) {
+    return typeof value === 'number' && Number.isFinite(value) && value >= 0;
+}
+function caseFailures(evidence, calls) {
+    const failures = [];
+    if (evidence.result !== 'passed')
+        failures.push(`${evidence.id} result is ${evidence.result}`);
+    if (evidence.fixtureCount < 3) {
+        failures.push(`${evidence.id} has ${evidence.fixtureCount} release fixtures; at least 3 are required`);
+    }
+    if (evidence.passedFixtures !== evidence.fixtureCount ||
+        evidence.failedFixtures !== 0 ||
+        evidence.cells.length !== evidence.fixtureCount) {
+        failures.push(`${evidence.id} passed ${evidence.passedFixtures}/${evidence.fixtureCount} fixtures with ${evidence.cells.length} recorded cells`);
+    }
+    if (evidence.productFailures.length > 0) {
+        failures.push(`${evidence.id} has ${evidence.productFailures.length} product presentation failures`);
+    }
+    if (evidence.campaign === null)
+        failures.push(`${evidence.id} has no campaign record`);
+    for (const cell of evidence.cells) {
+        if (!cell.pass)
+            failures.push(`${evidence.id}/${cell.fixtureId} did not pass`);
+        if (cell.productPath?.available !== true) {
+            failures.push(`${evidence.id}/${cell.fixtureId} did not use an available product presenter`);
+        }
+        if (cell.rawJudgeCallIds.length === 0) {
+            failures.push(`${evidence.id}/${cell.fixtureId} has no raw judge call`);
+        }
+        for (const callId of cell.rawJudgeCallIds) {
+            const call = calls.get(callId);
+            if (call === undefined || call.response === null || call.error !== null) {
+                failures.push(`${evidence.id}/${cell.fixtureId} judge call ${callId} is missing or failed`);
+            }
+        }
+        const cost = cell.cost;
+        if (!finiteNonNegative(cost.usd) ||
+            !finiteNonNegative(cost.inputTokens) ||
+            !finiteNonNegative(cost.outputTokens) ||
+            !finiteNonNegative(cost.cachedTokens) ||
+            !finiteNonNegative(cost.wallTimeMs) ||
+            cost.wallTimeMs === 0 ||
+            cost.provenance === null ||
+            cost.provenance === undefined) {
+            failures.push(`${evidence.id}/${cell.fixtureId} has incomplete usage, cost, or timing evidence`);
+        }
+    }
+    return failures;
+}
+/** Decide semantic release eligibility from complete recorded evidence only. */
+export function semanticReleaseDecision(input) {
+    const reasons = [];
+    if (!input.packageReady)
+        reasons.push('installed package provenance is unavailable');
+    if (!input.calibration.passed)
+        reasons.push('judge calibration did not pass');
+    if (input.pilot.status !== 'passed' || !input.pilot.inspection.passed) {
+        reasons.push('the pre-calibration pilot did not pass');
+    }
+    const calls = new Map(input.rawJudgeCalls.flatMap((call) => call.callId === null ? [] : [[call.callId, call]]));
+    for (const id of SEMANTIC_EVAL_CASE_IDS) {
+        const matches = input.cases.filter((evidence) => evidence.id === id);
+        if (matches.length !== 1) {
+            reasons.push(`${id} occurs ${matches.length} times instead of exactly once`);
+            continue;
+        }
+        const evidence = matches[0];
+        if (evidence !== undefined)
+            reasons.push(...caseFailures(evidence, calls));
+    }
+    const releaseAdmissible = reasons.length === 0;
+    return {
+        status: releaseAdmissible ? 'passed' : 'failed',
+        releaseAdmissible,
+        reasons,
+    };
+}
+//# sourceMappingURL=release-decision.js.map

package/dist/eval/release-decision.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"release-decision.js","sourceRoot":"","sources":["../../src/eval/release-decision.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAA;AAgBnD,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,YAAY,CACnB,QAA8B,EAC9B,KAA6C;IAE7C,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ;QAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,cAAc,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;IAC9F,IAAI,QAAQ,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CACX,GAAG,QAAQ,CAAC,EAAE,QAAQ,QAAQ,CAAC,YAAY,4CAA4C,CACxF,CAAA;IACH,CAAC;IACD,IACE,QAAQ,CAAC,cAAc,KAAK,QAAQ,CAAC,YAAY;QACjD,QAAQ,CAAC,cAAc,KAAK,CAAC;QAC7B,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,YAAY,EAC/C,CAAC;QACD,QAAQ,CAAC,IAAI,CACX,GAAG,QAAQ,CAAC,EAAE,WAAW,QAAQ,CAAC,cAAc,IAAI,QAAQ,CAAC,YAAY,kBAAkB,QAAQ,CAAC,KAAK,CAAC,MAAM,iBAAiB,CAClI,CAAA;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,QAAQ,CAAC,IAAI,CACX,GAAG,QAAQ,CAAC,EAAE,QAAQ,QAAQ,CAAC,eAAe,CAAC,MAAM,gCAAgC,CACtF,CAAA;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,QAAQ,KAAK,IAAI;QAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,yBAAyB,CAAC,CAAA;IACtF,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,IAAI,IAAI,CAAC,SAAS,eAAe,CAAC,CAAA;QAC9E,IAAI,IAAI,CAAC,WAAW,EAAE,SAAS,KAAK,IAAI,EAAE,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,IAAI,IAAI,CAAC,SAAS,6CAA6C,CAAC,CAAA;QAC9F,CAAC;QACD,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtC,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,IAAI,IAAI,CAAC,SAAS,wBAAwB,CAAC,CAAA;QACzE,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAC9B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;gBACxE,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,IAAI,IAAI,CAAC,SAAS,eAAe,MAAM,uBAAuB,CAAC,CAAA;YAC7F,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;QACtB,IACE,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC;YAC5B,CAAC,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC;YACpC,CAAC,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC;YACrC,CAAC,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC;YACrC,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC;YACnC,IAAI,CAAC,UAAU,KAAK,CAAC;YACrB,IAAI,CAAC,UAAU,KAAK,IAAI;YACxB,IAAI,CAAC,UAAU,KAAK,SAAS,EAC7B,CAAC;YACD,QAAQ,CAAC,IAAI,CACX,GAAG,QAAQ,CAAC,EAAE,IAAI,IAAI,CAAC,SAAS,iDAAiD,CAClF,CAAA;QACH,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,uBAAuB,CACrC,KAAmC;IAEnC,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,IAAI,CAAC,KAAK,CAAC,YAAY;QAAE,OAAO,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;IACpF,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM;QAAE,OAAO,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAA;IAC7E,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QACtE,OAAO,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAA;IACxD,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACnC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAW,CAC7D,CACF,CAAA;IACD,KAAK,MAAM,EAAE,IAAI,sBAAsB,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;QACpE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,OAAO,CAAC,MAAM,gCAAgC,CAAC,CAAA;YAC5E,SAAQ;QACV,CAAC;QACD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QAC3B,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;IAC5E,CAAC;IAED,MAAM,iBAAiB,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAA;IAC9C,OAAO;QACL,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QAC/C,iBAAiB;QACjB,OAAO;KACR,CAAA;AACH,CAAC"}

package/dist/eval/release-markers.d.ts

@@ -0,0 +1,10 @@
+import type { SemanticEvalRecord } from './records.js';
+export interface SemanticEvalMeasurement {
+    readonly kind: 'scalar';
+    readonly name: string;
+    readonly unit: string;
+    readonly value: number;
+}
+/** Complete scalar summary consumed by the signed outer release collector. */
+export declare function semanticEvalMeasurements(record: SemanticEvalRecord): readonly SemanticEvalMeasurement[];
+//# sourceMappingURL=release-markers.d.ts.map

package/dist/eval/release-markers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"release-markers.d.ts","sourceRoot":"","sources":["../../src/eval/release-markers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAEtD,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAcD,8EAA8E;AAC9E,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,kBAAkB,GACzB,SAAS,uBAAuB,EAAE,CAkFpC"}

package/dist/eval/release-markers.js

@@ -0,0 +1,53 @@
+function scalar(name, unit, value) {
+    if (!Number.isFinite(value))
+        throw new Error(`Semantic evaluation measurement ${name} is not finite`);
+    return { kind: 'scalar', name, unit, value };
+}
+function receiptNumber(receipt, key) {
+    if (receipt === null || typeof receipt !== 'object')
+        return 0;
+    const value = receipt[key];
+    return typeof value === 'number' && Number.isFinite(value) ? value : 0;
+}
+/** Complete scalar summary consumed by the signed outer release collector. */
+export function semanticEvalMeasurements(record) {
+    if (record.status !== 'passed' || !record.releaseAdmissible) {
+        throw new Error('Only an admissible passed semantic evaluation can emit release measurements');
+    }
+    const fixtures = record.cases.flatMap((evidence) => evidence.cells);
+    const successfulCalls = record.rawJudgeCalls.filter((call) => call.response !== null && call.error === null);
+    const reasoningReceipts = record.receipts.filter((receipt) => receiptNumber(receipt, 'reasoningTokens') > 0);
+    const measurements = [
+        scalar('semantic-cases-passed', 'count', record.cases.filter((entry) => entry.result === 'passed').length),
+        scalar('semantic-fixtures-passed', 'count', fixtures.filter((entry) => entry.pass).length),
+        scalar('semantic-fixtures-total', 'count', fixtures.length),
+        scalar('calibration-pairs', 'count', record.calibration.pairedExamples),
+        scalar('calibration-good-examples', 'count', record.calibration.goodExamples),
+        scalar('calibration-good-accepted', 'count', record.calibration.goodAccepted),
+        scalar('calibration-good-preferred', 'count', record.calibration.goodPreferred),
+        scalar('calibration-ties', 'count', record.calibration.ties),
+        scalar('calibration-reversals', 'count', record.calibration.reversals),
+        scalar('calibration-preference-rate', 'ratio', record.calibration.pairPreferenceRate),
+        scalar('calibration-good-mean', 'score', record.calibration.strongMean),
+        scalar('calibration-bad-mean', 'score', record.calibration.weakMean),
+        scalar('calibration-trivial-mean', 'score', record.calibration.trivialMean),
+        scalar('calibration-good-bad-margin', 'score', record.calibration.strongWeakMargin),
+        scalar('calibration-trivial-good-ratio', 'ratio', record.calibration.trivialStrongRatio),
+        scalar('pilot-score', 'score', record.pilot.score ?? 0),
+        scalar('judge-calls-total', 'count', record.rawJudgeCalls.length),
+        scalar('judge-calls-successful', 'count', successfulCalls.length),
+        scalar('cost-receipts', 'count', record.receipts.length),
+        scalar('input-tokens', 'tokens', record.receipts.reduce((sum, receipt) => sum + receiptNumber(receipt, 'inputTokens'), 0)),
+        scalar('output-tokens', 'tokens', record.receipts.reduce((sum, receipt) => sum + receiptNumber(receipt, 'outputTokens'), 0)),
+        scalar('reasoning-tokens', 'tokens', record.receipts.reduce((sum, receipt) => sum + receiptNumber(receipt, 'reasoningTokens'), 0)),
+        scalar('reasoning-token-receipts', 'count', reasoningReceipts.length),
+        scalar('cached-tokens', 'tokens', record.receipts.reduce((sum, receipt) => sum + receiptNumber(receipt, 'cachedTokens'), 0)),
+        scalar('evaluation-cost', 'USD', record.receipts.reduce((sum, receipt) => sum + receiptNumber(receipt, 'costUsd'), 0)),
+        scalar('evaluation-wall-time', 'ms', record.durationMs),
+    ];
+    for (const evidence of record.cases) {
+        measurements.push(scalar(evidence.id, 'fixtures-passed', evidence.passedFixtures), scalar(`${evidence.id.toLowerCase()}-fixtures-passed`, 'count', evidence.passedFixtures));
+    }
+    return measurements;
+}
+//# sourceMappingURL=release-markers.js.map

package/dist/eval/release-markers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"release-markers.js","sourceRoot":"","sources":["../../src/eval/release-markers.ts"],"names":[],"mappings":"AASA,SAAS,MAAM,CAAC,IAAY,EAAE,IAAY,EAAE,KAAa;IACvD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,gBAAgB,CAAC,CAAA;IAC1E,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAA;AAC9C,CAAC;AAED,SAAS,aAAa,CAAC,OAAgB,EAAE,GAAW;IAClD,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAA;IAC7D,MAAM,KAAK,GAAI,OAA6C,CAAC,GAAG,CAAC,CAAA;IACjE,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,wBAAwB,CACtC,MAA0B;IAE1B,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAA;IAChG,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACnE,MAAM,eAAe,GAAG,MAAM,CAAC,aAAa,CAAC,MAAM,CACjD,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CACxD,CAAA;IACD,MAAM,iBAAiB,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAC9C,CAAC,OAAO,EAAE,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAC3D,CAAA;IACD,MAAM,YAAY,GAA8B;QAC9C,MAAM,CACJ,uBAAuB,EACvB,OAAO,EACP,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CACjE;QACD,MAAM,CAAC,0BAA0B,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC1F,MAAM,CAAC,yBAAyB,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC;QAC3D,MAAM,CAAC,mBAAmB,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC;QACvE,MAAM,CAAC,2BAA2B,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC;QAC7E,MAAM,CAAC,2BAA2B,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC;QAC7E,MAAM,CAAC,4BAA4B,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC;QAC/E,MAAM,CAAC,kBAAkB,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC;QAC5D,MAAM,CAAC,uBAAuB,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC;QACtE,MAAM,CAAC,6BAA6B,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,kBAAkB,CAAC;QACrF,MAAM,CAAC,uBAAuB,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC;QACvE,MAAM,CAAC,sBAAsB,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC;QACpE,MAAM,CAAC,0BAA0B,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC;QAC3E,MAAM,CAAC,6BAA6B,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC;QACnF,MAAM,CAAC,gCAAgC,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,kBAAkB,CAAC;QACxF,MAAM,CAAC,aAAa,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;QACvD,MAAM,CAAC,mBAAmB,EAAE,OAAO,EAAE,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC;QACjE,MAAM,CAAC,wBAAwB,EAAE,OAAO,EAAE,eAAe,CAAC,MAAM,CAAC;QACjE,MAAM,CAAC,eAAe,EAAE,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QACxD,MAAM,CACJ,cAAc,EACd,QAAQ,EACR,MAAM,CAAC,QAAQ,CAAC,MAAM,CACpB,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,aAAa,CAAC,OAAO,EAAE,aAAa,CAAC,EAC7D,CAAC,CACF,CACF;QACD,MAAM,CACJ,eAAe,EACf,QAAQ,EACR,MAAM,CAAC,QAAQ,CAAC,MAAM,CACpB,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,aAAa,CAAC,OAAO,EAAE,cAAc,CAAC,EAC9D,CAAC,CACF,CACF;QACD,MAAM,CACJ,kBAAkB,EAClB,QAAQ,EACR,MAAM,CAAC,QAAQ,CAAC,MAAM,CACpB,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,aAAa,CAAC,OAAO,EAAE,iBAAiB,CAAC,EACjE,CAAC,CACF,CACF;QACD,MAAM,CAAC,0BAA0B,EAAE,OAAO,EAAE,iBAAiB,CAAC,MAAM,CAAC;QACrE,MAAM,CACJ,eAAe,EACf,QAAQ,EACR,MAAM,CAAC,QAAQ,CAAC,MAAM,CACpB,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,aAAa,CAAC,OAAO,EAAE,cAAc,CAAC,EAC9D,CAAC,CACF,CACF;QACD,MAAM,CACJ,iBAAiB,EACjB,KAAK,EACL,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,aAAa,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC,CAC7F;QACD,MAAM,CAAC,sBAAsB,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC;KACxD,CAAA;IACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACpC,YAAY,CAAC,IAAI,CACf,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,iBAAiB,EAAE,QAAQ,CAAC,cAAc,CAAC,EAC/D,MAAM,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,kBAAkB,EAAE,OAAO,EAAE,QAAQ,CAAC,cAAc,CAAC,CACzF,CAAA;IACH,CAAC;IACD,OAAO,YAAY,CAAA;AACrB,CAAC"}

package/dist/eval/runner.d.ts

@@ -0,0 +1,14 @@
+import { type SemanticEvalRecord } from './records.js';
+export interface SemanticEvalRunOptions {
+    readonly rootDir?: string;
+    readonly env?: Readonly<Record<string, string | undefined>>;
+    readonly fetchImpl?: typeof fetch;
+    readonly outputDir?: string;
+    readonly costCeilingUsd?: number;
+}
+export interface SemanticEvalRunResult {
+    readonly record: SemanticEvalRecord;
+    readonly outputDir: string;
+}
+export declare function runSemanticEvaluation(options?: SemanticEvalRunOptions): Promise<SemanticEvalRunResult>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/eval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AA+BA,OAAO,EAGL,KAAK,kBAAkB,EAExB,MAAM,cAAc,CAAA;AAIrB,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAA;IAC3D,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,KAAK,CAAA;IACjC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CACjC;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,MAAM,EAAE,kBAAkB,CAAA;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;CAC3B;AAmFD,wBAAsB,qBAAqB,CACzC,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,qBAAqB,CAAC,CAoUhC"}