@danceiny/gotry 0.0.1-rc.16 → 0.0.1-rc.17

package/dist/src/evaluation-contracts.js

@@ -0,0 +1,906 @@
+import { createHash } from 'node:crypto';
+export const BENCHMARK_IDS = [
+    'trek',
+    'travelplanner',
+    'chinatravel',
+    'travelbench',
+    'tau2',
+    'locomo',
+    'bfcl'
+];
+export const FAILURE_CATEGORIES = [
+    'schema_or_format',
+    'grounding_or_provenance',
+    'constraint_or_feasibility',
+    'time_or_location_continuity',
+    'cost_or_cardinality',
+    'tool_selection_or_arguments',
+    'policy_or_write_safety',
+    'preference_elicitation',
+    'memory_retrieval_or_temporal_reasoning',
+    'reliability_cost_or_latency'
+];
+function obj(value, label) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`${label} must be an object`);
+    return value;
+}
+function exact(value, label, keys) {
+    const unknown = Object.keys(value).filter((key)=>!keys.includes(key));
+    const missing = keys.filter((key)=>!(key in value));
+    if (unknown.length) throw new Error(`${label} contains undeclared fields: ${unknown.join(',')}`);
+    if (missing.length) throw new Error(`${label} missing fields: ${missing.join(',')}`);
+}
+function text(value, label) {
+    if (typeof value !== 'string' || value.length === 0) throw new Error(`${label} must be non-empty text`);
+    return value;
+}
+function id(value, label) {
+    const out = text(value, label);
+    if (!/^[a-z0-9][a-z0-9:._-]+$/.test(out)) throw new Error(`${label} must be a stable identifier`);
+    return out;
+}
+function lit(value, label, values) {
+    const out = text(value, label);
+    if (!values.includes(out)) throw new Error(`${label} invalid literal`);
+    return out;
+}
+function boolean(value, label) {
+    if (typeof value !== 'boolean') throw new Error(`${label} must be boolean`);
+    return value;
+}
+function finite(value, label, min = 0) {
+    if (typeof value !== 'number' || !Number.isFinite(value) || value < min) throw new Error(`${label} must be a finite number >= ${min}`);
+    return value;
+}
+function integer(value, label, min = 0) {
+    const out = finite(value, label, min);
+    if (!Number.isInteger(out)) throw new Error(`${label} must be an integer`);
+    return out;
+}
+function strings(value, label, empty = false) {
+    if (!Array.isArray(value) || !empty && value.length === 0) throw new Error(`${label} must be an array`);
+    const out = value.map((item, index)=>text(item, `${label}[${index}]`));
+    if (new Set(out).size !== out.length) throw new Error(`${label} must be unique`);
+    return out;
+}
+function digest(value, label) {
+    const out = text(value, label);
+    if (!/^[0-9a-f]{64}$/.test(out)) throw new Error(`${label} must be lowercase SHA-256`);
+    return out;
+}
+function commit(value, label) {
+    const out = text(value, label);
+    if (!/^[0-9a-f]{40}$/.test(out)) throw new Error(`${label} must be a lowercase 40-character Git commit`);
+    return out;
+}
+function url(value, label) {
+    const out = text(value, label);
+    let parsed;
+    try {
+        parsed = new URL(out);
+    } catch  {
+        throw new Error(`${label} must be a valid URL`);
+    }
+    ;
+    if (parsed.protocol !== 'https:') throw new Error(`${label} must use https`);
+    if (parsed.username || parsed.password) throw new Error(`${label} must not contain credentials`);
+    return out;
+}
+function instant(value, label) {
+    const out = text(value, label);
+    const date = new Date(out);
+    if (!Number.isFinite(date.valueOf()) || date.toISOString() !== out) throw new Error(`${label} must be canonical ISO UTC with milliseconds`);
+    return out;
+}
+function zone(value, label) {
+    const out = text(value, label);
+    if (out === 'UTC') return out;
+    try {
+        new Intl.DateTimeFormat('en', {
+            timeZone: out
+        }).format(new Date(0));
+    } catch  {
+        throw new Error(`${label} must be UTC or an IANA timezone`);
+    }
+    ;
+    if (!out.includes('/')) throw new Error(`${label} must be UTC or an IANA timezone`);
+    return out;
+}
+function revision(value, label) {
+    const root = obj(value, label);
+    exact(root, label, [
+        'kind',
+        'value'
+    ]);
+    const kind = lit(root.kind, `${label}.kind`, [
+        'git_commit',
+        'content_sha256',
+        'not_separately_declared'
+    ]);
+    if (kind === 'not_separately_declared') {
+        if (root.value !== null) throw new Error(`${label}.value must be null`);
+        return {
+            kind,
+            value: null
+        };
+    }
+    ;
+    return {
+        kind,
+        value: kind === 'git_commit' ? commit(root.value, `${label}.value`) : digest(root.value, `${label}.value`)
+    };
+}
+function pin(value, label) {
+    const root = obj(value, label);
+    exact(root, label, [
+        'url',
+        'revision',
+        'source_scope'
+    ]);
+    return {
+        url: url(root.url, `${label}.url`),
+        revision: revision(root.revision, `${label}.revision`),
+        source_scope: text(root.source_scope, `${label}.source_scope`)
+    };
+}
+function licenseDetermination(value, label) {
+    const root = obj(value, label);
+    exact(root, label, [
+        'value',
+        'determination',
+        'source_url'
+    ]);
+    const determination = lit(root.determination, `${label}.determination`, [
+        'declared',
+        'not_separately_declared'
+    ]);
+    const resolved = text(root.value, `${label}.value`);
+    if (determination === 'not_separately_declared' && resolved !== 'not_separately_declared') throw new Error(`${label}.value must be not_separately_declared`);
+    return {
+        value: resolved,
+        determination,
+        source_url: url(root.source_url, `${label}.source_url`)
+    };
+}
+export function stableEvaluationJson(value) {
+    const sort = (item)=>Array.isArray(item) ? item.map(sort) : item && typeof item === 'object' ? Object.fromEntries(Object.entries(item).sort(([a], [b])=>a.localeCompare(b)).map(([key, child])=>[
+                key,
+                sort(child)
+            ])) : item;
+    return `${JSON.stringify(sort(value), null, 2)}\n`;
+}
+export function evaluationFingerprint(value) {
+    return createHash('sha256').update(stableEvaluationJson(value)).digest('hex');
+}
+const sensitiveKeys = new Set([
+    'apikey',
+    'accesstoken',
+    'refreshtoken',
+    'authorization',
+    'password',
+    'clientsecret',
+    'credentials',
+    'cookie',
+    'rawprompt',
+    'goldanswer',
+    'oraclepayload',
+    'credential',
+    'rawanswer',
+    'privatepayload',
+    'secretpayload',
+    'trajectorypayload'
+]);
+export function assertPublicArtifactSafe(value, label = 'artifact') {
+    const walk = (item, path)=>{
+        if (Array.isArray(item)) return item.forEach((child, index)=>walk(child, `${path}[${index}]`));
+        if (typeof item === 'string') {
+            const withoutHttpsUrls = item.replace(/https:\/\/[^\s]+/g, '');
+            const highConfidenceSecret = /Bearer\s+[A-Za-z0-9._~-]{24,}|\bsk-[A-Za-z0-9_-]{20,}|\bgh[pousr]_[A-Za-z0-9_]{20,}|\bAKIA[0-9A-Z]{16}\b/;
+            if (/(?:^|[^A-Za-z0-9])(?:\/|~\/)|[A-Za-z]:[\\/]|file:\/\//.test(withoutHttpsUrls) || highConfidenceSecret.test(item)) throw new Error(`${path} contains an absolute path or secret`);
+            return;
+        }
+        if (!item || typeof item !== 'object') return;
+        for (const [key, child] of Object.entries(item)){
+            if (sensitiveKeys.has(key.replace(/[^a-z0-9]/gi, '').toLowerCase()) && child !== '' && child !== null) throw new Error(`${path}.${key} contains credentials or raw sensitive payload`);
+            walk(child, `${path}.${key}`);
+        }
+    };
+    walk(value, label);
+}
+function registryEntry(value, label) {
+    const root = obj(value, label);
+    exact(root, label, [
+        'schema_version',
+        'benchmark_id',
+        'provenance',
+        'license',
+        'task_scopes',
+        'native_metrics',
+        'source_fence',
+        'countability_default'
+    ]);
+    const provenance = obj(root.provenance, `${label}.provenance`);
+    exact(provenance, `${label}.provenance`, [
+        'official_entry',
+        'data',
+        'evaluator'
+    ]);
+    const license = obj(root.license, `${label}.license`);
+    exact(license, `${label}.license`, [
+        'upstream_rights',
+        'repo_storage_policy'
+    ]);
+    const rights = obj(license.upstream_rights, `${label}.license.upstream_rights`);
+    exact(rights, `${label}.license.upstream_rights`, [
+        'code',
+        'data',
+        'evaluator'
+    ]);
+    const metrics = obj(root.native_metrics, `${label}.native_metrics`);
+    exact(metrics, `${label}.native_metrics`, [
+        'status',
+        'values'
+    ]);
+    if (!Array.isArray(metrics.values)) throw new Error(`${label}.native_metrics.values must be an array`);
+    const status = lit(metrics.status, `${label}.native_metrics.status`, [
+        'declared',
+        'not_separately_declared'
+    ]);
+    const values = metrics.values.map((raw, index)=>{
+        const metric = obj(raw, `${label}.metric[${index}]`);
+        exact(metric, `${label}.metric[${index}]`, [
+            'receipt_key',
+            'upstream_label',
+            'scope',
+            'source_url'
+        ]);
+        return {
+            receipt_key: id(metric.receipt_key, `${label}.metric.receipt_key`),
+            upstream_label: text(metric.upstream_label, `${label}.metric.upstream_label`),
+            scope: text(metric.scope, `${label}.metric.scope`),
+            source_url: url(metric.source_url, `${label}.metric.source_url`)
+        };
+    });
+    if (status === 'declared' && values.length === 0) throw new Error(`${label} declared metrics must be non-empty`);
+    if (status === 'not_separately_declared' && values.length !== 0) throw new Error(`${label} undeclared metrics must be empty`);
+    const fence = obj(root.source_fence, `${label}.source_fence`);
+    exact(fence, `${label}.source_fence`, [
+        'solver_allowed_field_classes',
+        'solver_forbidden_field_classes'
+    ]);
+    return {
+        schema_version: lit(root.schema_version, `${label}.schema_version`, [
+            'gotry_benchmark_registry_entry_v0'
+        ]),
+        benchmark_id: lit(root.benchmark_id, `${label}.benchmark_id`, BENCHMARK_IDS),
+        provenance: {
+            official_entry: pin(provenance.official_entry, `${label}.official_entry`),
+            data: pin(provenance.data, `${label}.data`),
+            evaluator: pin(provenance.evaluator, `${label}.evaluator`)
+        },
+        license: {
+            upstream_rights: {
+                code: licenseDetermination(rights.code, `${label}.rights.code`),
+                data: licenseDetermination(rights.data, `${label}.rights.data`),
+                evaluator: licenseDetermination(rights.evaluator, `${label}.rights.evaluator`)
+            },
+            repo_storage_policy: lit(license.repo_storage_policy, `${label}.storage`, [
+                'metadata_only_no_upstream_payload'
+            ])
+        },
+        task_scopes: strings(root.task_scopes, `${label}.task_scopes`),
+        native_metrics: status === 'declared' ? {
+            status,
+            values
+        } : {
+            status,
+            values: []
+        },
+        source_fence: {
+            solver_allowed_field_classes: strings(fence.solver_allowed_field_classes, `${label}.allowed`),
+            solver_forbidden_field_classes: strings(fence.solver_forbidden_field_classes, `${label}.forbidden`)
+        },
+        countability_default: lit(root.countability_default, `${label}.countability_default`, [
+            'countable_if_qualified',
+            'diagnostic_only'
+        ])
+    };
+}
+export function parseBenchmarkRegistry(value) {
+    if (!Array.isArray(value) || value.length !== 7) throw new Error('registry must contain exactly seven entries');
+    const out = value.map((item, index)=>registryEntry(item, `registry[${index}]`));
+    if (out.some((item, index)=>item.benchmark_id !== BENCHMARK_IDS[index])) throw new Error('registry must use canonical unique order');
+    return out;
+}
+export function parseEvalCase(value) {
+    const root = obj(value, 'case');
+    exact(root, 'case', [
+        'schema_version',
+        'case_id',
+        'benchmark_id',
+        'input_ref',
+        'isolation',
+        'clock',
+        'allowed_effects',
+        'forbidden_effects',
+        'budget',
+        'scorer_revision',
+        'public_safety'
+    ]);
+    const input = obj(root.input_ref, 'case.input_ref');
+    exact(input, 'case.input_ref', [
+        'kind',
+        'revision',
+        'digest_sha256'
+    ]);
+    const isolation = obj(root.isolation, 'case.isolation');
+    exact(isolation, 'case.isolation', [
+        'state_root',
+        'network',
+        'writes'
+    ]);
+    const clock = obj(root.clock, 'case.clock');
+    exact(clock, 'case.clock', [
+        'now',
+        'timezone'
+    ]);
+    const budget = obj(root.budget, 'case.budget');
+    exact(budget, 'case.budget', [
+        'max_seconds',
+        'max_cost_usd',
+        'max_tool_calls',
+        'max_turns'
+    ]);
+    const safety = obj(root.public_safety, 'case.public_safety');
+    exact(safety, 'case.public_safety', [
+        'contains_third_party_prompt',
+        'contains_gold',
+        'contains_oracle',
+        'contains_private_data'
+    ]);
+    for (const [key, flag] of Object.entries(safety))if (boolean(flag, `case.public_safety.${key}`) !== false) throw new Error(`case.public_safety.${key} must be false`);
+    const out = {
+        schema_version: lit(root.schema_version, 'case.schema_version', [
+            'gotry_eval_case_v0'
+        ]),
+        case_id: id(root.case_id, 'case.case_id'),
+        benchmark_id: lit(root.benchmark_id, 'case.benchmark_id', BENCHMARK_IDS),
+        input_ref: {
+            kind: lit(input.kind, 'case.input_ref.kind', [
+                'gotry_owned_synthetic',
+                'external_opaque_reference'
+            ]),
+            revision: revision(input.revision, 'case.input_ref.revision'),
+            digest_sha256: digest(input.digest_sha256, 'case.input_ref.digest_sha256')
+        },
+        isolation: {
+            state_root: lit(isolation.state_root, 'case.state_root', [
+                'ephemeral'
+            ]),
+            network: lit(isolation.network, 'case.network', [
+                'denied',
+                'benchmark_declared'
+            ]),
+            writes: lit(isolation.writes, 'case.writes', [
+                'forbidden'
+            ])
+        },
+        clock: {
+            now: instant(clock.now, 'case.clock.now'),
+            timezone: zone(clock.timezone, 'case.clock.timezone')
+        },
+        allowed_effects: strings(root.allowed_effects, 'case.allowed_effects', true),
+        forbidden_effects: strings(root.forbidden_effects, 'case.forbidden_effects'),
+        budget: {
+            max_seconds: finite(budget.max_seconds, 'case.max_seconds'),
+            max_cost_usd: finite(budget.max_cost_usd, 'case.max_cost_usd'),
+            max_tool_calls: integer(budget.max_tool_calls, 'case.max_tool_calls'),
+            max_turns: integer(budget.max_turns, 'case.max_turns')
+        },
+        scorer_revision: revision(root.scorer_revision, 'case.scorer_revision'),
+        public_safety: {
+            contains_third_party_prompt: false,
+            contains_gold: false,
+            contains_oracle: false,
+            contains_private_data: false
+        }
+    };
+    assertPublicArtifactSafe(out, 'case');
+    return out;
+}
+function optionalDigest(value, label) {
+    return value === null ? null : digest(value, label);
+}
+function numberMap(value, label, integers = false) {
+    const root = obj(value, label);
+    return Object.fromEntries(Object.entries(root).map(([key, item])=>[
+            id(key, `${label}.key`),
+            integers ? integer(item, `${label}.${key}`) : finite(item, `${label}.${key}`, Number.NEGATIVE_INFINITY)
+        ]));
+}
+export function parseEvalRunReceipt(value) {
+    const root = obj(value, 'run');
+    exact(root, 'run', [
+        'schema_version',
+        'run_id',
+        'benchmark_id',
+        'case_id',
+        'evidence_kind',
+        'pairing',
+        'status',
+        'started_at',
+        'finished_at',
+        'gotry_sha',
+        'model',
+        'controls',
+        'qualification',
+        'experiment',
+        'native_metrics',
+        'guardrails',
+        'evidence_summary'
+    ]);
+    const model = obj(root.model, 'run.model');
+    exact(model, 'run.model', [
+        'provider',
+        'model'
+    ]);
+    const controls = obj(root.controls, 'run.controls');
+    const controlKeys = [
+        'case_set_sha256',
+        'protocol_control_sha256',
+        'model_parameters_sha256',
+        'scorer_sha256',
+        'tool_snapshot_sha256',
+        'source_fence_sha256',
+        'integrity_sha256',
+        'official_evaluator_sha256'
+    ];
+    exact(controls, 'run.controls', controlKeys);
+    const qualification = obj(root.qualification, 'run.qualification');
+    exact(qualification, 'run.qualification', [
+        'official_result',
+        'source_fence_passed',
+        'integrity_passed',
+        'evidence_receipts'
+    ]);
+    const receipts = obj(qualification.evidence_receipts, 'run.qualification.evidence_receipts');
+    exact(receipts, 'run.qualification.evidence_receipts', [
+        'official_evaluator_output_sha256',
+        'source_fence_audit_sha256',
+        'integrity_audit_sha256'
+    ]);
+    const experiment = obj(root.experiment, 'run.experiment');
+    exact(experiment, 'run.experiment', [
+        'changed_variables',
+        'candidate_sha256'
+    ]);
+    const guard = obj(root.guardrails, 'run.guardrails');
+    exact(guard, 'run.guardrails', [
+        'hard_violation_count',
+        'forbidden_leakage_hits',
+        'latency_ms',
+        'cost_usd',
+        'tool_calls',
+        'turns'
+    ]);
+    const summary = obj(root.evidence_summary, 'run.evidence_summary');
+    exact(summary, 'run.evidence_summary', [
+        'artifact_classification',
+        'sha256',
+        'counts',
+        'reason_codes',
+        'fixture_only',
+        'statement'
+    ]);
+    let pairing = null;
+    if (root.pairing !== null) {
+        const item = obj(root.pairing, 'run.pairing');
+        exact(item, 'run.pairing', [
+            'pair_id',
+            'role',
+            'counterpart_run_id'
+        ]);
+        pairing = {
+            pair_id: id(item.pair_id, 'run.pair_id'),
+            role: lit(item.role, 'run.role', [
+                'baseline',
+                'treatment'
+            ]),
+            counterpart_run_id: id(item.counterpart_run_id, 'run.counterpart')
+        };
+    }
+    const evidence_kind = lit(root.evidence_kind, 'run.evidence_kind', [
+        'synthetic_fixture',
+        'observed_external'
+    ]);
+    const status = lit(root.status, 'run.status', [
+        'running',
+        'succeeded',
+        'failed',
+        'blocked'
+    ]);
+    const finished_at = root.finished_at === null ? null : instant(root.finished_at, 'run.finished_at');
+    if (status === 'running' !== (finished_at === null)) throw new Error('run.finished_at must be null only while running');
+    const out = {
+        schema_version: lit(root.schema_version, 'run.schema_version', [
+            'gotry_eval_run_receipt_v0'
+        ]),
+        run_id: id(root.run_id, 'run.run_id'),
+        benchmark_id: lit(root.benchmark_id, 'run.benchmark_id', BENCHMARK_IDS),
+        case_id: id(root.case_id, 'run.case_id'),
+        evidence_kind,
+        pairing,
+        status,
+        started_at: instant(root.started_at, 'run.started_at'),
+        finished_at,
+        gotry_sha: commit(root.gotry_sha, 'run.gotry_sha'),
+        model: {
+            provider: text(model.provider, 'run.provider'),
+            model: text(model.model, 'run.model')
+        },
+        controls: Object.fromEntries(controlKeys.map((key)=>[
+                key,
+                digest(controls[key], `run.controls.${key}`)
+            ])),
+        qualification: {
+            official_result: boolean(qualification.official_result, 'run.official_result'),
+            source_fence_passed: boolean(qualification.source_fence_passed, 'run.source_fence_passed'),
+            integrity_passed: boolean(qualification.integrity_passed, 'run.integrity_passed'),
+            evidence_receipts: {
+                official_evaluator_output_sha256: optionalDigest(receipts.official_evaluator_output_sha256, 'run.evaluator_receipt'),
+                source_fence_audit_sha256: optionalDigest(receipts.source_fence_audit_sha256, 'run.fence_receipt'),
+                integrity_audit_sha256: optionalDigest(receipts.integrity_audit_sha256, 'run.integrity_receipt')
+            }
+        },
+        experiment: {
+            changed_variables: strings(experiment.changed_variables, 'run.changed_variables', true),
+            candidate_sha256: digest(experiment.candidate_sha256, 'run.candidate')
+        },
+        native_metrics: numberMap(root.native_metrics, 'run.native_metrics'),
+        guardrails: {
+            hard_violation_count: integer(guard.hard_violation_count, 'run.hard'),
+            forbidden_leakage_hits: integer(guard.forbidden_leakage_hits, 'run.leakage'),
+            latency_ms: finite(guard.latency_ms, 'run.latency'),
+            cost_usd: finite(guard.cost_usd, 'run.cost'),
+            tool_calls: integer(guard.tool_calls, 'run.tools'),
+            turns: integer(guard.turns, 'run.turns')
+        },
+        evidence_summary: {
+            artifact_classification: lit(summary.artifact_classification, 'run.classification', [
+                'public_safe'
+            ]),
+            sha256: strings(summary.sha256, 'run.sha256', true).map((item, index)=>digest(item, `run.sha256[${index}]`)),
+            counts: numberMap(summary.counts, 'run.counts', true),
+            reason_codes: strings(summary.reason_codes, 'run.reason_codes', true),
+            fixture_only: boolean(summary.fixture_only, 'run.fixture_only'),
+            statement: text(summary.statement, 'run.statement')
+        }
+    };
+    if (out.finished_at !== null && new Date(out.finished_at).valueOf() < new Date(out.started_at).valueOf()) throw new Error('run.finished_at must not precede started_at');
+    if (evidence_kind === 'synthetic_fixture' && (pairing !== null || out.qualification.official_result || out.qualification.source_fence_passed || out.qualification.integrity_passed || Object.values(out.qualification.evidence_receipts).some(Boolean) || !out.evidence_summary.fixture_only)) throw new Error('synthetic fixture must be unmatched, unqualified, receipt-free, and fixture-only');
+    if (evidence_kind === 'observed_external' && out.evidence_summary.fixture_only) throw new Error('observed_external receipt must set fixture_only=false');
+    assertPublicArtifactSafe(out, 'run');
+    return out;
+}
+export function parseEvalFailureCluster(value) {
+    const root = obj(value, 'failure');
+    exact(root, 'failure', [
+        'schema_version',
+        'cluster_id',
+        'category',
+        'severity',
+        'benchmark_ids',
+        'case_ids',
+        'run_ids',
+        'reproduction',
+        'falsifiable_hypothesis',
+        'gotry_regression_id',
+        'suggested_surface',
+        'state'
+    ]);
+    const reproduction = obj(root.reproduction, 'failure.reproduction');
+    exact(reproduction, 'failure.reproduction', [
+        'condition',
+        'minimum_repetitions',
+        'observed_repetitions'
+    ]);
+    const out = {
+        schema_version: lit(root.schema_version, 'failure.schema_version', [
+            'gotry_eval_failure_cluster_v0'
+        ]),
+        cluster_id: id(root.cluster_id, 'failure.cluster_id'),
+        category: lit(root.category, 'failure.category', FAILURE_CATEGORIES),
+        severity: lit(root.severity, 'failure.severity', [
+            'P0',
+            'P1',
+            'P2',
+            'P3'
+        ]),
+        benchmark_ids: strings(root.benchmark_ids, 'failure.benchmark_ids').map((item)=>lit(item, 'failure.benchmark', BENCHMARK_IDS)),
+        case_ids: strings(root.case_ids, 'failure.case_ids'),
+        run_ids: strings(root.run_ids, 'failure.run_ids'),
+        reproduction: {
+            condition: text(reproduction.condition, 'failure.condition'),
+            minimum_repetitions: integer(reproduction.minimum_repetitions, 'failure.minimum', 1),
+            observed_repetitions: integer(reproduction.observed_repetitions, 'failure.observed')
+        },
+        falsifiable_hypothesis: text(root.falsifiable_hypothesis, 'failure.hypothesis'),
+        gotry_regression_id: id(root.gotry_regression_id, 'failure.regression'),
+        suggested_surface: lit(root.suggested_surface, 'failure.surface', [
+            'agent',
+            'tool_policy',
+            'deterministic',
+            'evaluation'
+        ]),
+        state: lit(root.state, 'failure.state', [
+            'observed',
+            'confirmed',
+            'resolved',
+            'rejected'
+        ])
+    };
+    if (out.state === 'confirmed' && out.reproduction.observed_repetitions < out.reproduction.minimum_repetitions) throw new Error('confirmed failure lacks repetitions');
+    assertPublicArtifactSafe(out, 'failure');
+    return out;
+}
+export function parseEvaluationFoundation(value) {
+    const root = obj(value, 'foundation');
+    exact(root, 'foundation', [
+        'registry',
+        'cases',
+        'run_receipts',
+        'failure_clusters'
+    ]);
+    const registry = parseBenchmarkRegistry(root.registry);
+    if (!Array.isArray(root.cases) || !Array.isArray(root.run_receipts) || !Array.isArray(root.failure_clusters)) throw new Error('foundation collections must be arrays');
+    const cases = root.cases.map(parseEvalCase), runs = root.run_receipts.map(parseEvalRunReceipt), failures = root.failure_clusters.map(parseEvalFailureCluster);
+    const unique = (values, label)=>{
+        if (new Set(values).size !== values.length) throw new Error(`${label} ids must be unique`);
+    };
+    unique(cases.map((item)=>item.case_id), 'case');
+    unique(runs.map((item)=>item.run_id), 'run');
+    unique(failures.map((item)=>item.cluster_id), 'failure');
+    const caseById = new Map(cases.map((item)=>[
+            item.case_id,
+            item
+        ])), runById = new Map(runs.map((item)=>[
+            item.run_id,
+            item
+        ])), registryById = new Map(registry.map((item)=>[
+            item.benchmark_id,
+            item
+        ]));
+    for (const run of runs){
+        const linked = caseById.get(run.case_id), entry = registryById.get(run.benchmark_id);
+        if (!linked || linked.benchmark_id !== run.benchmark_id || !entry) throw new Error(`run ${run.run_id} must close to case and registry`);
+        const metricKeys = Object.keys(run.native_metrics).sort(), declared = entry.native_metrics.values.map((metric)=>metric.receipt_key).sort();
+        if (JSON.stringify(metricKeys) !== JSON.stringify(declared)) throw new Error(`run ${run.run_id} metric keys must equal registry receipt keys`);
+    }
+    for (const failure of failures){
+        const linkedCases = failure.case_ids.map((key)=>caseById.get(key)), linkedRuns = failure.run_ids.map((key)=>runById.get(key));
+        if (linkedCases.some((item)=>!item) || linkedRuns.some((item)=>!item)) throw new Error(`failure ${failure.cluster_id} has unclosed links`);
+        const actual = new Set([
+            ...linkedCases,
+            ...linkedRuns
+        ].map((item)=>item.benchmark_id));
+        if (actual.size !== failure.benchmark_ids.length || failure.benchmark_ids.some((key)=>!actual.has(key))) throw new Error(`failure ${failure.cluster_id} benchmark closure must equal linked case and run benchmarks`);
+    }
+    return {
+        registry,
+        cases,
+        run_receipts: runs,
+        failure_clusters: failures
+    };
+}
+function runBinding(run) {
+    const { evidence_receipts: _receipts, ...qualification } = run.qualification;
+    return evaluationFingerprint({
+        ...run,
+        qualification
+    });
+}
+function resolveArtifact(resolver, run, kind, evalCase) {
+    const expected = kind === 'official_evaluator' ? run.qualification.evidence_receipts.official_evaluator_output_sha256 : kind === 'source_fence_audit' ? run.qualification.evidence_receipts.source_fence_audit_sha256 : run.qualification.evidence_receipts.integrity_audit_sha256;
+    if (!expected) throw new Error(`pair ${run.run_id} lacks qualification evidence receipts`);
+    const value = obj(resolver.resolve(expected), `${kind} artifact`);
+    const common = [
+        'schema_version',
+        'artifact_kind',
+        'run_id',
+        'benchmark_id',
+        'case_id',
+        'run_binding_sha256'
+    ];
+    const specific = kind === 'official_evaluator' ? [
+        'evaluator_sha256',
+        'native_metrics_sha256',
+        'native_metrics',
+        'official_result'
+    ] : kind === 'source_fence_audit' ? [
+        'source_fence_sha256',
+        'input_digest_sha256',
+        'source_fence_passed',
+        'forbidden_field_hits'
+    ] : [
+        'integrity_sha256',
+        'candidate_sha256',
+        'integrity_passed'
+    ];
+    exact(value, `${kind} artifact`, [
+        ...common,
+        ...specific
+    ]);
+    lit(value.schema_version, `${kind}.schema_version`, [
+        'gotry_eval_evidence_artifact_v0'
+    ]);
+    if (value.artifact_kind !== kind) throw new Error(`${kind} artifact kind mismatch`);
+    if (typeof value.run_id !== 'string' || typeof value.benchmark_id !== 'string' || typeof value.case_id !== 'string' || typeof value.run_binding_sha256 !== 'string') throw new Error(`${kind} artifact schema types mismatch`);
+    assertPublicArtifactSafe(value, ` artifact`);
+    if (kind === 'official_evaluator' && (typeof value.official_result !== 'boolean' || typeof value.native_metrics_sha256 !== 'string')) throw new Error(`${kind} artifact schema types mismatch`);
+    if (kind === 'source_fence_audit' && (typeof value.source_fence_passed !== 'boolean' || typeof value.forbidden_field_hits !== 'number' || !Number.isInteger(value.forbidden_field_hits))) throw new Error(`${kind} artifact schema types mismatch`);
+    if (kind === 'integrity_audit' && typeof value.integrity_passed !== 'boolean') throw new Error(`${kind} artifact schema types mismatch`);
+    if (evaluationFingerprint(value) !== expected) throw new Error(`pair ${run.run_id} ${kind} artifact fingerprint mismatch`);
+    if (value.run_id !== run.run_id || value.benchmark_id !== run.benchmark_id || value.case_id !== run.case_id || value.run_binding_sha256 !== runBinding(run)) throw new Error(`pair ${run.run_id} ${kind} artifact binding mismatch`);
+    if (kind === 'official_evaluator' && (value.evaluator_sha256 !== run.controls.official_evaluator_sha256 || value.native_metrics_sha256 !== evaluationFingerprint(run.native_metrics) || value.official_result !== true || JSON.stringify(value.native_metrics) !== JSON.stringify(run.native_metrics))) throw new Error(`pair ${run.run_id} evaluator artifact mismatch`);
+    if (kind === 'source_fence_audit' && (value.source_fence_sha256 !== run.controls.source_fence_sha256 || value.source_fence_passed !== true || value.forbidden_field_hits !== 0 || evalCase !== undefined && value.input_digest_sha256 !== evalCase.input_ref.digest_sha256)) throw new Error(`pair ${run.run_id} source-fence artifact mismatch`);
+    if (kind === 'integrity_audit' && (value.integrity_sha256 !== run.controls.integrity_sha256 || value.candidate_sha256 !== run.experiment.candidate_sha256 || value.integrity_passed !== true)) throw new Error(`pair ${run.run_id} integrity artifact mismatch`);
+    return value;
+}
+export function deriveMatchedPairs(foundation, resolver) {
+    const groups = new Map();
+    for (const run of foundation.run_receipts)if (run.pairing) groups.set(run.pairing.pair_id, [
+        ...groups.get(run.pairing.pair_id) ?? [],
+        run
+    ]);
+    const registry = new Map(foundation.registry.map((item)=>[
+            item.benchmark_id,
+            item
+        ])), cases = new Map(foundation.cases.map((item)=>[
+            item.case_id,
+            item
+        ]));
+    return [
+        ...groups.entries()
+    ].sort(([a], [b])=>a.localeCompare(b)).map(([pairId, members])=>{
+        if (members.length !== 2) throw new Error(`pair ${pairId} must contain exactly two receipts`);
+        const baseline = members.find((item)=>item.pairing?.role === 'baseline'), treatment = members.find((item)=>item.pairing?.role === 'treatment');
+        if (!baseline || !treatment) throw new Error(`pair ${pairId} must contain opposite baseline and treatment roles`);
+        if (baseline.pairing.counterpart_run_id !== treatment.run_id || treatment.pairing.counterpart_run_id !== baseline.run_id) throw new Error(`pair ${pairId} counterpart references must be reciprocal`);
+        if (baseline.benchmark_id !== treatment.benchmark_id || baseline.case_id !== treatment.case_id) throw new Error(`pair ${pairId} must share benchmark and case`);
+        const entry = registry.get(baseline.benchmark_id), evalCase = cases.get(baseline.case_id);
+        if (entry.countability_default !== 'countable_if_qualified') throw new Error(`pair ${pairId} benchmark is diagnostic only`);
+        if (evalCase.input_ref.kind !== 'external_opaque_reference') throw new Error(`pair ${pairId} requires an external opaque case reference`);
+        if ([
+            baseline,
+            treatment
+        ].some((run)=>run.evidence_kind !== 'observed_external')) throw new Error(`pair ${pairId} requires observed_external evidence`);
+        if (baseline.status !== 'succeeded' || treatment.status !== 'succeeded') throw new Error(`pair ${pairId} requires succeeded terminal receipts`);
+        if (baseline.model.provider !== treatment.model.provider || baseline.model.model !== treatment.model.model) throw new Error(`pair ${pairId} model identity must match`);
+        for (const key of [
+            'case_set_sha256',
+            'protocol_control_sha256',
+            'model_parameters_sha256',
+            'scorer_sha256',
+            'tool_snapshot_sha256',
+            'source_fence_sha256',
+            'integrity_sha256',
+            'official_evaluator_sha256'
+        ])if (baseline.controls[key] !== treatment.controls[key]) throw new Error(`pair ${pairId} control mismatch: ${key}`);
+        if (baseline.controls.scorer_sha256 !== evaluationFingerprint(evalCase.scorer_revision)) throw new Error(`pair ${pairId} scorer fingerprint does not close`);
+        if (baseline.controls.case_set_sha256 !== evaluationFingerprint(foundation.cases.filter((item)=>item.benchmark_id === baseline.benchmark_id))) throw new Error(`pair ${pairId} case-set fingerprint does not close`);
+        if (baseline.controls.source_fence_sha256 !== evaluationFingerprint(entry.source_fence)) throw new Error(`pair ${pairId} source-fence fingerprint does not close`);
+        if (baseline.controls.official_evaluator_sha256 !== evaluationFingerprint(entry.provenance.evaluator)) throw new Error(`pair ${pairId} evaluator fingerprint does not close`);
+        if (baseline.experiment.changed_variables.length !== 0 || JSON.stringify(treatment.experiment.changed_variables) !== JSON.stringify([
+            'gotry_sha'
+        ])) throw new Error(`pair ${pairId} treatment variable must be exactly gotry_sha`);
+        if (baseline.gotry_sha === treatment.gotry_sha) throw new Error(`pair ${pairId} baseline and treatment gotry_sha must differ`);
+        if (baseline.guardrails.hard_violation_count !== 0 || treatment.guardrails.hard_violation_count !== 0 || baseline.guardrails.forbidden_leakage_hits !== 0 || treatment.guardrails.forbidden_leakage_hits !== 0) throw new Error(`pair ${pairId} hard violation or leakage`);
+        for (const run of [
+            baseline,
+            treatment
+        ])if (run.experiment.candidate_sha256 !== evaluationFingerprint({
+            treatment_variable: 'gotry_sha',
+            gotry_sha: run.gotry_sha
+        })) throw new Error(`pair ${pairId} candidate fingerprint does not close to gotry_sha`);
+        for (const run of [
+            baseline,
+            treatment
+        ]){
+            const receipts = Object.values(run.qualification.evidence_receipts);
+            if (!run.qualification.official_result || !run.qualification.source_fence_passed || !run.qualification.integrity_passed || receipts.some((item)=>item === null)) throw new Error(`pair ${pairId} lacks qualification evidence receipts`);
+            const evaluator = resolveArtifact(resolver, run, 'official_evaluator'), fence = resolveArtifact(resolver, run, 'source_fence_audit', evalCase), integrity = resolveArtifact(resolver, run, 'integrity_audit');
+        }
+        return {
+            schema_version: 'gotry_eval_matched_pair_derived_v0',
+            pair_id: pairId,
+            benchmark_id: baseline.benchmark_id,
+            case_id: baseline.case_id,
+            baseline_run_id: baseline.run_id,
+            treatment_run_id: treatment.run_id,
+            treatment_variable: 'gotry_sha',
+            matched_pair_countable: true
+        };
+    });
+}
+export function parseMutationVectors(value) {
+    if (!Array.isArray(value)) throw new Error('mutation vectors must be an array');
+    return value.map((raw, index)=>{
+        const root = obj(raw, `mutation[${index}]`), kind = lit(root.value_kind, 'mutation.value_kind', [
+            'literal',
+            'synthetic_absolute_path',
+            'nan',
+            'duplicate_treatment_receipt'
+        ]), keys = kind === 'literal' ? [
+            'id',
+            'foundation_kind',
+            'target',
+            'operation',
+            'path',
+            'value_kind',
+            'value',
+            'expected_error'
+        ] : [
+            'id',
+            'foundation_kind',
+            'target',
+            'operation',
+            'path',
+            'value_kind',
+            'expected_error'
+        ];
+        exact(root, `mutation[${index}]`, keys);
+        return {
+            id: id(root.id, 'mutation.id'),
+            foundation_kind: lit(root.foundation_kind, 'mutation.foundation_kind', [
+                'diagnostic_repository',
+                'countable_test_only'
+            ]),
+            target: lit(root.target, 'mutation.target', [
+                'registry',
+                'case',
+                'run',
+                'failure',
+                'foundation'
+            ]),
+            operation: lit(root.operation, 'mutation.operation', [
+                'replace',
+                'remove',
+                'append'
+            ]),
+            path: text(root.path, 'mutation.path'),
+            value_kind: kind,
+            ...kind === 'literal' ? {
+                value: root.value
+            } : {},
+            expected_error: text(root.expected_error, 'mutation.expected_error')
+        };
+    });
+}
+export function applyMutationVector(foundation, vector) {
+    const cloned = JSON.parse(JSON.stringify(foundation));
+    const selected = vector.target === 'foundation' ? cloned : vector.target === 'registry' ? cloned.registry : vector.target === 'case' ? cloned.cases[0] : vector.target === 'run' ? cloned.run_receipts[0] : cloned.failure_clusters[0];
+    let value = vector.value;
+    if (vector.value_kind === 'synthetic_absolute_path') value = [
+        '',
+        'Users',
+        'fixture',
+        'private.json'
+    ].join('/');
+    if (vector.value_kind === 'nan') value = Number.NaN;
+    if (vector.value_kind === 'duplicate_treatment_receipt') {
+        const run = structuredClone(cloned.run_receipts[1]);
+        run.run_id = 'run:trek:treatment-duplicate';
+        value = run;
+    }
+    const parts = vector.path.split('.').filter(Boolean);
+    let parent = selected;
+    for (const part of parts.slice(0, -1))parent = parent[part];
+    const leaf = parts.at(-1);
+    if (vector.operation === 'remove') Array.isArray(parent) ? parent.splice(Number(leaf), 1) : delete parent[leaf];
+    else if (vector.operation === 'append') parent[leaf].push(value);
+    else parent[leaf] = value;
+    return selected;
+}
+
+
+//# sourceURL=ts/src/evaluation-contracts.ts