@danceiny/gotry 0.0.1-rc.9 → 0.2.0-rc.28

package/dist/scripts/evaluation-cadence-tests.js

@@ -0,0 +1,174 @@
+/**
+ * Contract tests for the evaluation cadence policy and planner.
+ * Run from ts/: npx tsx scripts/evaluation-cadence-tests.ts
+ */
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import assert from 'node:assert/strict';
+import { parseEvaluationCadencePolicy, planEvaluationCadence, } from '../src/evaluation-cadence.js';
+import { assertPublicArtifactSafe } from '../src/evaluation-contracts.js';
+const policyPath = join(process.cwd(), 'data', 'evaluation', 'cadence-policy.json');
+async function loadPolicyFixture() {
+    // This must remain a real fixture load: policy behavior is not tested with an inline mock.
+    return JSON.parse(await readFile(policyPath, 'utf8'));
+}
+function expectThrows(fn, message) {
+    assert.throws(fn, /./, message);
+}
+async function testCanonicalPolicyAndHealthyWeeklyPlan() {
+    const raw = await loadPolicyFixture();
+    const policy = parseEvaluationCadencePolicy(raw);
+    assertPublicArtifactSafe(policy, 'test policy safety');
+    assert.deepEqual(policy.levels.map((level) => level.cadence), ['pr', 'nightly', 'weekly', 'milestone']);
+    assert.deepEqual(policy.levels.map((level) => level.pass_k), [1, 2, 3, 5]);
+    assert.deepEqual(policy.levels.map((level) => level.max_cost_usd), [0, 2, 10, 50]);
+    assert.deepEqual(policy.levels.map((level) => level.max_wall_minutes), [20, 90, 360, 1440]);
+    assert.deepEqual(policy.levels.map((level) => level.max_tool_calls_per_case), [18, 18, 18, 18]);
+    assert.deepEqual(policy.levels.map((level) => [level.human_calibration.required, level.human_calibration.min_samples]), [[false, 0], [false, 0], [true, 10], [true, 30]]);
+    const decision = planEvaluationCadence(policy, {
+        cadence: 'weekly',
+        signals: [],
+        recent_optimization_prs: [81, 84, 87, 89],
+    });
+    assert.deepEqual(decision, {
+        cadence: 'weekly',
+        admission: true,
+        failure_registry: false,
+        stop_signals: [],
+        launches_external_runs: false,
+        pass_k: 3,
+        max_cost_usd: 10,
+        max_wall_minutes: 360,
+        max_tool_calls_per_case: 18,
+        human_calibration: { required: true, min_samples: 10 },
+        cross_benchmark_window: { min_prs: 3, max_prs: 5 },
+        performance_tradeoff_disclosure_threshold: 0.2,
+        delivery: {
+            exactly_one_gotry_pr: true,
+            exactly_one_discussion_78_comment: true,
+            update_existing_comment: true,
+        },
+        cross_benchmark_synthesis: { required: true, overdue: false },
+    });
+    assertPublicArtifactSafe(decision, 'test decision safety');
+}
+async function testStopSignalsDenyAdmissionAndRegisterFailures() {
+    const policy = parseEvaluationCadencePolicy(await loadPolicyFixture());
+    const decision = planEvaluationCadence(policy, {
+        cadence: 'weekly',
+        signals: ['source_revision_drift', 'hard_violation'],
+        recent_optimization_prs: [81, 84, 87],
+    });
+    assert.equal(decision.admission, false);
+    assert.deepEqual(decision.stop_signals, ['source_revision_drift', 'hard_violation']);
+    assert.equal(decision.failure_registry, true);
+    const nonRegistry = planEvaluationCadence(policy, {
+        cadence: 'weekly',
+        signals: ['license_unresolved'],
+        recent_optimization_prs: [81, 84, 87],
+    });
+    assert.equal(nonRegistry.admission, false);
+    assert.equal(nonRegistry.failure_registry, false);
+    const allSignals = planEvaluationCadence(policy, {
+        cadence: 'weekly',
+        signals: ['source_revision_drift', 'license_unresolved', 'scorer_ceiling_invalid', 'known_bad_no_longer_fails', 'hard_violation', 'matched_regression', 'variance_breach', 'budget_breach'],
+        recent_optimization_prs: [],
+    });
+    assert.equal(allSignals.admission, false);
+    assert.equal(allSignals.failure_registry, true);
+    const duplicatePrs = { cadence: 'weekly', signals: [], recent_optimization_prs: [81, 81, 82] };
+    expectThrows(() => planEvaluationCadence(policy, duplicatePrs), 'duplicate optimization PR IDs must fail closed');
+    const sparsePrs = [];
+    sparsePrs[2] = 81;
+    expectThrows(() => planEvaluationCadence(policy, { cadence: 'weekly', signals: [], recent_optimization_prs: sparsePrs }), 'sparse optimization PR IDs must fail closed');
+    expectThrows(() => planEvaluationCadence(policy, { cadence: 'weekly', signals: [], recent_optimization_prs: [0] }), 'non-positive optimization PR IDs must fail closed');
+    expectThrows(() => planEvaluationCadence(policy, { cadence: 'weekly', signals: [], recent_optimization_prs: [1.5] }), 'fractional optimization PR IDs must fail closed');
+}
+async function testRoundDeliveryAndMilestoneCalibration() {
+    const policy = parseEvaluationCadencePolicy(await loadPolicyFixture());
+    const milestone = planEvaluationCadence(policy, {
+        cadence: 'milestone',
+        signals: [],
+        recent_optimization_prs: [81, 84, 87, 89, 91],
+    });
+    assert.equal(milestone.admission, true);
+    assert.equal(milestone.pass_k, 5);
+    assert.deepEqual(milestone.human_calibration, { required: true, min_samples: 30 });
+    assert.deepEqual(milestone.cross_benchmark_window, { min_prs: 3, max_prs: 5 });
+    assert.equal(milestone.performance_tradeoff_disclosure_threshold, 0.2);
+    assert.deepEqual(milestone.delivery, {
+        exactly_one_gotry_pr: true,
+        exactly_one_discussion_78_comment: true,
+        update_existing_comment: true,
+    });
+    assert.deepEqual(milestone.cross_benchmark_synthesis, { required: true, overdue: false });
+    assert.deepEqual(planEvaluationCadence(policy, { cadence: 'weekly', signals: [], recent_optimization_prs: [] }).cross_benchmark_synthesis, { required: false, overdue: false });
+    assert.deepEqual(planEvaluationCadence(policy, { cadence: 'weekly', signals: [], recent_optimization_prs: [81, 82, 83, 84, 85, 86] }).cross_benchmark_synthesis, { required: true, overdue: true });
+}
+async function testMalformedPolicyFailsClosed() {
+    const fixture = await loadPolicyFixture();
+    const malformed = (change) => {
+        const copy = structuredClone(fixture);
+        change(copy);
+        expectThrows(() => parseEvaluationCadencePolicy(copy), 'malformed policy must fail closed');
+    };
+    malformed((copy) => { copy.unknown_key = true; });
+    malformed((copy) => { copy.levels[0].cadence = 'hourly'; });
+    malformed((copy) => { copy.levels[0].pass_k = -1; });
+    malformed((copy) => { copy.levels[0].pass_k = 1.5; });
+    malformed((copy) => { copy.levels = [...copy.levels].reverse(); });
+    malformed((copy) => { const sparse = []; sparse[3] = copy.levels[3]; copy.levels = sparse; });
+    malformed((copy) => { delete copy.stop_signals; });
+    const policy = parseEvaluationCadencePolicy(fixture);
+    const tamperedPolicy = structuredClone(policy);
+    tamperedPolicy.levels[2].max_cost_usd = 999;
+    expectThrows(() => planEvaluationCadence(tamperedPolicy, { cadence: 'weekly', signals: [], recent_optimization_prs: [] }), 'tampered policy budget must fail closed');
+    const tamperedDelivery = structuredClone(policy);
+    tamperedDelivery.round_delivery.exactly_one_gotry_pr = false;
+    expectThrows(() => planEvaluationCadence(tamperedDelivery, { cadence: 'weekly', signals: [], recent_optimization_prs: [] }), 'tampered policy delivery must fail closed');
+    expectThrows(() => planEvaluationCadence(policy, {
+        cadence: 'hourly',
+        signals: [],
+        recent_optimization_prs: [],
+    }), 'unknown cadence must fail closed');
+    expectThrows(() => planEvaluationCadence(policy, {
+        cadence: 'weekly',
+        signals: ['unknown_signal'],
+        recent_optimization_prs: [],
+    }), 'unknown stop signal must fail closed');
+    expectThrows(() => planEvaluationCadence(policy, {
+        cadence: 'weekly',
+        signals: ['hard_violation', 'hard_violation'],
+        recent_optimization_prs: [],
+    }), 'duplicate stop signals must fail closed');
+    const sparseSignals = [];
+    sparseSignals[1] = 'hard_violation';
+    expectThrows(() => planEvaluationCadence(policy, {
+        cadence: 'weekly',
+        signals: sparseSignals,
+        recent_optimization_prs: [],
+    }), 'sparse stop signals must fail closed');
+}
+async function testPolicyPublicSafety() {
+    const policy = parseEvaluationCadencePolicy(await loadPolicyFixture());
+    assert.deepEqual(policy.public_safety, {
+        artifact_classification: 'public_safe',
+        contains_third_party_prompt: false,
+        contains_gold: false,
+        contains_oracle: false,
+        contains_private_data: false,
+    });
+}
+async function main() {
+    await testCanonicalPolicyAndHealthyWeeklyPlan();
+    await testStopSignalsDenyAdmissionAndRegisterFailures();
+    await testRoundDeliveryAndMilestoneCalibration();
+    await testMalformedPolicyFailsClosed();
+    await testPolicyPublicSafety();
+    console.log('evaluation cadence tests: PASS');
+}
+main().catch((error) => {
+    console.error('evaluation cadence tests: FAIL');
+    console.error(error);
+    process.exitCode = 1;
+});

package/dist/scripts/evaluation-contract-tests.js

@@ -0,0 +1,268 @@
+import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { BENCHMARK_IDS, applyMutationVector, assertPublicArtifactSafe, deriveMatchedPairs, evaluationFingerprint, parseBenchmarkRegistry, parseEvalCase, parseEvalFailureCluster, parseEvalRunReceipt, parseEvaluationFoundation, parseMutationVectors, stableEvaluationJson, } from '../src/evaluation-contracts.js';
+const load = (file) => JSON.parse(readFileSync(file, 'utf8'));
+const registry = parseBenchmarkRegistry(load('data/evaluation/benchmark-registry.json'));
+assert.deepEqual(registry.map(item => item.benchmark_id), [...BENCHMARK_IDS]);
+assert.equal(registry.length, 7);
+assert.deepEqual(Object.fromEntries(registry.map(item => [item.benchmark_id, item.native_metrics.values.map(metric => metric.receipt_key)])), {
+    trek: ['task_perfect_feasible', 'task_perfect_infeasible', 'cat_efficiency'],
+    travelplanner: ['commonsense_micro_pass_rate', 'commonsense_macro_pass_rate', 'hard_micro_pass_rate', 'hard_macro_pass_rate', 'final_pass_rate'],
+    chinatravel: ['epr_micro', 'epr_macro', 'c_lpr', 'fpr', 'dav', 'att', 'ddr', 'overall_score'],
+    travelbench: ['reasoning_planning_score', 'summarization_extraction_score', 'presentation_score', 'user_interaction_score', 'average_score', 'unsolved_accuracy'],
+    tau2: ['avg_reward', 'pass_hat_1'], locomo: ['qa_f1'], bfcl: ['category_accuracy', 'overall_accuracy'],
+});
+const registryFacts = Object.fromEntries(registry.map(entry => [entry.benchmark_id, {
+        owner: new URL(entry.provenance.official_entry.url).pathname.split('/')[1],
+        pins: Object.entries(entry.provenance).map(([kind, pin]) => `${kind}|${pin.url}|${pin.revision.kind}|${pin.revision.value ?? 'null'}|${pin.source_scope}`),
+        rights: Object.entries(entry.license.upstream_rights).map(([kind, right]) => `${kind}|${right.value}|${right.determination}|${right.source_url}`),
+        metrics: entry.native_metrics.values.map(metric => `${metric.receipt_key}|${metric.upstream_label}|${metric.scope}|${metric.source_url}`),
+    }]));
+assert.deepEqual(registryFacts, {
+    "trek": { "owner": "TonyQJH", "pins": ["official_entry|https://github.com/TonyQJH/TREK-A-Travel-Reasoning-and-Evaluation-Kit-for-LLM-Agents-in-Complex-Trip-Planning/blob/6ceb4ebb2debd69c5c7c4ba34b5b17524756912b/README.md|git_commit|6ceb4ebb2debd69c5c7c4ba34b5b17524756912b|official definition", "data|https://github.com/TonyQJH/TREK-A-Travel-Reasoning-and-Evaluation-Kit-for-LLM-Agents-in-Complex-Trip-Planning/tree/6ceb4ebb2debd69c5c7c4ba34b5b17524756912b/api/data/v2|git_commit|6ceb4ebb2debd69c5c7c4ba34b5b17524756912b|task data v2", "evaluator|https://github.com/TonyQJH/TREK-A-Travel-Reasoning-and-Evaluation-Kit-for-LLM-Agents-in-Complex-Trip-Planning/blob/6ceb4ebb2debd69c5c7c4ba34b5b17524756912b/scoring.py|git_commit|6ceb4ebb2debd69c5c7c4ba34b5b17524756912b|official nine-dimension scoring implementation"], "rights": ["code|MIT|declared|https://github.com/TonyQJH/TREK-A-Travel-Reasoning-and-Evaluation-Kit-for-LLM-Agents-in-Complex-Trip-Planning/blob/6ceb4ebb2debd69c5c7c4ba34b5b17524756912b/LICENSE", "data|CC-BY-4.0|declared|https://github.com/TonyQJH/TREK-A-Travel-Reasoning-and-Evaluation-Kit-for-LLM-Agents-in-Complex-Trip-Planning/blob/6ceb4ebb2debd69c5c7c4ba34b5b17524756912b/README.md#data-and-license", "evaluator|MIT|declared|https://github.com/TonyQJH/TREK-A-Travel-Reasoning-and-Evaluation-Kit-for-LLM-Agents-in-Complex-Trip-Planning/blob/6ceb4ebb2debd69c5c7c4ba34b5b17524756912b/LICENSE"], "metrics": ["task_perfect_feasible|task_perfect_feasible|task-perfect feasible|https://github.com/TonyQJH/TREK-A-Travel-Reasoning-and-Evaluation-Kit-for-LLM-Agents-in-Complex-Trip-Planning/blob/6ceb4ebb2debd69c5c7c4ba34b5b17524756912b/scoring.py", "task_perfect_infeasible|task_perfect_infeasible|task-perfect infeasible|https://github.com/TonyQJH/TREK-A-Travel-Reasoning-and-Evaluation-Kit-for-LLM-Agents-in-Complex-Trip-Planning/blob/6ceb4ebb2debd69c5c7c4ba34b5b17524756912b/scoring.py", "cat_efficiency|cat_efficiency|category efficiency|https://github.com/TonyQJH/TREK-A-Travel-Reasoning-and-Evaluation-Kit-for-LLM-Agents-in-Complex-Trip-Planning/blob/6ceb4ebb2debd69c5c7c4ba34b5b17524756912b/scoring.py"] },
+    "travelplanner": { "owner": "OSU-NLP-Group", "pins": ["official_entry|https://github.com/OSU-NLP-Group/TravelPlanner/blob/e52c87f4ac348a3410c46dc3553c519db5ec5e23/README.md|git_commit|e52c87f4ac348a3410c46dc3553c519db5ec5e23|official definition", "data|https://huggingface.co/datasets/osunlp/TravelPlanner/tree/8736504ecfc31b7f8b7e40122873c337e83fff7c|git_commit|8736504ecfc31b7f8b7e40122873c337e83fff7c|official Hugging Face dataset git revision", "evaluator|https://github.com/OSU-NLP-Group/TravelPlanner/blob/e52c87f4ac348a3410c46dc3553c519db5ec5e23/evaluation/eval.py|git_commit|e52c87f4ac348a3410c46dc3553c519db5ec5e23|official evaluator"], "rights": ["code|MIT|declared|https://github.com/OSU-NLP-Group/TravelPlanner/blob/e52c87f4ac348a3410c46dc3553c519db5ec5e23/LICENSE", "data|CC-BY-4.0|declared|https://huggingface.co/datasets/osunlp/TravelPlanner/blob/8736504ecfc31b7f8b7e40122873c337e83fff7c/README.md", "evaluator|MIT|declared|https://github.com/OSU-NLP-Group/TravelPlanner/blob/e52c87f4ac348a3410c46dc3553c519db5ec5e23/LICENSE"], "metrics": ["commonsense_micro_pass_rate|Commonsense Constraint Micro Pass Rate|commonsense micro|https://github.com/OSU-NLP-Group/TravelPlanner/blob/e52c87f4ac348a3410c46dc3553c519db5ec5e23/evaluation/eval.py", "commonsense_macro_pass_rate|Commonsense Constraint Macro Pass Rate|commonsense macro|https://github.com/OSU-NLP-Group/TravelPlanner/blob/e52c87f4ac348a3410c46dc3553c519db5ec5e23/evaluation/eval.py", "hard_micro_pass_rate|Hard Constraint Micro Pass Rate|hard-constraint micro|https://github.com/OSU-NLP-Group/TravelPlanner/blob/e52c87f4ac348a3410c46dc3553c519db5ec5e23/evaluation/eval.py", "hard_macro_pass_rate|Hard Constraint Macro Pass Rate|hard-constraint macro|https://github.com/OSU-NLP-Group/TravelPlanner/blob/e52c87f4ac348a3410c46dc3553c519db5ec5e23/evaluation/eval.py", "final_pass_rate|Final Pass Rate|complete itinerary|https://github.com/OSU-NLP-Group/TravelPlanner/blob/e52c87f4ac348a3410c46dc3553c519db5ec5e23/evaluation/eval.py"] },
+    "chinatravel": { "owner": "chinatravel-competition", "pins": ["official_entry|https://github.com/chinatravel-competition/IJCAI2026/blob/49d02bc322dda7ffbf53dfb7c3d2ced6b4bd4e8b/index.html|git_commit|49d02bc322dda7ffbf53dfb7c3d2ced6b4bd4e8b|official competition entry", "data|https://github.com/chinatravel-competition/IJCAI2026/blob/49d02bc322dda7ffbf53dfb7c3d2ced6b4bd4e8b/TPC_IJCAI_2026_phase2_familiar_100_data.zip|git_commit|49d02bc322dda7ffbf53dfb7c3d2ced6b4bd4e8b|public familiar-track archive", "evaluator|https://github.com/LAMDA-NeSy/ChinaTravel/blob/b071db251905b14002ec98e8b36afca7b6d6cd04/eval_tpc.py|git_commit|b071db251905b14002ec98e8b36afca7b6d6cd04|official TPC evaluator implementation"], "rights": ["code|not_separately_declared|not_separately_declared|https://github.com/LAMDA-NeSy/ChinaTravel/blob/b071db251905b14002ec98e8b36afca7b6d6cd04/README.md", "data|CC-BY-NC-SA-4.0|declared|https://huggingface.co/datasets/LAMDA-NeSy/ChinaTravel/blob/44d5dbf3bba26bdf9a212c3e76d3242b67f0d349/README.md", "evaluator|not_separately_declared|not_separately_declared|https://github.com/LAMDA-NeSy/ChinaTravel/blob/b071db251905b14002ec98e8b36afca7b6d6cd04/README.md"], "metrics": ["epr_micro|EPR-micro|element pass micro|https://github.com/LAMDA-NeSy/ChinaTravel/blob/b071db251905b14002ec98e8b36afca7b6d6cd04/eval_tpc.py", "epr_macro|EPR-macro|element pass macro|https://github.com/LAMDA-NeSy/ChinaTravel/blob/b071db251905b14002ec98e8b36afca7b6d6cd04/eval_tpc.py", "c_lpr|C-LPR|constraint-level pass|https://github.com/LAMDA-NeSy/ChinaTravel/blob/b071db251905b14002ec98e8b36afca7b6d6cd04/eval_tpc.py", "fpr|FPR|final pass|https://github.com/LAMDA-NeSy/ChinaTravel/blob/b071db251905b14002ec98e8b36afca7b6d6cd04/eval_tpc.py", "dav|DAV|Daily Average Attractions Visited|https://github.com/LAMDA-NeSy/ChinaTravel/blob/b071db251905b14002ec98e8b36afca7b6d6cd04/eval_tpc.py", "att|ATT|Averaged Transportation Time|https://github.com/LAMDA-NeSy/ChinaTravel/blob/b071db251905b14002ec98e8b36afca7b6d6cd04/eval_tpc.py", "ddr|DDR|Daily Dining Recommendations|https://github.com/LAMDA-NeSy/ChinaTravel/blob/b071db251905b14002ec98e8b36afca7b6d6cd04/eval_tpc.py", "overall_score|Overall Score|weighted overall score|https://github.com/LAMDA-NeSy/ChinaTravel/blob/b071db251905b14002ec98e8b36afca7b6d6cd04/eval_tpc.py"] },
+    "travelbench": { "owner": "small-xiangcheng", "pins": ["official_entry|https://github.com/small-xiangcheng/TravelBench/blob/445a29d9a9b6457fc95fe647c532b6b79e21c43f/README.md|git_commit|445a29d9a9b6457fc95fe647c532b6b79e21c43f|official definition", "data|https://github.com/small-xiangcheng/TravelBench/tree/445a29d9a9b6457fc95fe647c532b6b79e21c43f/datas|git_commit|445a29d9a9b6457fc95fe647c532b6b79e21c43f|official data directory", "evaluator|https://github.com/small-xiangcheng/TravelBench/blob/445a29d9a9b6457fc95fe647c532b6b79e21c43f/travelbench/evaluation/evaluate.py|git_commit|445a29d9a9b6457fc95fe647c532b6b79e21c43f|official evaluator"], "rights": ["code|MIT|declared|https://github.com/small-xiangcheng/TravelBench/blob/445a29d9a9b6457fc95fe647c532b6b79e21c43f/LICENSE", "data|CC-BY-NC-4.0|declared|https://github.com/small-xiangcheng/TravelBench/blob/445a29d9a9b6457fc95fe647c532b6b79e21c43f/datas/LICENSE", "evaluator|MIT|declared|https://github.com/small-xiangcheng/TravelBench/blob/445a29d9a9b6457fc95fe647c532b6b79e21c43f/LICENSE"], "metrics": ["reasoning_planning_score|reasoning_planning_score|reasoning and planning|https://github.com/small-xiangcheng/TravelBench/blob/445a29d9a9b6457fc95fe647c532b6b79e21c43f/travelbench/evaluation/evaluate.py", "summarization_extraction_score|summarization_extraction_score|summarization and extraction|https://github.com/small-xiangcheng/TravelBench/blob/445a29d9a9b6457fc95fe647c532b6b79e21c43f/travelbench/evaluation/evaluate.py", "presentation_score|presentation_score|presentation|https://github.com/small-xiangcheng/TravelBench/blob/445a29d9a9b6457fc95fe647c532b6b79e21c43f/travelbench/evaluation/evaluate.py", "user_interaction_score|user_interaction_score|user interaction|https://github.com/small-xiangcheng/TravelBench/blob/445a29d9a9b6457fc95fe647c532b6b79e21c43f/travelbench/evaluation/evaluate.py", "average_score|average_score|average across dimensions|https://github.com/small-xiangcheng/TravelBench/blob/445a29d9a9b6457fc95fe647c532b6b79e21c43f/travelbench/evaluation/evaluate.py", "unsolved_accuracy|unsolved_accuracy|unsolved cases|https://github.com/small-xiangcheng/TravelBench/blob/445a29d9a9b6457fc95fe647c532b6b79e21c43f/travelbench/evaluation/evaluate_unsolved.py"] },
+    "tau2": { "owner": "sierra-research", "pins": ["official_entry|https://github.com/sierra-research/tau2-bench/blob/a2c024725189473d2d7cea3a5cfdbcc67478e41f/README.md|git_commit|a2c024725189473d2d7cea3a5cfdbcc67478e41f|official definition", "data|https://github.com/sierra-research/tau2-bench/tree/a2c024725189473d2d7cea3a5cfdbcc67478e41f/data/tau2|git_commit|a2c024725189473d2d7cea3a5cfdbcc67478e41f|official data", "evaluator|https://github.com/sierra-research/tau2-bench/blob/a2c024725189473d2d7cea3a5cfdbcc67478e41f/src/tau2/metrics/agent_metrics.py|git_commit|a2c024725189473d2d7cea3a5cfdbcc67478e41f|official agent metrics implementation"], "rights": ["code|MIT|declared|https://github.com/sierra-research/tau2-bench/blob/a2c024725189473d2d7cea3a5cfdbcc67478e41f/LICENSE", "data|not_separately_declared|not_separately_declared|https://github.com/sierra-research/tau2-bench/blob/a2c024725189473d2d7cea3a5cfdbcc67478e41f/README.md", "evaluator|MIT|declared|https://github.com/sierra-research/tau2-bench/blob/a2c024725189473d2d7cea3a5cfdbcc67478e41f/LICENSE"], "metrics": ["avg_reward|avg_reward|average reward|https://github.com/sierra-research/tau2-bench/blob/a2c024725189473d2d7cea3a5cfdbcc67478e41f/src/tau2/metrics/agent_metrics.py", "pass_hat_1|pass^1|mean task pass^1|https://github.com/sierra-research/tau2-bench/blob/a2c024725189473d2d7cea3a5cfdbcc67478e41f/src/tau2/metrics/agent_metrics.py"] },
+    "locomo": { "owner": "snap-research", "pins": ["official_entry|https://github.com/snap-research/locomo/blob/3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376/README.MD|git_commit|3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376|official definition", "data|https://github.com/snap-research/locomo/blob/3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376/data/locomo10.json|git_commit|3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376|official data", "evaluator|https://github.com/snap-research/locomo/tree/3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376/task_eval|git_commit|3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376|official task evaluation directory"], "rights": ["code|CC-BY-NC-4.0|declared|https://github.com/snap-research/locomo/blob/3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376/LICENSE.txt", "data|CC-BY-NC-4.0|declared|https://github.com/snap-research/locomo/blob/3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376/LICENSE.txt", "evaluator|CC-BY-NC-4.0|declared|https://github.com/snap-research/locomo/blob/3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376/LICENSE.txt"], "metrics": ["qa_f1|F1|question-answering token F1|https://github.com/snap-research/locomo/blob/3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376/task_eval/evaluate_qa.py"] },
+    "bfcl": { "owner": "ShishirPatil", "pins": ["official_entry|https://github.com/ShishirPatil/gorilla/blob/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/berkeley-function-call-leaderboard/README.md|git_commit|6ea57973c7a6097fd7c5915698c54c17c5b1b6c8|official BFCL definition", "data|https://github.com/ShishirPatil/gorilla/tree/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/berkeley-function-call-leaderboard/bfcl_eval/data|git_commit|6ea57973c7a6097fd7c5915698c54c17c5b1b6c8|BFCL data", "evaluator|https://github.com/ShishirPatil/gorilla/tree/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/berkeley-function-call-leaderboard/bfcl_eval|git_commit|6ea57973c7a6097fd7c5915698c54c17c5b1b6c8|BFCL evaluator"], "rights": ["code|Apache-2.0|declared|https://github.com/ShishirPatil/gorilla/blob/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/LICENSE", "data|Apache-2.0|declared|https://github.com/ShishirPatil/gorilla/blob/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/berkeley-function-call-leaderboard/README.md", "evaluator|Apache-2.0|declared|https://github.com/ShishirPatil/gorilla/blob/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/LICENSE"], "metrics": ["category_accuracy|accuracy|per test_category|https://github.com/ShishirPatil/gorilla/blob/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/berkeley-function-call-leaderboard/bfcl_eval/eval_checker/eval_runner_helper.py", "overall_accuracy|Overall Acc|overall accuracy|https://github.com/ShishirPatil/gorilla/blob/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/berkeley-function-call-leaderboard/README.md"] },
+});
+const raw = load('data/evaluation/known-good.json');
+const diagnostic = parseEvaluationFoundation({ registry, ...raw });
+assert.equal(diagnostic.cases.length, 1);
+assert.equal(diagnostic.run_receipts.length, 1);
+assert.equal(diagnostic.run_receipts[0].evidence_kind, 'synthetic_fixture');
+assert.equal(diagnostic.run_receipts[0].pairing, null);
+assert.equal(diagnostic.run_receipts[0].qualification.official_result, false);
+const emptyResolver = { resolve: () => undefined };
+assert.deepEqual(deriveMatchedPairs(diagnostic, emptyResolver), []);
+for (const item of diagnostic.cases)
+    parseEvalCase(item);
+for (const item of diagnostic.run_receipts)
+    parseEvalRunReceipt(item);
+for (const item of diagnostic.failure_clusters)
+    parseEvalFailureCluster(item);
+for (const item of [...diagnostic.cases, ...diagnostic.run_receipts, ...diagnostic.failure_clusters])
+    assertPublicArtifactSafe(item, 'repository fixture');
+const trek = registry.find(item => item.benchmark_id === 'trek');
+const evalCase = diagnostic.cases[0];
+const observedCase = { ...evalCase, input_ref: { ...evalCase.input_ref, kind: 'external_opaque_reference' } };
+const seed = diagnostic.run_receipts[0];
+const controls = {
+    ...seed.controls,
+    case_set_sha256: evaluationFingerprint([observedCase]),
+    scorer_sha256: evaluationFingerprint(observedCase.scorer_revision),
+    source_fence_sha256: evaluationFingerprint(trek.source_fence),
+    official_evaluator_sha256: evaluationFingerprint(trek.provenance.evaluator),
+};
+const observed = (role) => {
+    const run = {
+        ...seed,
+        run_id: `run:trek:${role}-test-only`, evidence_kind: 'observed_external',
+        gotry_sha: role === 'baseline' ? '1111111111111111111111111111111111111111' : '2222222222222222222222222222222222222222',
+        pairing: { pair_id: 'pair:trek:test-only', role, counterpart_run_id: `run:trek:${role === 'baseline' ? 'treatment' : 'baseline'}-test-only` },
+        model: { provider: 'test-provider', model: 'test-model' }, controls,
+        qualification: {
+            official_result: true, source_fence_passed: true, integrity_passed: true,
+            evidence_receipts: { official_evaluator_output_sha256: null, source_fence_audit_sha256: null, integrity_audit_sha256: null },
+        },
+        experiment: {
+            changed_variables: role === 'baseline' ? [] : ['gotry_sha'],
+            candidate_sha256: evaluationFingerprint({ treatment_variable: 'gotry_sha', gotry_sha: role === 'baseline' ? '1111111111111111111111111111111111111111' : '2222222222222222222222222222222222222222' }),
+        },
+        evidence_summary: { ...seed.evidence_summary, fixture_only: false, statement: 'test-only observed-external aggregate admission object' },
+    };
+    const { evidence_receipts: _receipts, ...qualification } = run.qualification;
+    const bound = { ...run, qualification };
+    const base = { schema_version: 'gotry_eval_evidence_artifact_v0', run_id: run.run_id, benchmark_id: run.benchmark_id, case_id: run.case_id, run_binding_sha256: evaluationFingerprint(bound) };
+    const artifacts = {
+        official_evaluator: { ...base, artifact_kind: 'official_evaluator', evaluator_sha256: run.controls.official_evaluator_sha256, native_metrics_sha256: evaluationFingerprint(run.native_metrics), native_metrics: run.native_metrics, official_result: true },
+        source_fence_audit: { ...base, artifact_kind: 'source_fence_audit', source_fence_sha256: run.controls.source_fence_sha256, input_digest_sha256: observedCase.input_ref.digest_sha256, source_fence_passed: true, forbidden_field_hits: 0 },
+        integrity_audit: { ...base, artifact_kind: 'integrity_audit', integrity_sha256: run.controls.integrity_sha256, candidate_sha256: run.experiment.candidate_sha256, integrity_passed: true },
+    };
+    run.qualification.evidence_receipts = { official_evaluator_output_sha256: evaluationFingerprint(artifacts.official_evaluator), source_fence_audit_sha256: evaluationFingerprint(artifacts.source_fence_audit), integrity_audit_sha256: evaluationFingerprint(artifacts.integrity_audit) };
+    return run;
+};
+const countable = parseEvaluationFoundation({
+    registry, cases: [observedCase], run_receipts: [observed('baseline'), observed('treatment')],
+    failure_clusters: [{ ...diagnostic.failure_clusters[0], run_ids: ['run:trek:baseline-test-only', 'run:trek:treatment-test-only'] }],
+});
+const artifactResolver = { resolve(sha256) {
+        for (const run of countable.run_receipts) {
+            const { evidence_receipts: _receipts, ...qualification } = run.qualification;
+            const bound = { ...run, qualification };
+            const base = { schema_version: 'gotry_eval_evidence_artifact_v0', run_id: run.run_id, benchmark_id: run.benchmark_id, case_id: run.case_id, run_binding_sha256: evaluationFingerprint(bound) };
+            const artifacts = [{ ...base, artifact_kind: 'official_evaluator', evaluator_sha256: run.controls.official_evaluator_sha256, native_metrics_sha256: evaluationFingerprint(run.native_metrics), native_metrics: run.native_metrics, official_result: true }, { ...base, artifact_kind: 'source_fence_audit', source_fence_sha256: run.controls.source_fence_sha256, input_digest_sha256: observedCase.input_ref.digest_sha256, source_fence_passed: true, forbidden_field_hits: 0 }, { ...base, artifact_kind: 'integrity_audit', integrity_sha256: run.controls.integrity_sha256, candidate_sha256: run.experiment.candidate_sha256, integrity_passed: true }];
+            const found = artifacts.find(item => evaluationFingerprint(item) === sha256);
+            if (found)
+                return found;
+        }
+        return undefined;
+    } };
+const fingerprintMismatchResolver = { resolve(sha256) { const artifact = artifactResolver.resolve(sha256); if (!artifact)
+        return undefined; const changed = structuredClone(artifact); if (changed.artifact_kind === 'official_evaluator')
+        changed.official_result = false; return changed; } };
+assert.throws(() => deriveMatchedPairs(countable, fingerprintMismatchResolver), /fingerprint mismatch/);
+const pairs = deriveMatchedPairs(countable, artifactResolver);
+assert.deepEqual(pairs, [{
+        schema_version: 'gotry_eval_matched_pair_derived_v0', pair_id: 'pair:trek:test-only', benchmark_id: 'trek',
+        case_id: 'gotry:foundation:case-001', baseline_run_id: 'run:trek:baseline-test-only',
+        treatment_run_id: 'run:trek:treatment-test-only', treatment_variable: 'gotry_sha', matched_pair_countable: true,
+    }]);
+// Each adversarial artifact is re-signed and its receipt updated, so deriveMatchedPairs
+// reaches the semantic closure under test rather than stopping at the fingerprint gate.
+const exerciseArtifact = (runIndex, kind, mutate, expected) => {
+    const foundation = structuredClone(countable);
+    const run = foundation.run_receipts[runIndex];
+    const receiptKey = kind === 'official_evaluator' ? 'official_evaluator_output_sha256' : kind === 'source_fence_audit' ? 'source_fence_audit_sha256' : 'integrity_audit_sha256';
+    const original = artifactResolver.resolve(run.qualification.evidence_receipts[receiptKey]);
+    const artifact = structuredClone(original);
+    mutate(artifact);
+    const digest = evaluationFingerprint(artifact);
+    run.qualification.evidence_receipts[receiptKey] = digest;
+    const resolver = { resolve(sha256) { return sha256 === digest ? artifact : artifactResolver.resolve(sha256); } };
+    assert.throws(() => deriveMatchedPairs(foundation, resolver), expected);
+};
+assert.throws(() => deriveMatchedPairs(countable, { resolve: () => undefined }), /artifact/);
+exerciseArtifact(0, 'official_evaluator', a => { a.evaluator_sha256 = 'f'.repeat(64); }, /evaluator artifact mismatch/);
+exerciseArtifact(0, 'official_evaluator', a => { a.native_metrics = { altered: 1 }; a.native_metrics_sha256 = evaluationFingerprint(a.native_metrics); }, /evaluator artifact mismatch/);
+exerciseArtifact(0, 'source_fence_audit', a => { a.input_digest_sha256 = 'f'.repeat(64); }, /source-fence artifact mismatch/);
+exerciseArtifact(0, 'source_fence_audit', a => { a.source_fence_passed = false; }, /source-fence artifact mismatch/);
+exerciseArtifact(0, 'source_fence_audit', a => { a.forbidden_field_hits = 1; }, /source-fence artifact mismatch/);
+exerciseArtifact(0, 'integrity_audit', a => { a.candidate_sha256 = 'f'.repeat(64); }, /integrity artifact mismatch/);
+exerciseArtifact(0, 'integrity_audit', a => { a.integrity_passed = false; }, /integrity artifact mismatch/);
+const treatmentReuse = structuredClone(countable);
+const baselineReceipt = treatmentReuse.run_receipts[0].qualification.evidence_receipts.official_evaluator_output_sha256;
+treatmentReuse.run_receipts[1].qualification.evidence_receipts.official_evaluator_output_sha256 = baselineReceipt;
+assert.throws(() => deriveMatchedPairs(treatmentReuse, artifactResolver), /fingerprint mismatch|binding mismatch/);
+// D-28 admission gate solidification (issue #203): the registry file's countability
+// face is a frozen fact — chinatravel/travelbench/locomo are diagnostic_only. The
+// `pair-diagnostic-default` vector proves the mechanism on registry[0]; this block
+// pins the published face itself and proves an otherwise fully-qualified ChinaTravel
+// paired construction (reciprocal observed_external roles, identical model identity,
+// all eight controls, evaluator/source-fence/integrity artifacts resolved and closed)
+// is rejected as diagnostic-only, and becomes derivable only when that frozen face is
+// flipped in memory. Per docs/evaluation/evaluation-foundation.md these in-memory
+// observed_external objects are countable_test_only falsification inputs: never
+// written to git, never a baseline, production result, official score, or
+// Agent-quality evidence.
+const countability = Object.fromEntries(registry.map(item => [item.benchmark_id, item.countability_default]));
+assert.deepEqual(countability, {
+    trek: 'countable_if_qualified',
+    travelplanner: 'countable_if_qualified',
+    chinatravel: 'diagnostic_only',
+    travelbench: 'diagnostic_only',
+    tau2: 'countable_if_qualified',
+    locomo: 'diagnostic_only',
+    bfcl: 'countable_if_qualified',
+});
+const chinatravel = registry.find(item => item.benchmark_id === 'chinatravel');
+const ctCase = {
+    ...structuredClone(observedCase),
+    benchmark_id: 'chinatravel',
+    case_id: 'gotry:foundation:chinatravel-admission-probe',
+};
+const ctMetrics = Object.fromEntries(chinatravel.native_metrics.values.map(metric => [metric.receipt_key, 0.5]));
+const ctControls = {
+    ...seed.controls,
+    case_set_sha256: evaluationFingerprint([ctCase]),
+    scorer_sha256: evaluationFingerprint(ctCase.scorer_revision),
+    source_fence_sha256: evaluationFingerprint(chinatravel.source_fence),
+    official_evaluator_sha256: evaluationFingerprint(chinatravel.provenance.evaluator),
+};
+const ctRun = (role) => {
+    const sha = role === 'baseline' ? '3333333333333333333333333333333333333333' : '4444444444444444444444444444444444444444';
+    const run = {
+        ...seed,
+        run_id: `run:chinatravel:${role}-probe`, benchmark_id: 'chinatravel', case_id: ctCase.case_id,
+        evidence_kind: 'observed_external',
+        gotry_sha: sha,
+        pairing: { pair_id: 'pair:chinatravel:admission-gate-probe', role, counterpart_run_id: `run:chinatravel:${role === 'baseline' ? 'treatment' : 'baseline'}-probe` },
+        model: { provider: 'test-provider', model: 'test-model' }, controls: ctControls,
+        qualification: {
+            official_result: true, source_fence_passed: true, integrity_passed: true,
+            evidence_receipts: { official_evaluator_output_sha256: null, source_fence_audit_sha256: null, integrity_audit_sha256: null },
+        },
+        experiment: {
+            changed_variables: role === 'baseline' ? [] : ['gotry_sha'],
+            candidate_sha256: evaluationFingerprint({ treatment_variable: 'gotry_sha', gotry_sha: sha }),
+        },
+        native_metrics: ctMetrics,
+        evidence_summary: { ...seed.evidence_summary, fixture_only: false, statement: 'test-only diagnostic-gate falsification pair' },
+    };
+    const { evidence_receipts: _receipts, ...qualification } = run.qualification;
+    const bound = { ...run, qualification };
+    const base = { schema_version: 'gotry_eval_evidence_artifact_v0', run_id: run.run_id, benchmark_id: run.benchmark_id, case_id: run.case_id, run_binding_sha256: evaluationFingerprint(bound) };
+    const artifacts = {
+        official_evaluator: { ...base, artifact_kind: 'official_evaluator', evaluator_sha256: run.controls.official_evaluator_sha256, native_metrics_sha256: evaluationFingerprint(run.native_metrics), native_metrics: run.native_metrics, official_result: true },
+        source_fence_audit: { ...base, artifact_kind: 'source_fence_audit', source_fence_sha256: run.controls.source_fence_sha256, input_digest_sha256: ctCase.input_ref.digest_sha256, source_fence_passed: true, forbidden_field_hits: 0 },
+        integrity_audit: { ...base, artifact_kind: 'integrity_audit', integrity_sha256: run.controls.integrity_sha256, candidate_sha256: run.experiment.candidate_sha256, integrity_passed: true },
+    };
+    run.qualification.evidence_receipts = { official_evaluator_output_sha256: evaluationFingerprint(artifacts.official_evaluator), source_fence_audit_sha256: evaluationFingerprint(artifacts.source_fence_audit), integrity_audit_sha256: evaluationFingerprint(artifacts.integrity_audit) };
+    return run;
+};
+const ctFoundation = parseEvaluationFoundation({
+    registry, cases: [ctCase], run_receipts: [ctRun('baseline'), ctRun('treatment')], failure_clusters: [],
+});
+const ctResolver = { resolve(sha256) {
+        for (const run of ctFoundation.run_receipts) {
+            const { evidence_receipts: _receipts, ...qualification } = run.qualification;
+            const bound = { ...run, qualification };
+            const base = { schema_version: 'gotry_eval_evidence_artifact_v0', run_id: run.run_id, benchmark_id: run.benchmark_id, case_id: run.case_id, run_binding_sha256: evaluationFingerprint(bound) };
+            const artifacts = [{ ...base, artifact_kind: 'official_evaluator', evaluator_sha256: run.controls.official_evaluator_sha256, native_metrics_sha256: evaluationFingerprint(run.native_metrics), native_metrics: run.native_metrics, official_result: true }, { ...base, artifact_kind: 'source_fence_audit', source_fence_sha256: run.controls.source_fence_sha256, input_digest_sha256: ctCase.input_ref.digest_sha256, source_fence_passed: true, forbidden_field_hits: 0 }, { ...base, artifact_kind: 'integrity_audit', integrity_sha256: run.controls.integrity_sha256, candidate_sha256: run.experiment.candidate_sha256, integrity_passed: true }];
+            const found = artifacts.find(item => evaluationFingerprint(item) === sha256);
+            if (found)
+                return found;
+        }
+        return undefined;
+    } };
+assert.throws(() => deriveMatchedPairs(ctFoundation, ctResolver), /diagnostic only/);
+const flippedRegistry = structuredClone(registry);
+flippedRegistry[2].countability_default = 'countable_if_qualified';
+const flippedFoundation = parseEvaluationFoundation({
+    registry: flippedRegistry, cases: [ctCase], run_receipts: [ctRun('baseline'), ctRun('treatment')], failure_clusters: [],
+});
+assert.deepEqual(deriveMatchedPairs(flippedFoundation, ctResolver), [{
+        schema_version: 'gotry_eval_matched_pair_derived_v0', pair_id: 'pair:chinatravel:admission-gate-probe', benchmark_id: 'chinatravel',
+        case_id: 'gotry:foundation:chinatravel-admission-probe', baseline_run_id: 'run:chinatravel:baseline-probe',
+        treatment_run_id: 'run:chinatravel:treatment-probe', treatment_variable: 'gotry_sha', matched_pair_countable: true,
+    }]);
+const canonical1 = stableEvaluationJson(diagnostic);
+const canonical2 = stableEvaluationJson(parseEvaluationFoundation(JSON.parse(canonical1)));
+const digest1 = createHash('sha256').update(canonical1).digest('hex');
+const digest2 = createHash('sha256').update(canonical2).digest('hex');
+assert.equal(canonical1, canonical2);
+assert.equal(digest1, digest2);
+assert.match(digest1, /^[0-9a-f]{64}$/);
+const vectors = parseMutationVectors(load('data/evaluation/known-bad.json'));
+assert.equal(vectors.length, 39);
+for (const vector of vectors) {
+    const base = vector.foundation_kind === 'countable_test_only' ? countable : diagnostic;
+    const mutated = applyMutationVector(base, vector);
+    assert.throws(() => vector.target === 'foundation' ? deriveMatchedPairs(parseEvaluationFoundation(mutated), vector.foundation_kind === 'countable_test_only' ? artifactResolver : emptyResolver)
+        : vector.target === 'registry' ? parseBenchmarkRegistry(mutated)
+            : vector.target === 'case' ? parseEvalCase(mutated)
+                : vector.target === 'run' ? parseEvalRunReceipt(mutated) : parseEvalFailureCluster(mutated), new RegExp(vector.expected_error), vector.id);
+}
+const runAll = readFileSync('../scripts/run-all-tests.sh', 'utf8');
+assert.match(runAll, /=== 46\. Evaluation Phase 0 foundation/);
+assert.match(runAll, /npx tsx scripts\/evaluation-contract-tests\.ts/);
+assert.throws(() => deriveMatchedPairs(countable, emptyResolver), /artifact/);
+for (const [key, value] of [['apikey', 'secret'], ['access-token', 'secret'], ['apiKey', 'secret'], ['ACCESS-TOKEN', 'secret'], ['client_secret', 'secret'], ['authorization', 'Bearer abcdefghijklmnopqrstuvwxyz123456'], ['value', '/private/file'], ['value', '~/private/file'], ['value', 'C:\\private\\file'], ['value', 'file:///private/file'], ['value', 'sk-abcdefghijklmnopqrstuv'], ['value', 'ghp_abcdefghijklmnopqrstuvwxyz123456'], ['value', 'AKIA1234567890ABCDEF']])
+    assert.throws(() => assertPublicArtifactSafe({ [key]: value }, 'adversarial'), /absolute path or secret|credentials/);
+assert.doesNotThrow(() => assertPublicArtifactSafe({ url: 'https://github.com/org/repo/blob/main/README.md', label: 'question-answering token F1' }, 'benign'));
+assert.throws(() => assertPublicArtifactSafe({ value: 'https://example.test/?token=sk-abcdefghijklmnopqrstuv' }, 'https-url-secret'), /absolute path or secret/);
+for (const value of ['log=/Users/a/private.json', 'C:\\Users\\a\\secret', 'file:///tmp/x', '~/x', 'https://github.com/org/repo log=/Users/a/x', 'Bearer abcdefghijklmnopqrstuvwxyz123456', 'sk-abcdefghijklmnopqrstuv', 'ghp_abcdefghijklmnopqrstuvwxyz123456', 'AKIA1234567890ABCDEF'])
+    assert.throws(() => assertPublicArtifactSafe({ value }, 'path-or-secret'), /absolute path or secret/);
+assert.doesNotThrow(() => assertPublicArtifactSafe({ github: 'https://github.com/org/repo', huggingface: 'https://huggingface.co/datasets/org/name' }, 'public urls'));
+for (const key of ['source_fence_passed', 'integrity_passed']) {
+    const syntheticFlags = JSON.parse(JSON.stringify(load('data/evaluation/known-good.json')));
+    const run = syntheticFlags.run_receipts[0];
+    const qualification = run.qualification;
+    qualification[key] = true;
+    assert.equal(qualification[key === 'source_fence_passed' ? 'integrity_passed' : 'source_fence_passed'], false);
+    assert.throws(() => parseEvalRunReceipt(run), /synthetic fixture/);
+}
+console.log(`canonical sha256: ${digest1}`);
+console.log(`evaluation-contract tests: ${registry.length} registry, 1 test-only matched pair, 39 negative vectors, chinatravel diagnostic-only gate green`);

package/dist/scripts/extension-distribution-cli.js

@@ -0,0 +1,39 @@
+/**
+ * extension-distribution CLI 入口(gotry setup --extension-from=github,ADR-21):
+ *
+ * bin/gotry-bootstrap.js setup 子命令 spawn `npx tsx scripts/extension-distribution-cli.ts`,
+ * 本脚本调 installExtensionFromGithub,把结果以单行 JSON stdout 打印(父进程解析后人话化)。
+ *
+ * 参数:
+ *   --dest <dir>             落位目录(默认 ~/.gotry/extension)
+ *   --source-dir <dir>       bundled 钉扎参照(默认仓库 extension/,由 bootstrap 传入)
+ *   --release-base <url>     Releases 基址(默认 GitHub 官方;测试/镜像可覆盖)
+ *   --check-only             只报告远端版本,不落盘
+ *
+ * 退出码:0 = installed/up-to-date;2 = fallback-bundled(JSON 字段区分原因,父进程据此降级);
+ * 非 0 且无 JSON = 脚本自身异常。
+ */
+import process from 'node:process';
+import { homedir } from 'node:os';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { DEFAULT_RELEASE_BASE, installExtensionFromGithub } from '../capabilities/session/extension-distribution.js';
+function argValue(argv, name) {
+    const i = argv.indexOf(name);
+    return i >= 0 ? argv[i + 1] : undefined;
+}
+const argv = process.argv.slice(2);
+const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
+const dest = argValue(argv, '--dest') ?? join(homedir(), '.gotry', 'extension');
+const sourceDir = argValue(argv, '--source-dir') ?? join(repoRoot, 'extension');
+const releaseBase = argValue(argv, '--release-base') ?? DEFAULT_RELEASE_BASE;
+const checkOnly = argv.includes('--check-only');
+try {
+    const r = await installExtensionFromGithub({ destDir: dest, pinnedSourceDir: sourceDir, releaseBase, checkOnly });
+    process.stdout.write(`${JSON.stringify(r)}\n`);
+    process.exit(r.ok ? 0 : 2);
+}
+catch (e) {
+    process.stdout.write(`${JSON.stringify({ ok: false, action: 'fallback-bundled', error: `CLI 异常 ${e.message}` })}\n`);
+    process.exit(2);
+}