harness-everything 0.47.0 → 0.47.2

package/benchmarks/issue290-pilot/host-hook.cjs

@@ -0,0 +1,35 @@
+'use strict';
+
+// Experiment-only host adapter. Calls the unchanged canonical router through
+// UserPromptSubmit; all other event handling is read-only observation.
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+let raw = '';
+process.stdin.setEncoding('utf8');
+process.stdin.on('data', chunk => { raw += chunk; });
+process.stdin.on('end', () => {
+  const payload = JSON.parse(raw);
+  const config = JSON.parse(fs.readFileSync(path.join(process.cwd(), '.pilot.json'), 'utf8'));
+  const event = payload.hook_event_name || process.argv[2];
+  const record = {
+    event, session: payload.session_id, tool: payload.tool_name,
+    input: payload.tool_input, time: new Date().toISOString(),
+  };
+  fs.appendFileSync(config.trace, JSON.stringify(record) + '\n');
+  if (event !== 'UserPromptSubmit') return;
+  const result = spawnSync(process.execPath, [path.join(config.source, 'harness-everything/scripts/kernel-router.js')], {
+    input: raw, encoding: 'utf8', cwd: process.cwd(), env: process.env,
+  });
+  fs.appendFileSync(config.routerTrace, result.stdout || '');
+  if (result.stderr) fs.appendFileSync(config.routerTrace, result.stderr);
+  let context = result.stdout || '';
+  context += '\nEXPERIMENT REFERENCE TIMING: All files in .harness are identical in both arms. Preserve selected workflow, checks, permissions, bindings, status and labels.\n';
+  if (config.arm === 'A') {
+    context += 'For this controlled timing ablation only, read every listed reference now, including later-step material. Apply a contract only when its step is applicable. This experiment instruction overrides only the active-step read-timing restriction.\n';
+    context += config.documents.map(p => path.join(process.cwd(), '.harness', p)).join('\n');
+  } else {
+    context += 'Read only the references needed by the current active step; defer later-step documents until applicable. Resolve the required bindings before passing their steps.\n';
+  }
+  console.log(JSON.stringify({hookSpecificOutput: {hookEventName: 'UserPromptSubmit', additionalContext: context}}));
+});

package/benchmarks/issue290-pilot/metrics.test.py

@@ -0,0 +1,131 @@
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import analyze
+from read_metrics import collect_reads, observed_read
+from reporting import paired_rows, summarize_groups
+
+
+class ReadMetricTest(unittest.TestCase):
+    def setUp(self):
+        self.path = 'find-skills/SKILL.md'
+        self.content = '# A sufficiently distinctive reference title\nA second sufficiently distinctive line of reference text.\n'
+        self.documents = {self.path: self.content}
+
+    def test_compound_both_orders_separate_mixed_output(self):
+        for command in ('cat .harness/find-skills/SKILL.md && ls', 'ls && cat .harness/find-skills/SKILL.md'):
+            with self.subTest(command=command):
+                row = observed_read(command, self.content+'inventory\n', self.documents)
+                self.assertEqual(row['returned_bytes'], len(self.content.encode()))
+                self.assertEqual(row['uncertain_or_mixed_output_bytes'], len('inventory\n'))
+                self.assertEqual(row['filesystem_access_candidates'], 1)
+
+    def test_transformed_output_is_not_content(self):
+        row = observed_read('cat .harness/find-skills/SKILL.md | wc -l', '2\n', self.documents)
+        self.assertEqual(row['returned_bytes'], 0)
+        self.assertEqual(row['uncertain_or_mixed_output_bytes'], 2)
+
+    def test_partial_content_counts_only_matched_lines(self):
+        partial = self.content.splitlines(keepends=True)[0]
+        row = observed_read("sed -n '1p' .harness/find-skills/SKILL.md", partial, self.documents)
+        self.assertEqual(row['returned_bytes'], len(partial.encode()))
+        self.assertEqual(row['matched_content'][0]['kind'], 'partial-content')
+
+    def test_powershell_paths_and_alias(self):
+        for command in (r'Get-Content .harness\find-skills\SKILL.md', r'gc -Raw .harness\find-skills\SKILL.md'):
+            self.assertEqual(observed_read(command, self.content, self.documents)['returned_bytes'], len(self.content.encode()))
+
+    def test_frozen_source_placeholder_and_wrapped_shell(self):
+        for command in ('cat <HARNESS_SOURCE>/find-skills/SKILL.md', '/bin/bash -lc "cat .harness/find-skills/SKILL.md && ls"'):
+            self.assertEqual(observed_read(command, self.content, self.documents)['returned_bytes'], len(self.content.encode()))
+
+    def test_bundle_excludes_arbitrary_repository_files_and_mentions(self):
+        for command in ('cat <HARNESS_SOURCE>/grader.cjs', 'cat .harness/unknown/SKILL.md', 'echo .harness/find-skills/SKILL.md', 'rg --files .harness/find-skills/SKILL.md'):
+            self.assertIsNone(observed_read(command, self.content, self.documents))
+
+    def test_native_read_cannot_identify_unknown_content_as_bytes(self):
+        row = observed_read('', 'unmatched output', self.documents, native_path='.harness/'+self.path)
+        self.assertEqual(row['returned_bytes'], 0)
+        self.assertEqual(row['uncertain_or_mixed_output_bytes'], 16)
+
+
+class AccountingTest(unittest.TestCase):
+    def episode(self, root, engine='claude', workload='profile-lookup', events=None):
+        row = {'model': {'engine': engine, 'model': 'fixture'}, 'workload': workload,
+               'grader': {'pass': False}, 'usage': None, 'router_attributed': True}
+        (root/'analysis.json').write_text(json.dumps(row))
+        (root/'manifest.json').write_text('{}')
+        (root/'transcript.jsonl').write_text('\n'.join(json.dumps(e) for e in (events or [])))
+
+    def test_offline_never_reads_host_home_or_invents_cost(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            self.episode(root)
+            with patch.object(Path, 'home', side_effect=AssertionError('host home accessed')):
+                row = analyze.inspect_episode(root, offline=True, documents={})
+            self.assertIsNone(row['api_equivalent_cost_usd'])
+            self.assertIn('unknown', row['child_usage_observability'])
+
+    def test_missing_claude_child_logs_make_whole_cost_unknown(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            self.episode(root, events=[{'type':'assistant','message':{'content':[{'type':'tool_use','name':'Task','id':'worker'}]}},
+                                      {'tool_use_result':{'agentId':'missing'}}])
+            with patch.object(Path, 'home', return_value=root):
+                row = analyze.inspect_episode(root, documents={})
+            self.assertIsNone(row['api_equivalent_cost_usd'])
+            self.assertIn('unknown', row['child_usage_observability'])
+
+    def test_polyglot_grader_timeout_is_retained_failure(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            self.episode(root, workload='polyglot-simple-linked-list-staged')
+            with patch('analyze.subprocess.run', side_effect=subprocess.TimeoutExpired('node', 20)):
+                row = analyze.inspect_episode(root, offline=True, documents={})
+            self.assertEqual(row['independent_grader'], {'pass':False, 'timeout':True})
+
+    def test_filesystem_enumeration_order_does_not_change_binding_attribution(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            content = 'A sufficiently distinctive reference content line.\n'
+            self.episode(root, events=[{'type':'item.completed', 'item':{
+                'type':'command_execution', 'command':'cat .harness/find-skills/SKILL.md',
+                'aggregated_output':content, 'exit_code':0}}])
+            state = root/'state'
+            state.mkdir()
+            for name in ('z', 'a'):
+                (state/(name+'.json')).write_text(json.dumps({'schemaVersion':2, 'stageId':name,
+                    'runId':'fixture', 'requiredBindings':[{'id':'read', 'path':'find-skills/SKILL.md'}]}))
+            documents = {'find-skills/SKILL.md':content}
+            first = analyze.inspect_episode(root, offline=True, documents=documents)
+            original = Path.rglob
+            def reverse_listing(path, pattern):
+                return iter(reversed(list(original(path, pattern))))
+            with patch.object(Path, 'rglob', reverse_listing):
+                second = analyze.inspect_episode(root, offline=True, documents=documents)
+            self.assertEqual(first['observed_reference_reads'], second['observed_reference_reads'])
+
+    def test_null_costs_and_metric_specific_denominators(self):
+        base = {'series':'test', 'model':{'engine':'codex'}, 'workload':'test', 'arm':'A', 'repeat':1,
+                'preflight':False, 'classification':'definitive', 'independent_grader':{'pass':True},
+                'budget_failure':False, 'reference_returned_bytes':3, 'exact_reference_bytes_lower_bound':3,
+                'wall_seconds':1, 'api_equivalent_cost_usd':1, 'api_equivalent_input_cost_usd':1}
+        failure = {**base, 'arm':'B', 'classification':'definitive-agent-budget-failure', 'budget_failure':True,
+                   'api_equivalent_cost_usd':None, 'api_equivalent_input_cost_usd':None}
+        infrastructure = {**base, 'classification':'inconclusive', 'repeat':2}
+        groups = summarize_groups([base, failure, infrastructure])
+        self.assertEqual(groups[0]['n_quality_included'], 1)
+        self.assertEqual(groups[0]['n_quality_excluded'], 1)
+        self.assertEqual(groups[1]['artifact_passes'], 0)
+        self.assertEqual(groups[1]['budget_failures'], 1)
+        self.assertIsNone(groups[1]['api_equivalent_cost_usd'])
+        self.assertFalse(paired_rows([base, failure])[0]['api_equivalent_cost_usd']['included'])
+
+
+if __name__ == '__main__':
+    unittest.main()

package/benchmarks/issue290-pilot/preflight/amendment.md

@@ -0,0 +1,32 @@
+# Preflight amendment — 2026-10-14
+
+Before formal A/B calls, the initial two host probes established real
+UserPromptSubmit/router attribution and model access. The Haiku probe exhausted
+the runner's three-turn smoke cap after opening the correct lookup references,
+before producing its final answer. This was a preflight configuration failure,
+not a product failure or a formal A/B observation. Both original probes are
+retained; the corrected preflight repeats both models with the same 35-turn cap
+as the formal runner. No formal trial outcome has been observed yet.
+
+The frozen fixture adaptation runs all 23 Polyglot assertions using the
+explicit matcher adapter in grader.cjs, avoiding unrelated npm setup work. An
+independently authored reference passes all 23; the untouched skeleton fails.
+QuixBugs uses all seven pinned upstream JSON cases with Python's built-in
+assertions, avoiding a pytest installation; its upstream reference passes and
+the original fails with RecursionError. This is an adapted subset, not an
+official leaderboard score. Gold implementations are outside agent fixtures.
+
+The staged prompt adds three ordered, dependent implementation stages to the
+public linked-list instruction. Profile lookup is a synthetic Harness contract
+probe. Only the small repair directly uses an unmodified public repair task.
+The experiment tests eager *required reads*, not merely file availability;
+eager treatment may not reproduce organic behavior of any historical release.
+The exact per-family document bundles and prompts are fixed in run.py.
+
+Codex uses reviewed explicit native hooks with its documented automation hook
+trust option; Claude uses project-scoped settings. Neither comparison claims
+full-plugin parity. Normal native UserPromptSubmit invokes the unchanged
+canonical router in both hosts. Subscription auth is reused without reading or
+copying credential material. Workflow state is isolated per fixture.
+
+Lookup completion checks both the six exact selection fields and JSON-only output after removing the mandatory hidden turn label (an enclosing JSON fence is accepted). Prose outside JSON fails. This rubric is frozen before formal trials; preflight showed why field extraction alone was insufficient.

package/benchmarks/issue290-pilot/preflight/environment.json

@@ -0,0 +1,15 @@
+{
+  "os": "macOS-26.6.2-arm64-arm-64bit-Mach-O",
+  "python": "3.14.4",
+  "architecture": "arm64",
+  "macos": "26.6.2",
+  "cpu": "Apple M4",
+  "ram_bytes": "17179869184",
+  "logical_cpus": "10",
+  "codex": "codex-cli 0.162.1",
+  "claude": "2.1.299 (Claude Code)",
+  "inspection_shell_node": "v25.9.0",
+  "agent_node_versions_observed": [
+    "v24.14.1"
+  ]
+}

package/benchmarks/issue290-pilot/preflight/fable-amendment.md

@@ -0,0 +1,20 @@
+# Fable family correction — preregistered 2026-10-14 before staged trials
+
+The deterministic current router selects iterative-single for the original
+three-stage single-module linked-list prompt. This is a bounded staged task,
+not a demonstrated Fable execution, and is not classified as a router bug.
+All eight planned bounded-stage rows remain retained and reported separately.
+Their outcomes have not been observed when this correction is registered.
+
+The genuine Fable supplement prefixes the identical original prompt with
+`Use Fable staged execution.` Current tier-router selects tier3/fable-staged for
+that prompt. This deliberately requests the topology whose obligations #290
+must preserve; it is not a neutral whole-Harness effectiveness benchmark.
+
+Eight additional episodes: same two models, two repeats, AB then BA. The
+supplement creates a linked Git worktree before each agent starts, satisfying
+the existing Fable isolation obligation symmetrically. Grading targets that
+linked worktree. The original aggregate 7,200 agent wall-second / 500,000
+output-token limits are shared across the original and supplemental series;
+no higher budget or outcome-selected rerun is authorized by this amendment.
+Other A/B intervention, documents, API contract, tests and rubric are unchanged.

package/benchmarks/issue290-pilot/preflight/fable-frozen-files.json

@@ -0,0 +1,4 @@
+{
+  "fable-supplement.py": "bdba9df11ad45c03abdee895f017ed01474a3f7caa727982e8ac02f2a395f73b",
+  "preflight/fable-amendment.md": "d1a752cd5e4385c888eab89df27233ab7c86862f420a12c94bc1023cc9b5f978"
+}

package/benchmarks/issue290-pilot/preflight/fable-infrastructure-amendment.md

@@ -0,0 +1,27 @@
+# Linked-fixture infrastructure diagnosis — 2026-10-14
+
+The first four Claude Fable launches exited before model execution with
+`Settings file not found: .../hook-settings.json`. None emitted UserPromptSubmit,
+model usage, or router attribution. They are infrastructure/inconclusive rows,
+not four model failures. All are retained without selective removal.
+
+Read-only root-cause reconstruction: run.py prepares and commits fixture files,
+then writes hook-settings.json. A new linked Git worktree receives only the
+committed snapshot. Codex passes its hooks inline, but Claude requires the
+missing settings file. The original linked-fixture preflight checked Git linkage
+and source presence without asserting this host-specific startup prerequisite.
+The unattended supplement also failed to stop on absent router attribution.
+
+After these repeated same-signature setup failures, stop edits, reconstruct the
+full setup path, and record this diagnosis before recovery. Add a failing
+fixture prerequisite test, copy the identical settings into the linked checkout,
+assert all required launch files exist, and stop immediately on absent initial
+router attribution. The original script/hash is preserved as v1.
+
+One symmetric infrastructure retry for the two Claude pairs is registered
+before any retry model call: AB then BA, identical prompt/model/effort/documents,
+tests, budgets and canonical Harness revision. The four original startup errors
+remain outside the behavioral denominator. No Luna outcome is rerun, and no
+valid agent failure is excluded. The same original 7,200 agent wall-second and
+500,000 output-token budgets include original + retry attempts. No higher limit
+or additional outcome-selected repeat is introduced.

package/benchmarks/issue290-pilot/preflight/fable-retry-frozen-files.json

@@ -0,0 +1,4 @@
+{
+  "fable-supplement.py": "f8a77fd584eb555eff56681307be95ea5df53d4ad6bc0f3a7eabd0844b033982",
+  "preflight/fable-infrastructure-amendment.md": "cea92a4a6277b4992dc0299b12bc2b3005fcc580e26ce9a13783832806a5c999"
+}

package/benchmarks/issue290-pilot/preflight/fable-supplement-v1.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Preregistered Fable-only supplement; original bounded-stage rows stay retained."""
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import time
+
+HERE=Path(__file__).resolve().parent
+spec=importlib.util.spec_from_file_location('pilot',HERE/'run.py')
+pilot=importlib.util.module_from_spec(spec)
+spec.loader.exec_module(pilot)
+original_prepare=pilot.prepare
+WORKLOAD='polyglot-simple-linked-list-staged'
+PROMPT="Use Fable staged execution. "+pilot.PROMPTS[WORKLOAD]
+
+def linked_prepare(*args, **kwargs):
+    episode,primary,hooks=original_prepare(*args,**kwargs)
+    linked=Path(tempfile.mkdtemp(prefix='harness290-fable-'))
+    subprocess.run(['git','worktree','add','-q','-b','pilot-linked',str(linked)],cwd=primary,check=True)
+    return episode,linked,hooks
+
+if __name__=='__main__':
+    import argparse
+    parser=argparse.ArgumentParser();parser.add_argument('--output',required=True)
+    args=parser.parse_args();base=Path(args.output).resolve();base.mkdir(parents=True,exist_ok=True)
+    pilot.prepare=linked_prepare
+    pilot.PROMPTS[WORKLOAD]=PROMPT
+    start=time.monotonic()
+    for model in pilot.PROTOCOL['models']:
+        for repeat in [1,2]:
+            for arm in (['A','B'] if repeat==1 else ['B','A']):
+                existing_seconds=0;existing_output=0
+                for root in [Path('/tmp/issue290-formal-20261014'),base]:
+                    for file in root.glob('*/result.json'):
+                        existing_seconds+=json.loads(file.read_text())['wall_seconds']
+                    for file in root.glob('*/transcript.jsonl'):
+                        for line in file.read_text().splitlines():
+                            try:e=json.loads(line)
+                            except json.JSONDecodeError:continue
+                            if e.get('type') in ['result','turn.completed']:
+                                existing_output+=e.get('usage',{}).get('output_tokens',0)
+                if existing_seconds>pilot.PROTOCOL['aggregate_wall_budget_seconds'] or existing_output>pilot.PROTOCOL['aggregate_output_token_cap']:
+                    raise SystemExit('shared original budget reached; remaining Fable rows unattempted')
+                pilot.run_episode(base,model,WORKLOAD,arm,repeat)
+    pilot.summarize(base)

package/benchmarks/issue290-pilot/preflight/frozen-files.json

@@ -0,0 +1,18 @@
+{
+  "sources.json": "453260eac66351761d91603c4d6653470d81eb3002097dc672bb5919837b6bb2",
+  "run.py": "5b7b80ab9ed9177cff3fc5b8bd707e52d9ad3aac39dcd8f47726b8c2ae727389",
+  "grader.cjs": "e14dfba2b13d4027279c8ac3e8735f08dba9657791941eb432c1d4d8a2c773fe",
+  "protocol.json": "ad0c893f71a4c6ff73ae6cc723e3bf7bfd91e678b10cac43f9a538b6c4528b32",
+  "host-hook.cjs": "30d1775088aebc10bbe09eb60f01d5af0b9cf2eb574a81cf54ca8d1604ad3021",
+  "hook.test.cjs": "36e5e30b7b139177a9732af5bb3f20a25d0836281695b550c4da7d3e52a202a8",
+  "preflight/graders.json": "33e9c64af560d1a04db7f8e44d53b00fec7fe463f04cbf19d09c9f262fed4be0",
+  "preflight/amendment.md": "5340c74ff3a462ff7885ee23d5253d29c32629fe4570a6081e1faf6d91e1413c",
+  "fixtures/polyglot/simple-linked-list.js": "f305a2ba4fea7f8dd036989bd635f2466bd36c75d25bec049145a683dd22b8f1",
+  "fixtures/polyglot/simple-linked-list.spec.js": "7a181633077ca3586a47961a00858f1071b90d12cca9f114284707fb82d30255",
+  "fixtures/polyglot/package.json": "14a0a23e784a338496939c2f1a8719661c1b2eb36c17f4c7d6d146f2fa33519e",
+  "fixtures/polyglot/instructions.md": "ddf0b0d7a6d0a7b39a70a8dac0ae2e5cc4b7d9ad4163f4a103988bb53abeb914",
+  "fixtures/quixbugs/LICENSE": "3bb03003929be645f6557a66fcb9820f527df7ce2b51b491405d233cb191d517",
+  "fixtures/quixbugs/test_find_in_sorted.py": "a943c5c1df1e87360308b23341a651da9cfb31b1f1e2954cdca75c2cf4b6ba54",
+  "fixtures/quixbugs/cases.json": "21e0348affa246a85a6907f1d0f8b0b76454bf3c2c133ed1bff808a97ee82dfe",
+  "fixtures/quixbugs/find_in_sorted.py": "b640b4bcb1b2c0f7875de8f13df191f4e61ddf7e6fbdcfc42d477ec60885d3c7"
+}

package/benchmarks/issue290-pilot/preflight/graders.json

@@ -0,0 +1,67 @@
+[
+  {
+    "name": "polyglot-untouched",
+    "result": {
+      "argv": [
+        "node",
+        "<HARNESS_SOURCE>/benchmarks/issue290-pilot/grader.cjs",
+        "<TMP>/issue290-grader-ae68mbog/simple-linked-list.js"
+      ],
+      "exit": 1,
+      "stdout": "{\"pass\":false,\"tests\":23,\"results\":[{\"name\":\"has constructor\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"value reflects constructor arg\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"has null for next by default\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"has constructor\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"new lists should have length 0\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"can add a element\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"adding a element increments length\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"adding two elements increments twice\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"new Lists have a null head element\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"adding an Element to an empty list sets the head Element\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"adding a second Element updates the head Element\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"can get the next Element from the head\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"can be initialized with an array\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"with correct length\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"with correct head value\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"can traverse the list\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"can convert to an array\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"head of list is final element from input array\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"can convert longer list to an array\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"can be reversed\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"can be reversed when it has more elements\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"can reverse with many elements\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"},{\"name\":\"can reverse a reversal\",\"pass\":false,\"error\":\"Remove this statement and implement this function\"}]}\n",
+      "stderr": ""
+    }
+  },
+  {
+    "name": "polyglot-independent-reference",
+    "result": {
+      "argv": [
+        "node",
+        "<HARNESS_SOURCE>/benchmarks/issue290-pilot/grader.cjs",
+        "<TMP>/issue290-grader-ae68mbog/simple-linked-list.js"
+      ],
+      "exit": 0,
+      "stdout": "{\"pass\":true,\"tests\":23,\"results\":[{\"name\":\"has constructor\",\"pass\":true},{\"name\":\"value reflects constructor arg\",\"pass\":true},{\"name\":\"has null for next by default\",\"pass\":true},{\"name\":\"has constructor\",\"pass\":true},{\"name\":\"new lists should have length 0\",\"pass\":true},{\"name\":\"can add a element\",\"pass\":true},{\"name\":\"adding a element increments length\",\"pass\":true},{\"name\":\"adding two elements increments twice\",\"pass\":true},{\"name\":\"new Lists have a null head element\",\"pass\":true},{\"name\":\"adding an Element to an empty list sets the head Element\",\"pass\":true},{\"name\":\"adding a second Element updates the head Element\",\"pass\":true},{\"name\":\"can get the next Element from the head\",\"pass\":true},{\"name\":\"can be initialized with an array\",\"pass\":true},{\"name\":\"with correct length\",\"pass\":true},{\"name\":\"with correct head value\",\"pass\":true},{\"name\":\"can traverse the list\",\"pass\":true},{\"name\":\"can convert to an array\",\"pass\":true},{\"name\":\"head of list is final element from input array\",\"pass\":true},{\"name\":\"can convert longer list to an array\",\"pass\":true},{\"name\":\"can be reversed\",\"pass\":true},{\"name\":\"can be reversed when it has more elements\",\"pass\":true},{\"name\":\"can reverse with many elements\",\"pass\":true},{\"name\":\"can reverse a reversal\",\"pass\":true}]}\n",
+      "stderr": ""
+    }
+  },
+  {
+    "name": "quixbugs-untouched",
+    "result": {
+      "argv": [
+        "python3",
+        "check.py"
+      ],
+      "exit": 1,
+      "stdout": "",
+      "stderr": "Traceback (most recent call last):\n  File \"<TMP>/issue290-grader-ae68mbog/check.py\", line 6, in <module>\n    assert find_in_sorted(*args) == expected, (args,expected)\n           ~~~~~~~~~~~~~~^^^^^^^\n  File \"<TMP>/issue290-grader-ae68mbog/find_in_sorted.py\", line 13, in find_in_sorted\n    return binsearch(0, len(arr))\n  File \"<TMP>/issue290-grader-ae68mbog/find_in_sorted.py\", line 9, in binsearch\n    return binsearch(mid, end)\n  File \"<TMP>/issue290-grader-ae68mbog/find_in_sorted.py\", line 7, in binsearch\n    return binsearch(start, mid)\n  File \"<TMP>/issue290-grader-ae68mbog/find_in_sorted.py\", line 7, in binsearch\n    return binsearch(start, mid)\n  File \"<TMP>/issue290-grader-ae68mbog/find_in_sorted.py\", line 9, in binsearch\n    return binsearch(mid, end)\n  File \"<TMP>/issue290-grader-ae68mbog/find_in_sorted.py\", line 9, in binsearch\n    return binsearch(mid, end)\n  File \"<TMP>/issue290-grader-ae68mbog/find_in_sorted.py\", line 9, in binsearch\n    return binsearch(mid, end)\n  [Previous line repeated 992 more times]\nRecursionError: maximum recursion depth exceeded\n"
+    }
+  },
+  {
+    "name": "quixbugs-upstream-reference",
+    "source": "https://raw.githubusercontent.com/jkoppel/QuixBugs/4257f44b0ff1181dedaedee6a447e133219fcebf/correct_python_programs/find_in_sorted.py",
+    "sha256": "cce13769543e2972d66ea61c9d1f1751836c719abf336be8f50f2702d19477ea",
+    "result": {
+      "argv": [
+        "python3",
+        "check.py"
+      ],
+      "exit": 0,
+      "stdout": "PASS: all 7 pinned upstream cases\n",
+      "stderr": ""
+    }
+  },
+  {
+    "name": "normal-node-require",
+    "result": {
+      "argv": [
+        "node",
+        "-e",
+        "require(\"node:fs\");console.log(\"PASS require\")"
+      ],
+      "exit": 0,
+      "stdout": "PASS require\n",
+      "stderr": ""
+    }
+  }
+]

package/benchmarks/issue290-pilot/pricing-assumptions.json

@@ -0,0 +1,29 @@
+{
+  "status": "frozen historical pilot assumptions, not a verified provider price capture",
+  "recorded_date": "2026-10-14",
+  "currency": "USD per million tokens",
+  "codex": {
+    "model": "gpt-6-luna",
+    "input": 0.10,
+    "cache_read": 0.01,
+    "cache_write": 0.125,
+    "output": 0.50,
+    "large_context_above": 272000,
+    "large_context_input_multiplier": 2,
+    "large_context_output_multiplier": 1.5,
+    "reported_source": "https://developers.openai.com/api/docs/models/gpt-6-luna"
+  },
+  "claude": {
+    "model": "claude-haiku-5-5",
+    "input": 0.10,
+    "cache_read": 0.01,
+    "cache_write_5m": 0.125,
+    "cache_write_1h": 0.20,
+    "output": 0.50,
+    "large_context_above": 100000,
+    "large_context_multiplier": 5,
+    "reported_source": "https://platform.claude.com/docs/en/about-claude/pricing",
+    "multiplier_provenance": "Present in the original analyzer; no immutable provider page capture was retained. Assumption only."
+  },
+  "limitation": "Replays reproduce these historical assumptions; they do not validate actual charges or current/provider pricing."
+}

package/benchmarks/issue290-pilot/protocol.json

@@ -0,0 +1,32 @@
+{
+  "schema_version": 1,
+  "registered_date": "2026-10-14",
+  "status": "preregistered-before-model-calls",
+  "source_revision": "ac519fa204ec90feb5067bc2525381b8820d045e",
+  "question": "Does active-step reference disclosure reduce observed reference reads and execution overhead without losing required workflow behavior?",
+  "models": [
+    {"engine": "codex", "model": "gpt-6-luna", "effort": "high", "cli": "0.162.1"},
+    {"engine": "claude", "model": "claude-haiku-5-5", "effort": "high", "cli": "2.1.299"}
+  ],
+  "arms": {
+    "A": "Same canonical router and available documents, plus experiment-only eager document-read timing instruction.",
+    "B": "Same canonical router and available documents, current active-step-only disclosure/read timing."
+  },
+  "workloads": ["profile-lookup", "quixbugs-find-in-sorted", "polyglot-simple-linked-list-staged"],
+  "repeats_per_model_workload": 2,
+  "planned_episodes": 24,
+  "host_preflight_episodes": 2,
+  "order": "Repeat 1 AB; repeat 2 BA within each model/workload block; alternating model blocks. Provider cache residency is uncontrolled.",
+  "episode_timeout_seconds": {"profile-lookup": 180, "quixbugs-find-in-sorted": 300, "polyglot-simple-linked-list-staged": 600},
+  "aggregate_wall_budget_seconds": 7200,
+  "episode_output_token_cap": 80000,
+  "aggregate_output_token_cap": 500000,
+  "budget_note": "Subscription login, no new API keys. Caps checked at emitted usage events; telemetry can arrive only at episode end. No actual-billing assertion. Stop on inaccessible model, absent router attribution or invalid grader before expanding trials.",
+  "useful_effect": "At least 10% lower paired price-weighted input cost or reference-read content tokens, with no observed critical correctness/semantic regression; workload-specific screening only.",
+  "quality": "External acceptance checks plus attributed router/read/workflow evidence. Two repeats cannot establish a <=2 pp non-inferiority bound.",
+  "exclusions": "Infrastructure failures retained and marked inconclusive. Agent errors, valid budget exhaustion and incomplete work count as failures. No outcome-selected reruns.",
+  "closure": "Pilot only. Does not satisfy issue290 >=5 paired repeats per family; no automatic closure or general benefit claim.",
+  "public_selection_provenance": "0b221bf75b89b5834bc72869ddb9a6847db9760d; source selection reused, not the unrelated skill-text vs vanilla contrast.",
+  "required_preflight": ["pinned public fixture downloaded", "untouched fixture fails", "reference solution passes", "normal Node require and fixture commands work", "real host invokes canonical router", "actual model/effort recorded", "reference-open trace and usage collected"],
+  "reporting": "Per model/host and workload: completion, actual reference reads, uncached/cache creation/cache read/input/output/reasoning where exposed, estimated cost only where priced, wall time, tool counts, all errors/exclusions. Missing fields are unknown."
+}