harness-everything 0.47.1 → 0.47.2

package/benchmarks/issue290-pilot/evidence/replay/pairs.json

@@ -0,0 +1,482 @@
+[
+  {
+    "api_equivalent_cost_usd": {
+      "A": null,
+      "B": null,
+      "delta_percent": null,
+      "included": false
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": null,
+      "B": null,
+      "delta_percent": null,
+      "included": false
+    },
+    "engine": "codex",
+    "reference_returned_bytes": {
+      "A": 22023,
+      "B": 3066,
+      "delta_percent": -86.07819098215502,
+      "included": true
+    },
+    "repeat": 1,
+    "series": "issue290-fable-20261014",
+    "wall_seconds": {
+      "A": 600.392,
+      "B": 601.001,
+      "delta_percent": 0.10143372996307803,
+      "included": true
+    },
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": null,
+      "B": null,
+      "delta_percent": null,
+      "included": false
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": null,
+      "B": null,
+      "delta_percent": null,
+      "included": false
+    },
+    "engine": "codex",
+    "reference_returned_bytes": {
+      "A": 30683,
+      "B": 15124,
+      "delta_percent": -50.708861584590814,
+      "included": true
+    },
+    "repeat": 2,
+    "series": "issue290-fable-20261014",
+    "wall_seconds": {
+      "A": 439.355,
+      "B": 600.761,
+      "delta_percent": 36.73703497171989,
+      "included": true
+    },
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.04073616,
+      "B": 0.05423884000000001,
+      "delta_percent": 33.14666870907815,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.024593660000000003,
+      "B": 0.031938339999999996,
+      "delta_percent": 29.864119451923756,
+      "included": true
+    },
+    "engine": "claude",
+    "reference_returned_bytes": {
+      "A": 8300,
+      "B": 16914,
+      "delta_percent": 103.78313253012048,
+      "included": true
+    },
+    "repeat": 1,
+    "series": "issue290-fable-claude-retry-20261014",
+    "wall_seconds": {
+      "A": 265.736,
+      "B": 238.026,
+      "delta_percent": -10.427642472228069,
+      "included": true
+    },
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.029205180000000004,
+      "B": 0.050429640000000005,
+      "delta_percent": 72.67361474916434,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.017736180000000004,
+      "B": 0.03127264,
+      "delta_percent": 76.3211694964755,
+      "included": true
+    },
+    "engine": "claude",
+    "reference_returned_bytes": {
+      "A": 20633,
+      "B": 13910,
+      "delta_percent": -32.58372510056705,
+      "included": true
+    },
+    "repeat": 2,
+    "series": "issue290-fable-claude-retry-20261014",
+    "wall_seconds": {
+      "A": 128.821,
+      "B": 199.68,
+      "delta_percent": 55.00578321857462,
+      "included": true
+    },
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.01959743,
+      "B": 0.03243657,
+      "delta_percent": 65.51440673598528,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.01159493,
+      "B": 0.02028157,
+      "delta_percent": 74.91757173178276,
+      "included": true
+    },
+    "engine": "claude",
+    "reference_returned_bytes": {
+      "A": 5250,
+      "B": 3064,
+      "delta_percent": -41.63809523809524,
+      "included": true
+    },
+    "repeat": 1,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 88.905,
+      "B": 154.669,
+      "delta_percent": 73.97109273944099,
+      "included": true
+    },
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.01798511,
+      "B": 0.023325329999999995,
+      "delta_percent": 29.69245114430769,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.01129711,
+      "B": 0.014302330000000002,
+      "delta_percent": 26.601670692770107,
+      "included": true
+    },
+    "engine": "claude",
+    "reference_returned_bytes": {
+      "A": 3065,
+      "B": 3066,
+      "delta_percent": 0.03262642740619902,
+      "included": true
+    },
+    "repeat": 2,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 84.91,
+      "B": 110.124,
+      "delta_percent": 29.694971145919208,
+      "included": true
+    },
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.00581582,
+      "B": 0.0050241700000000005,
+      "delta_percent": -13.612010000309493,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.00481082,
+      "B": 0.004093670000000001,
+      "delta_percent": -14.907022087710612,
+      "included": true
+    },
+    "engine": "claude",
+    "reference_returned_bytes": {
+      "A": 18957,
+      "B": 3571,
+      "delta_percent": -81.16263121801973,
+      "included": true
+    },
+    "repeat": 1,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 12.103,
+      "B": 16.151,
+      "delta_percent": 33.44625299512518,
+      "included": true
+    },
+    "workload": "profile-lookup"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.005840049999999999,
+      "B": 0.00497036,
+      "delta_percent": -14.891824556296598,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.0048185500000000004,
+      "B": 0.00400936,
+      "delta_percent": -16.793226177999614,
+      "included": true
+    },
+    "engine": "claude",
+    "reference_returned_bytes": {
+      "A": 18957,
+      "B": 3571,
+      "delta_percent": -81.16263121801973,
+      "included": true
+    },
+    "repeat": 2,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 12.116,
+      "B": 15.131,
+      "delta_percent": 24.88445031363487,
+      "included": true
+    },
+    "workload": "profile-lookup"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.009865480000000001,
+      "B": 0.01780674,
+      "delta_percent": 80.49542444969732,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.00746998,
+      "B": 0.012818239999999998,
+      "delta_percent": 71.59671110230546,
+      "included": true
+    },
+    "engine": "claude",
+    "reference_returned_bytes": {
+      "A": 15078,
+      "B": 3182,
+      "delta_percent": -78.8964053588009,
+      "included": true
+    },
+    "repeat": 1,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 36.318,
+      "B": 84.865,
+      "delta_percent": 133.67200837050498,
+      "included": true
+    },
+    "workload": "quixbugs-find-in-sorted"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.01293407,
+      "B": 0.01267222,
+      "delta_percent": -2.0244980891552387,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.008934570000000001,
+      "B": 0.008777720000000001,
+      "delta_percent": -1.755540557631761,
+      "included": true
+    },
+    "engine": "claude",
+    "reference_returned_bytes": {
+      "A": 15078,
+      "B": 3065,
+      "delta_percent": -79.67237034089402,
+      "included": true
+    },
+    "repeat": 2,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 54.511,
+      "B": 69.67,
+      "delta_percent": 27.809066060061266,
+      "included": true
+    },
+    "workload": "quixbugs-find-in-sorted"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.02572438,
+      "B": 0.02799634,
+      "delta_percent": 8.831932975644117,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.015737880000000003,
+      "B": 0.016671340000000003,
+      "delta_percent": 5.931294431016124,
+      "included": true
+    },
+    "engine": "codex",
+    "reference_returned_bytes": {
+      "A": 22023,
+      "B": 3066,
+      "delta_percent": -86.07819098215502,
+      "included": true
+    },
+    "repeat": 1,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 218.852,
+      "B": 250.42,
+      "delta_percent": 14.424359841353967,
+      "included": true
+    },
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.0254005,
+      "B": 0.022629200000000002,
+      "delta_percent": -10.910415149308074,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.015807000000000005,
+      "B": 0.014696699999999998,
+      "delta_percent": -7.024103245397649,
+      "included": true
+    },
+    "engine": "codex",
+    "reference_returned_bytes": {
+      "A": 22023,
+      "B": 3066,
+      "delta_percent": -86.07819098215502,
+      "included": true
+    },
+    "repeat": 2,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 237.141,
+      "B": 177.548,
+      "delta_percent": -25.129775112696663,
+      "included": true
+    },
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.00220756,
+      "B": 0.00229032,
+      "delta_percent": 3.7489354762724596,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.00194956,
+      "B": 0.00192282,
+      "delta_percent": -1.3715915386035786,
+      "included": true
+    },
+    "engine": "codex",
+    "reference_returned_bytes": {
+      "A": 18957,
+      "B": 3571,
+      "delta_percent": -81.16263121801973,
+      "included": true
+    },
+    "repeat": 1,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 17.148,
+      "B": 17.136,
+      "delta_percent": -0.06997900629811322,
+      "included": true
+    },
+    "workload": "profile-lookup"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.00204398,
+      "B": 0.0029894,
+      "delta_percent": 46.25387723950333,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.00154448,
+      "B": 0.0026714,
+      "delta_percent": 72.96436341033875,
+      "included": true
+    },
+    "engine": "codex",
+    "reference_returned_bytes": {
+      "A": 18957,
+      "B": 3571,
+      "delta_percent": -81.16263121801973,
+      "included": true
+    },
+    "repeat": 2,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 19.145,
+      "B": 16.12,
+      "delta_percent": -15.800470096630967,
+      "included": true
+    },
+    "workload": "profile-lookup"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.00857276,
+      "B": 0.01922112,
+      "delta_percent": 124.21157246907649,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.00615976,
+      "B": 0.014933620000000002,
+      "delta_percent": 142.43834175357483,
+      "included": true
+    },
+    "engine": "codex",
+    "reference_returned_bytes": {
+      "A": 15078,
+      "B": 4665,
+      "delta_percent": -69.0608834062873,
+      "included": true
+    },
+    "repeat": 1,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 69.562,
+      "B": 179.458,
+      "delta_percent": 157.98280670481012,
+      "included": true
+    },
+    "workload": "quixbugs-find-in-sorted"
+  },
+  {
+    "api_equivalent_cost_usd": {
+      "A": 0.01109618,
+      "B": 0.0148342,
+      "delta_percent": 33.6874491942272,
+      "included": true
+    },
+    "api_equivalent_input_cost_usd": {
+      "A": 0.007470180000000001,
+      "B": 0.0114482,
+      "delta_percent": 53.251996605168806,
+      "included": true
+    },
+    "engine": "codex",
+    "reference_returned_bytes": {
+      "A": 15078,
+      "B": 3066,
+      "delta_percent": -79.66573816155989,
+      "included": true
+    },
+    "repeat": 2,
+    "series": "issue290-formal-20261014",
+    "wall_seconds": {
+      "A": 146.2,
+      "B": 98.86,
+      "delta_percent": -32.38030095759233,
+      "included": true
+    },
+    "workload": "quixbugs-find-in-sorted"
+  }
+]

package/benchmarks/issue290-pilot/evidence/replay/provenance.json

@@ -0,0 +1,24 @@
+{
+  "archive_sha256": "93bcb349c6ebbdcd3673641f3dc23148e73e87a0ec975d31bf31d2731d828723",
+  "frozen_documents_sha256": "d294738c15c313f9800d900fcaf4c0689ecc150f71a3de2443119676d0adc136",
+  "frozen_not_independently_recovered": [
+    "request usage",
+    "child dispatch metadata",
+    "effort",
+    "usage aggregates"
+  ],
+  "irretrievable": [
+    "unobserved worker usage/model identities",
+    "interrupted final usage"
+  ],
+  "pricing_assumptions_sha256": "b8cfd7cd8afb882a2bf55d666110a6967fefa90d307f9b2105fc7cece2e935b7",
+  "recomputed": [
+    "parent-stream content attribution",
+    "artifact checks",
+    "gate-name invocation",
+    "metric denominators",
+    "paired deltas"
+  ],
+  "unknown_policy": "non-reconstructable whole-run costs remain null; no host-home reads or model calls",
+  "verified_exported_files": 621
+}

package/benchmarks/issue290-pilot/evidence/replay/tables.md

@@ -0,0 +1,37 @@
+<!-- Generated by replay.py; parent-stream content attribution only. -->
+| Family | Model | Pass / quality n | Excluded attempts |
+|---|---|---:|---:|
+| True Fable | claude | 4/4 | 4 |
+| True Fable | codex | 1/4 | 0 |
+| polyglot-simple-linked-list-staged | claude | 4/4 | 0 |
+| polyglot-simple-linked-list-staged | codex | 3/4 | 0 |
+| profile-lookup | claude | 0/4 | 0 |
+| profile-lookup | codex | 4/4 | 0 |
+| quixbugs-find-in-sorted | claude | 4/4 | 0 |
+| quixbugs-find-in-sorted | codex | 4/4 | 0 |
+
+### Attributable reference bytes (lower bound)
+
+| Family | Model | Included / candidate pairs | A | B | Delta |
+|---|---|---:|---:|---:|---:|
+| True Fable | claude | 2/2 | 28933 | 30824 | +6.5% |
+| True Fable | codex | 2/2 | 52706 | 18190 | -65.5% |
+| polyglot-simple-linked-list-staged | claude | 2/2 | 8315 | 6130 | -26.3% |
+| polyglot-simple-linked-list-staged | codex | 2/2 | 44046 | 6132 | -86.1% |
+| profile-lookup | claude | 2/2 | 37914 | 7142 | -81.2% |
+| profile-lookup | codex | 2/2 | 37914 | 7142 | -81.2% |
+| quixbugs-find-in-sorted | claude | 2/2 | 30156 | 6247 | -79.3% |
+| quixbugs-find-in-sorted | codex | 2/2 | 30156 | 7731 | -74.4% |
+
+### List-price input cost (frozen telemetry)
+
+| Family | Model | Included / candidate pairs | A | B | Delta |
+|---|---|---:|---:|---:|---:|
+| True Fable | claude | 2/2 | 0.042330 | 0.063211 | +49.3% |
+| True Fable | codex | 0/2 | unknown | unknown | unknown |
+| polyglot-simple-linked-list-staged | claude | 2/2 | 0.022892 | 0.034584 | +51.1% |
+| polyglot-simple-linked-list-staged | codex | 2/2 | 0.031545 | 0.031368 | -0.6% |
+| profile-lookup | claude | 2/2 | 0.009629 | 0.008103 | -15.9% |
+| profile-lookup | codex | 2/2 | 0.003494 | 0.004594 | +31.5% |
+| quixbugs-find-in-sorted | claude | 2/2 | 0.016405 | 0.021596 | +31.6% |
+| quixbugs-find-in-sorted | codex | 2/2 | 0.013630 | 0.026382 | +93.6% |

package/benchmarks/issue290-pilot/export.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Sanitize attributed evidence while retaining raw and exported SHA-256 hashes."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import re
+import socket
+import tarfile
+
+HERE=Path(__file__).resolve().parent
+SOURCE=str(HERE.parent.parent)
+HOME=str(Path.home())
+HOST=socket.gethostname().split('.')[0]
+
+SECRET_PATTERNS = {
+    'provider-token': r'\b(?:sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|xox[baprs]-[A-Za-z0-9-]{20,}|AKIA[A-Z0-9]{16})\b',
+    'private-key': r'-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----',
+    'authorization': r'(?i)\b(?:bearer|basic)\s+[A-Za-z0-9+/_.=-]{16,}',
+    'assigned-secret': r'''(?i)["']?\b(?:api[_-]?key|access[_-]?token|client[_-]?secret|password|aws_secret_access_key)["']?\s*[:=]\s*["']?[A-Za-z0-9+/_.=-]{16,}''',
+}
+
+def validate_no_secrets(text):
+    # JSON escaping cannot hide a credential in a nested transcript/log field.
+    candidates = [text]
+    try:
+        value = json.loads(text)
+    except (ValueError, TypeError):
+        value = None
+    def strings(value):
+        if isinstance(value, str):
+            candidates.append(value)
+        elif isinstance(value, dict):
+            for key, item in value.items():
+                candidates.append(str(key))
+                strings(item)
+        elif isinstance(value, list):
+            for item in value:
+                strings(item)
+    strings(value)
+    for candidate in candidates:
+        for category, pattern in SECRET_PATTERNS.items():
+            if re.search(pattern, candidate):
+                raise ValueError('possible credential class: ' + category)
+
+def scan_final(paths):
+    count = 0
+    for path in map(Path, paths):
+        if path.name.endswith(('.tar.gz', '.tgz')):
+            with tarfile.open(path) as archive:
+                for member in archive.getmembers():
+                    validate_no_secrets(member.name)
+                    if member.isfile():
+                        validate_no_secrets(archive.extractfile(member).read().decode('utf8'))
+                        count += 1
+        else:
+            for file in sorted(path.rglob('*')) if path.is_dir() else [path]:
+                if file.is_file() and '__pycache__' not in file.parts:
+                    validate_no_secrets(str(file))
+                    validate_no_secrets(file.read_text())
+                    count += 1
+    return count
+
+def sanitize(text):
+    validate_no_secrets(text)
+    text=text.replace(SOURCE,'<HARNESS_SOURCE>').replace(HOME,'<USER_HOME>')
+    text=re.sub(r'(?:/private)?/var/folders/[^/]+/[^/]+/T/', '<TMP>/',text)
+    text=text.replace('/private/tmp/','<TMP>/').replace('/tmp/','<TMP>/')
+    text=re.sub(r'\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\b','<REDACTED_EMAIL>',text)
+    if HOST:text=text.replace(HOST,'<HOST>')
+    # Local login name leaks through `ls -l` owner columns in captured tool output.
+    text=re.sub(r'\bdaniel\.chang\b','<USER>',text)
+    # Fail closed rather than publish a possible credential silently.
+    validate_no_secrets(text)
+    return text
+
+def strip_private_reasoning(value):
+    if isinstance(value,list):return [strip_private_reasoning(v) for v in value]
+    if isinstance(value,dict):
+        if value.get('type') in ['thinking','redacted_thinking','reasoning']:
+            return {'type':'private_reasoning_redacted'}
+        return {k:strip_private_reasoning(v) for k,v in value.items() if k not in ['signature','thinking']}
+    return value
+
+if __name__=='__main__':
+    p=argparse.ArgumentParser();p.add_argument('roots',nargs='+');p.add_argument('--output');p.add_argument('--scan-only',action='store_true')
+    a=p.parse_args()
+    if a.scan_only:
+        print('credential scan PASS:', scan_final(a.roots), 'UTF-8 files; configured classes only')
+        raise SystemExit(0)
+    if not a.output:p.error('--output is required for export')
+    out=Path(a.output);out.mkdir(parents=True,exist_ok=True);inventory=[]
+    for root in map(Path,a.roots):
+        for file in sorted(root.rglob('*')):
+            if not file.is_file() or file.suffix not in ['.json','.jsonl','.txt','.py','.js','.cjs','.log']:
+                continue
+            relative=Path(root.name)/file.relative_to(root)
+            target=out/relative;target.parent.mkdir(parents=True,exist_ok=True)
+            raw=file.read_bytes();text=raw.decode('utf8')
+            if file.suffix=='.jsonl':
+                text='\n'.join(json.dumps(strip_private_reasoning(json.loads(line)),ensure_ascii=False)
+                    for line in text.splitlines() if line.strip())+'\n'
+            elif file.suffix=='.json':text=json.dumps(strip_private_reasoning(json.loads(text)),indent=2,ensure_ascii=False)+'\n'
+            clean=sanitize(text).encode()
+            target.write_bytes(clean)
+            inventory.append({'path':str(relative),'raw_sha256':hashlib.sha256(raw).hexdigest(),
+                              'exported_sha256':hashlib.sha256(clean).hexdigest(),'bytes':len(clean)})
+    (out/'inventory.json').write_text(json.dumps(inventory,indent=2)+'\n')
+    scan_final([out])
+    print('exported',len(inventory),'files;',sum(x['bytes'] for x in inventory),'bytes')