@andresmassello/uscha 1.83.0 → 1.85.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4007,6 +4007,86 @@ def _extract_static_py(repo_path, tracked):
4007
4007
  return obs, unsupported
4008
4008
 
4009
4009
 
4010
+ # Node one-liner (ADR-028): `require`s the impl and reports ITS OWN module.exports --
4011
+ # the target's own runtime reporting its own public surface, never a regex-narrated AST. A
4012
+ # module that guards its entry point with `require.main === module` (the SPEC contract) will
4013
+ # not auto-run under `require()`, since `require.main` here is this -e script, not the module.
4014
+ _NODE_EXPORTS_PROBE = (
4015
+ "const p=process.argv[1]; const m=require(require('path').resolve(p)); "
4016
+ "const ks=Object.keys(m).filter(k=>typeof m[k]==='function'||typeof m[k]==='object'); "
4017
+ "process.stdout.write(JSON.stringify(ks.sort()))"
4018
+ )
4019
+ _STATIC_JS_TIMEOUT = 15
4020
+
4021
+
4022
+ def _extract_static_js(repo_path, tracked):
4023
+ """v0 static extractor for JS (ADR-028): the target's OWN runtime reports its own public
4024
+ surface via `node -e` + `Object.keys(module.exports)` -- measured, not parsed by a
4025
+ heuristic. Function-vs-object and the observation's line number are best-effort source
4026
+ regexes (Node reports WHICH names are exported; it does not report where they are
4027
+ defined). If node is absent, or a file fails to load/print a clean JSON array, that file's
4028
+ surface is UNMEASURED -- named in `unsupported`, never silently empty-as-measured.
4029
+ Returns (observations, unsupported) where unsupported is a list of "<file>: <reason>"."""
4030
+ obs, unsupported = [], []
4031
+ js_files = sorted(rel for rel in tracked if rel.lower().endswith(".js"))
4032
+ if not js_files:
4033
+ return obs, unsupported
4034
+ node = shutil.which("node")
4035
+ if not node:
4036
+ return obs, ["%s: node not on PATH" % rel for rel in js_files]
4037
+ for rel in js_files:
4038
+ full = os.path.join(repo_path, rel.replace("/", os.sep))
4039
+ abs_path = os.path.abspath(full)
4040
+ try:
4041
+ r = subprocess.run([node, "-e", _NODE_EXPORTS_PROBE, "--", abs_path],
4042
+ capture_output=True, text=True, encoding="utf-8",
4043
+ errors="replace", timeout=_STATIC_JS_TIMEOUT,
4044
+ cwd=os.path.dirname(full))
4045
+ except subprocess.TimeoutExpired:
4046
+ unsupported.append("%s: node timed out extracting exports" % rel)
4047
+ continue
4048
+ except OSError as exc:
4049
+ unsupported.append("%s: could not run node: %s" % (rel, exc))
4050
+ continue
4051
+ if r.returncode != 0:
4052
+ unsupported.append("%s: node exited %d extracting exports (%s)"
4053
+ % (rel, r.returncode, (r.stderr or "").strip()[:200]))
4054
+ continue
4055
+ try:
4056
+ names = json.loads(r.stdout)
4057
+ if not isinstance(names, list):
4058
+ raise ValueError("exports output was not a JSON list")
4059
+ except ValueError as exc:
4060
+ unsupported.append("%s: could not parse exports output: %s" % (rel, exc))
4061
+ continue
4062
+ try:
4063
+ with open(full, encoding="utf-8", errors="replace") as fh:
4064
+ src = fh.read()
4065
+ except OSError:
4066
+ src = ""
4067
+ src_lines = src.splitlines()
4068
+ for name in sorted(names):
4069
+ is_function = bool(re.search(r"\bfunction\s+%s\s*\(" % re.escape(name), src))
4070
+ lineno = 0
4071
+ patterns = (r"\bfunction\s+%s\b" % re.escape(name),
4072
+ r"\b%s\s*=" % re.escape(name),
4073
+ r"exports\.%s\b" % re.escape(name),
4074
+ r"module\.exports\b")
4075
+ for i, ln in enumerate(src_lines, 1):
4076
+ if any(re.search(p, ln) for p in patterns):
4077
+ lineno = i
4078
+ break
4079
+ stmt = ("%s exports function %s" % (rel, name) if is_function
4080
+ else "%s exports object %s" % (rel, name))
4081
+ prov = "%s:%d" % (rel, lineno)
4082
+ obs.append({"id": _obs_id("contract", stmt, prov), "type": "contract",
4083
+ "statement": stmt, "evidence_class": "static",
4084
+ "provenance": {"files": [prov],
4085
+ "derivation": "node -e Object.keys(module.exports)",
4086
+ "tool": "qa_ledger-static-js"}})
4087
+ return obs, unsupported
4088
+
4089
+
4010
4090
  def _under_bound(rel, bound):
4011
4091
  return bound is None or rel == bound or rel.startswith(bound + "/")
4012
4092
 
@@ -5307,26 +5387,69 @@ def cmd_compile_ingest(args):
5307
5387
  _BOOTSTRAP_CASE_TIMEOUT = 15 # a compiled hook must decide fast
5308
5388
 
5309
5389
 
5390
+ def _entry_unit(source_list):
5391
+ """The unit the oracle runs (ADR-029): the one whose basename starts with `cli.` if any,
5392
+ else the first declared source unit. Single-unit compilations resolve to their only unit,
5393
+ unchanged. Returns None when there is no usable unit."""
5394
+ units = [u.get("unit") for u in (source_list or []) if isinstance(u, dict) and u.get("unit")]
5395
+ if not units:
5396
+ return None
5397
+ for u in units:
5398
+ if os.path.basename(u).lower().startswith("cli."):
5399
+ return u
5400
+ return units[0]
5401
+
5402
+
5403
+ def _static_surface_for(cd, unit):
5404
+ """Route the reverse-discovery organ by the unit's extension (ADR-028/029)."""
5405
+ if os.path.splitext(unit)[1].lower() == ".js":
5406
+ return _extract_static_js(cd, [unit])
5407
+ so, _n = _extract_static_py(cd, [unit])
5408
+ return so, []
5409
+
5410
+
5411
+ def _impl_interpreter(impl_path):
5412
+ """Resolve the interpreter argv prefix for one implementation file by extension (ADR-028):
5413
+ `.py` runs under this same Python (unchanged); `.js` runs under `node`, resolved from PATH.
5414
+ Returns None when the extension's interpreter cannot be resolved (today: node absent) --
5415
+ the caller must treat that as UNMEASURED, never a fake red or green."""
5416
+ if impl_path.lower().endswith(".js"):
5417
+ node = shutil.which("node")
5418
+ return [node] if node else None
5419
+ return [sys.executable]
5420
+
5421
+
5310
5422
  def _run_oracle_case(impl_path, case):
5311
5423
  """Run ONE withheld oracle case against a compiled implementation: feed the case's stdin
5312
- (raw_stdin verbatim if present, else json.dumps(payload)) to `python <impl>`, and check its
5313
- result against whichever expectations the case declares. Deterministic execution -- the
5314
- oracle is a `measured` fact, never an LLM judgment. Returns a per-case result dict.
5424
+ (raw_stdin verbatim if present, else json.dumps(payload)) to the impl's interpreter (ADR-028:
5425
+ `python` for `.py`, `node` for `.js`), and check its result against whichever expectations
5426
+ the case declares. Deterministic execution -- the oracle is a `measured` fact, never an LLM
5427
+ judgment. Returns a per-case result dict.
5315
5428
 
5316
5429
  A case may assert any of: `expected_exit` (process exit code -- the M4 guard's contract),
5317
5430
  `expected_stdout` (the program's stdout, compared stripped -- for archetypes that COMPUTE an
5318
5431
  output, e.g. a parser or transformer), and `expected_json` (stdout parsed as JSON and
5319
5432
  compared structurally, so an output whose formatting is free but whose value is fixed is not
5320
5433
  penalised for whitespace or key order). The case passes iff EVERY declared expectation holds;
5321
- a case that declares none proves nothing and fails."""
5434
+ a case that declares none proves nothing and fails.
5435
+
5436
+ If the impl's interpreter cannot be resolved (a `.js` impl and no `node` on PATH), the case
5437
+ is UNMEASURED -- never a fake red or green (ADR-028)."""
5322
5438
  if "raw_stdin" in case:
5323
5439
  stdin = case["raw_stdin"]
5324
5440
  else:
5325
5441
  stdin = json.dumps(case.get("payload"))
5442
+ interp = _impl_interpreter(impl_path)
5443
+ if interp is None:
5444
+ return {"name": case.get("name"), "expected": case.get("expected_exit"), "got": None,
5445
+ "ok": False, "error": "node not on PATH", "unmeasured": True}
5326
5446
  try:
5327
- r = subprocess.run([sys.executable, impl_path], input=stdin, capture_output=True,
5328
- text=True, encoding="utf-8", errors="replace",
5329
- timeout=_BOOTSTRAP_CASE_TIMEOUT)
5447
+ # cwd = the impl's own directory (ADR-029): a multi-unit compilation imports its
5448
+ # sibling modules by bare name; single-unit impls are unaffected by their cwd
5449
+ r = subprocess.run(interp + [os.path.abspath(impl_path)], input=stdin,
5450
+ capture_output=True, text=True, encoding="utf-8", errors="replace",
5451
+ timeout=_BOOTSTRAP_CASE_TIMEOUT,
5452
+ cwd=os.path.dirname(os.path.abspath(impl_path)) or None)
5330
5453
  got, out, err = r.returncode, r.stdout, None
5331
5454
  except subprocess.TimeoutExpired:
5332
5455
  got, out, err = None, "", "timeout"
@@ -5369,22 +5492,30 @@ def cmd_bootstrap_oracle(args):
5369
5492
  file=sys.stderr)
5370
5493
  sys.exit(2)
5371
5494
  results = [_run_oracle_case(args.impl, c) for c in cases]
5495
+ unmeasured = bool(results) and all(r.get("unmeasured") for r in results)
5372
5496
  passed = sum(1 for r in results if r["ok"])
5373
5497
  failed = [r for r in results if not r["ok"]]
5374
5498
  report = {"impl": args.impl, "oracle": args.oracle, "total": len(results),
5375
5499
  "passed": passed, "failed": len(failed),
5376
- "oracle_green": not failed, "results": results}
5500
+ "oracle_green": (None if unmeasured else not failed), "results": results}
5501
+ if unmeasured:
5502
+ report["unmeasured"] = results[0]["error"]
5377
5503
  if args.ledger and args.repo:
5378
5504
  ledger = _load(args.ledger)
5379
5505
  _repo_node(ledger, args.repo)
5380
5506
  rec = {"impl": os.path.basename(args.impl), "oracle": os.path.basename(args.oracle),
5381
5507
  "total": len(results), "passed": passed, "failed": len(failed),
5382
- "oracle_green": not failed,
5508
+ "oracle_green": (None if unmeasured else not failed),
5383
5509
  "failing": [r["name"] for r in failed], "at": _now()}
5510
+ if unmeasured:
5511
+ rec["unmeasured"] = results[0]["error"]
5384
5512
  ledger.setdefault("bootstrap_oracle", []).append(rec)
5385
5513
  _save(args.ledger, ledger)
5386
5514
  if args.json:
5387
5515
  print(json.dumps(report, indent=2, ensure_ascii=False))
5516
+ elif unmeasured:
5517
+ print("BOOTSTRAP-ORACLE %s: UNMEASURED -- %s"
5518
+ % (os.path.basename(args.impl), report["unmeasured"]))
5388
5519
  else:
5389
5520
  print("BOOTSTRAP-ORACLE %s: %d/%d cases pass -- %s"
5390
5521
  % (os.path.basename(args.impl), passed, len(results),
@@ -5394,17 +5525,60 @@ def cmd_bootstrap_oracle(args):
5394
5525
  print(" x %s: expected exit %s, got %s%s"
5395
5526
  % (r["name"], r["expected"], r["got"],
5396
5527
  " (%s)" % r["error"] if r["error"] else ""))
5397
- sys.exit(0 if not failed else 1)
5528
+ sys.exit(2 if unmeasured else (0 if not failed else 1))
5529
+
5530
+
5531
+ _JS_IMPORT_RE = re.compile(
5532
+ r"""require\(\s*['"]([^'"]+)['"]\s*\)"""
5533
+ r"""|import\s+(?:.+?\s+from\s+)?['"]([^'"]+)['"]"""
5534
+ )
5535
+
5536
+
5537
+ def _impl_metrics_js(path, src):
5538
+ """JS structural fingerprint (ADR-028), honestly NARROWER than Python's: `loc` (non-blank,
5539
+ non-`//` lines, with `/*..*/` blocks toggled by a simple per-line state machine) and
5540
+ `imports` (require()/import specifiers -- a lexical fact over string literals, not a
5541
+ narrated structure) are measured; `ast_nodes`/`functions`/`classes` are UNMEASURED (None) --
5542
+ no stdlib JS AST exists and the engine does not invent one."""
5543
+ loc = 0
5544
+ in_block = False
5545
+ for ln in src.splitlines():
5546
+ s = ln.strip()
5547
+ if in_block:
5548
+ if "*/" in s:
5549
+ in_block = False
5550
+ continue
5551
+ if not s:
5552
+ continue
5553
+ if s.startswith("/*"):
5554
+ if "*/" not in s[2:]:
5555
+ in_block = True
5556
+ continue
5557
+ if s.startswith("//"):
5558
+ continue
5559
+ loc += 1
5560
+ imports = set()
5561
+ for m in _JS_IMPORT_RE.finditer(src):
5562
+ spec = m.group(1) or m.group(2)
5563
+ if spec:
5564
+ imports.add(spec)
5565
+ return {"path": path, "loc": loc, "ast_nodes": None, "functions": None,
5566
+ "classes": None, "imports": sorted(imports),
5567
+ "sha256": hashlib.sha256(src.encode("utf-8")).hexdigest()}
5398
5568
 
5399
5569
 
5400
5570
  def _impl_metrics(path):
5401
5571
  """Structural fingerprint of one implementation: physical LOC, AST node count, function
5402
- and class counts, and the set of imported top-level modules. Deterministic; no judgment."""
5572
+ and class counts, and the set of imported top-level modules. Deterministic; no judgment.
5573
+ Routed by extension (ADR-028): `.py` unchanged; `.js` delegates to `_impl_metrics_js`,
5574
+ which returns the same keys with `ast_nodes`/`functions`/`classes` UNMEASURED (None)."""
5403
5575
  try:
5404
5576
  with open(path, encoding="utf-8", errors="replace") as fh:
5405
5577
  src = fh.read()
5406
5578
  except OSError as exc:
5407
5579
  return {"path": path, "error": "unreadable: %s" % exc}
5580
+ if path.lower().endswith(".js"):
5581
+ return _impl_metrics_js(path, src)
5408
5582
  loc = sum(1 for ln in src.splitlines() if ln.strip())
5409
5583
  try:
5410
5584
  tree = ast.parse(src)
@@ -5531,8 +5705,14 @@ def _bench_curation_map(records):
5531
5705
  def _bench_oracle_all(impl_path, cases):
5532
5706
  results = [_run_oracle_case(impl_path, c) for c in cases]
5533
5707
  passed = sum(1 for r in results if r["ok"])
5534
- return {"passed": passed, "total": len(results), "green": passed == len(results),
5535
- "failing": [r["name"] for r in results if not r["ok"]]}
5708
+ out = {"passed": passed, "total": len(results), "green": passed == len(results),
5709
+ "failing": [r["name"] for r in results if not r["ok"]]}
5710
+ # every case UNMEASURED (ADR-028: a `.js` impl with no `node` on PATH) is NOT a fake red --
5711
+ # name the reason and let the caller treat it as absent, never as a scored FAIL.
5712
+ if results and all(r.get("unmeasured") for r in results):
5713
+ out["unmeasured"] = results[0]["error"]
5714
+ out["green"] = False
5715
+ return out
5536
5716
 
5537
5717
 
5538
5718
  def _bench_entry(entry_dir, name, fidelity=False, curation=None):
@@ -5564,15 +5744,16 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
5564
5744
  with open(cj, encoding="utf-8-sig") as fh:
5565
5745
  c = json.load(fh)
5566
5746
  src = c.get("source") or []
5567
- unit = src[0].get("unit") if src else None
5747
+ unit = _entry_unit(src)
5748
+ all_units = [u.get("unit") for u in src if isinstance(u, dict) and u.get("unit")]
5568
5749
  model = (c.get("compilation_report") or {}).get("model")
5569
5750
  except (OSError, ValueError, AttributeError, IndexError):
5570
- pass
5751
+ all_units = []
5571
5752
  impl = os.path.join(cd, unit.replace("/", os.sep)) if unit else None
5572
5753
  ores = (_bench_oracle_all(impl, cases) if impl and os.path.isfile(impl)
5573
5754
  else {"passed": 0, "total": len(cases), "green": False, "failing": ["<no impl>"]})
5574
- comp_rec = {"dir": d, "model": model, "impl": impl,
5575
- "compile_valid": not errors, "oracle": ores}
5755
+ comp_rec = {"dir": d, "model": model, "impl": impl, "entry_unit": unit,
5756
+ "units": len(all_units), "compile_valid": not errors, "oracle": ores}
5576
5757
  if fidelity and impl and os.path.isfile(impl) and unit:
5577
5758
  # the per-compiler fidelity descriptor (ADR-022): the M1 static extractor applied
5578
5759
  # to the compiled artifact -- reverse discovery per compiler. Advisory by
@@ -5597,20 +5778,36 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
5597
5778
  units_f.add(u2["unit"])
5598
5779
  except AttributeError:
5599
5780
  pass
5600
- sobs, _uns = _extract_static_py(cd, [unit])
5781
+ # ADR-028: the static extractor is routed by the source unit's extension -- `.py`
5782
+ # via the AST (unchanged), `.js` via the target's own `node` runtime. `uns_f` is a
5783
+ # list of "<file>: <reason>" for JS (an int count for Python, discarded either way).
5784
+ sobs, uns_f = [], []
5785
+ for u_all in (all_units or [unit]):
5786
+ so, un = _static_surface_for(cd, u_all)
5787
+ sobs.extend(so)
5788
+ uns_f.extend(un)
5601
5789
  fn_names, cls_names = [], []
5602
5790
  for o2 in sobs:
5603
- m2 = re.search(r"defines function (\w+)", o2.get("statement", ""))
5791
+ m2 = re.search(r"defines function (\w+)|exports function (\w+)",
5792
+ o2.get("statement", ""))
5604
5793
  if m2:
5605
- fn_names.append(m2.group(1))
5606
- m2 = re.search(r"defines class (\w+)", o2.get("statement", ""))
5794
+ fn_names.append(m2.group(1) or m2.group(2))
5795
+ m2 = re.search(r"defines class (\w+)|exports object (\w+)",
5796
+ o2.get("statement", ""))
5607
5797
  if m2:
5608
- cls_names.append(m2.group(1))
5798
+ cls_names.append(m2.group(1) or m2.group(2))
5609
5799
  unex = sorted(u for u in units_f if u not in traced_f)
5800
+ if not sobs and uns_f:
5801
+ # node absent (or every JS file failed to load): the surface is UNMEASURED,
5802
+ # never a silent "0 functions, 0 classes" that reads as a measured empty
5803
+ # surface (ADR-028 -- absence named, same discipline as curation_closure below).
5804
+ static_surface = "UNMEASURED: %s" % "; ".join(uns_f)
5805
+ else:
5806
+ static_surface = {"functions": len(fn_names), "classes": len(cls_names),
5807
+ "names": sorted(fn_names + cls_names)}
5610
5808
  comp_rec["fidelity"] = {
5611
5809
  "trace_coverage": round(len(covered_f) / max(len(node_ids_f), 1), 3),
5612
- "static_surface": {"functions": len(fn_names), "classes": len(cls_names),
5613
- "names": sorted(fn_names + cls_names)},
5810
+ "static_surface": static_surface,
5614
5811
  "oracle_passrate": (round(ores["passed"] / ores["total"], 3)
5615
5812
  if ores["total"] else None),
5616
5813
  "unexplained_share": round(len(unex) / max(len(units_f), 1), 3),
@@ -5634,9 +5831,17 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
5634
5831
  stub_dir = os.path.join(entry_dir, "stub")
5635
5832
  stub_green = False
5636
5833
  if os.path.isdir(stub_dir):
5637
- stubs = sorted(f for f in os.listdir(stub_dir) if f.endswith(".py"))
5638
- if stubs:
5639
- sres = _bench_oracle_all(os.path.join(stub_dir, stubs[0]), cases)
5834
+ stubs = sorted(f for f in os.listdir(stub_dir) if f.endswith((".py", ".js")))
5835
+ stub_path = os.path.join(stub_dir, stubs[0]) if stubs else None
5836
+ if stub_path is None and os.path.isdir(os.path.join(stub_dir, "source")):
5837
+ # multi-unit entries (ADR-029): the stub is a directory shaped like a compilation;
5838
+ # its entry unit is the cli.* under source/
5839
+ cands = sorted(f for f in os.listdir(os.path.join(stub_dir, "source"))
5840
+ if f.lower().startswith("cli.") and f.endswith((".py", ".js")))
5841
+ if cands:
5842
+ stub_path = os.path.join(stub_dir, "source", cands[0])
5843
+ if stub_path:
5844
+ sres = _bench_oracle_all(stub_path, cases)
5640
5845
  stub_green = sres["green"]
5641
5846
  rec["discrimination"] = {"stub_passed": sres["passed"], "total": sres["total"],
5642
5847
  "stub_green": stub_green}
@@ -5651,7 +5856,14 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
5651
5856
  # distinct is None when variance could not be computed (fewer than 2 resolvable impl files),
5652
5857
  # which is NOT the same as a byte-identical convergence -- keep the two reasons apart.
5653
5858
  distinct = rec["variance"]["all_distinct"] if rec["variance"] else None
5654
- if stub_green:
5859
+ # ADR-028: node absent for a `.js` entry means EVERY case of that compilation came back
5860
+ # UNMEASURED -- never score that as a FAIL (nor a PASS); the entry stays PENDING with the
5861
+ # reason named, ahead of every other verdict check.
5862
+ unmeasured_js = next((i["oracle"]["unmeasured"] for i in impls
5863
+ if i["oracle"].get("unmeasured")), None)
5864
+ if unmeasured_js:
5865
+ rec["verdict"], rec["reason"] = "PENDING", "%s -- JS entry unmeasured" % unmeasured_js
5866
+ elif stub_green:
5655
5867
  rec["verdict"], rec["reason"] = "FAIL", ("oracle satisfied by a degenerate stub -- not "
5656
5868
  "discriminating; the entry proves nothing")
5657
5869
  elif not all_valid:
@@ -5720,15 +5932,23 @@ def _render_bench_md(table, anon, recs):
5720
5932
  if isinstance(cur_v, float):
5721
5933
  cur_v = "%.3f (judged %d/%d)" % (cur_v, fd["curation"]["judged"],
5722
5934
  fd["curation"]["total"])
5723
- lines.append("- fidelity `%s` (%s): trace %.2f · surface %d fn / %d cls · "
5935
+ ss = fd["static_surface"]
5936
+ # ADR-028: `static_surface` is UNMEASURED (a named string) when a JS entry's
5937
+ # extractor could not run (node absent) -- never a silent "0 fn / 0 cls".
5938
+ surface_txt = ("%d fn / %d cls" % (ss["functions"], ss["classes"])
5939
+ if isinstance(ss, dict) else ss)
5940
+ lines.append("- fidelity `%s` (%s): trace %.2f · surface %s · "
5724
5941
  "oracle %s · unexplained %.2f · curation %s" % (
5725
5942
  i["dir"], anon.get(i["model"], i["model"] or "?"),
5726
- fd["trace_coverage"], fd["static_surface"]["functions"],
5727
- fd["static_surface"]["classes"],
5943
+ fd["trace_coverage"], surface_txt,
5728
5944
  ("%.3f" % fd["oracle_passrate"])
5729
5945
  if fd["oracle_passrate"] is not None else "n/a",
5730
5946
  fd["unexplained_share"], cur_v))
5731
5947
  lines.append("")
5948
+ if any((i.get("impl") or "").lower().endswith(".js")
5949
+ for r in recs for i in r["compilations"]):
5950
+ lines += ["*JS pairs use a 2-dimensional distance (LOC + import Jaccard; no stdlib JS "
5951
+ "AST).*", ""]
5732
5952
  return "\n".join(lines)
5733
5953
 
5734
5954
 
@@ -5786,6 +6006,444 @@ def cmd_bench(args):
5786
6006
  sys.exit(0)
5787
6007
 
5788
6008
 
6009
+ _R2_DIR = "r2"
6010
+ _R2_SIGNAL = 0.5 # intra/inter below this: inter-compiler variance is real signal
6011
+ _R2_NOISE = 1.0 # intra/inter at/above this: same-model reruns differ as much as models
6012
+
6013
+
6014
+ def _r2_class(ratio):
6015
+ if ratio is None:
6016
+ return None
6017
+ if ratio < _R2_SIGNAL:
6018
+ return "SIGNAL"
6019
+ if ratio < _R2_NOISE:
6020
+ return "NOISY"
6021
+ return "NOISE"
6022
+
6023
+
6024
+ def _r2_entry(entry_dir, name):
6025
+ """Intra-model variance for ONE bench entry (ADR-027): for each model with a run-1 (top-level
6026
+ c-<model>) AND a run-2 (r2/c-<model>), the structural distance between the two runs via the
6027
+ SAME _struct_distance the bench uses between compilers, each run's oracle pass-rate, and
6028
+ whether the two runs agree on every oracle case. Entries without r2/ report absent, never 0."""
6029
+ ir_graph, ir_errors = _load_ir_at(os.path.join(entry_dir, IR_FILE))
6030
+ try:
6031
+ with open(os.path.join(entry_dir, "oracle", "ORACLE.json"), encoding="utf-8-sig") as fh:
6032
+ cases = (json.load(fh) or {}).get("cases") or []
6033
+ except (OSError, ValueError):
6034
+ cases = []
6035
+ rec = {"archetype": name, "has_r2": False, "models": [], "intra_mean": None,
6036
+ "inter": None, "intra_over_inter": None, "class": None, "reason": None}
6037
+ r2 = os.path.join(entry_dir, _R2_DIR)
6038
+ if not os.path.isdir(r2):
6039
+ rec["reason"] = "no r2/ directory -- intra-model variance not measured"
6040
+ return rec, False
6041
+ rec["has_r2"] = True
6042
+ if ir_graph is None or ir_errors or not cases:
6043
+ rec["reason"] = "entry incomplete (missing/invalid IR or oracle)"
6044
+ return rec, False
6045
+
6046
+ def load_run(cd):
6047
+ cj = os.path.join(cd, "COMPILATION.json")
6048
+ if not os.path.isfile(cj):
6049
+ return None
6050
+ errors, _adv = _validate_compilation(cj, ir_graph)
6051
+ try:
6052
+ with open(cj, encoding="utf-8-sig") as fh:
6053
+ c = json.load(fh)
6054
+ unit = _entry_unit(c.get("source") or [])
6055
+ model = (c.get("compilation_report") or {}).get("model")
6056
+ except (OSError, ValueError, AttributeError, IndexError):
6057
+ return None
6058
+ impl = os.path.join(cd, unit.replace("/", os.sep)) if unit else None
6059
+ if not impl or not os.path.isfile(impl):
6060
+ return None
6061
+ ores = _bench_oracle_all(impl, cases)
6062
+ m = _impl_metrics(impl)
6063
+ return {"model": model, "compile_valid": not errors, "oracle": ores,
6064
+ "metrics": m if "error" not in m else None,
6065
+ "case_vector": tuple(sorted(ores.get("failing") or []))}
6066
+
6067
+ def rate(o):
6068
+ return round(o["passed"] / o["total"], 3) if o["total"] else None
6069
+
6070
+ intra = []
6071
+ run1_metrics = []
6072
+ has_js = False # footer disclosure only; not serialized
6073
+ for d in sorted(os.listdir(entry_dir)):
6074
+ if not d.startswith("c-") or not os.path.isdir(os.path.join(entry_dir, d)):
6075
+ continue
6076
+ r1 = load_run(os.path.join(entry_dir, d))
6077
+ if r1 is None:
6078
+ continue
6079
+ if r1["metrics"]:
6080
+ run1_metrics.append(r1["metrics"])
6081
+ if r1["metrics"]["path"].lower().endswith(".js"):
6082
+ has_js = True
6083
+ r2run = load_run(os.path.join(r2, d))
6084
+ if r2run is None:
6085
+ rec["models"].append({"dir": d, "model": r1["model"], "r2": "absent"})
6086
+ continue
6087
+ if r2run["metrics"] and r2run["metrics"]["path"].lower().endswith(".js"):
6088
+ has_js = True
6089
+ dist = (_struct_distance(r1["metrics"], r2run["metrics"])
6090
+ if r1["metrics"] and r2run["metrics"] else None)
6091
+ stable = r1["case_vector"] == r2run["case_vector"]
6092
+ rec["models"].append({"dir": d, "model": r1["model"],
6093
+ "intra_distance": round(dist, 4) if dist is not None else None,
6094
+ "run1_passrate": rate(r1["oracle"]),
6095
+ "run2_passrate": rate(r2run["oracle"]),
6096
+ "run2_compile_valid": r2run["compile_valid"],
6097
+ "behaviour_stable": stable})
6098
+ if dist is not None:
6099
+ intra.append(dist)
6100
+ if intra:
6101
+ rec["intra_mean"] = round(sum(intra) / len(intra), 4)
6102
+ inter = []
6103
+ for i in range(len(run1_metrics)):
6104
+ for j in range(i + 1, len(run1_metrics)):
6105
+ inter.append(_struct_distance(run1_metrics[i], run1_metrics[j]))
6106
+ if inter:
6107
+ rec["inter"] = round(sum(inter) / len(inter), 4)
6108
+ if rec["intra_mean"] is not None and rec["inter"]:
6109
+ # 2 dp on purpose: the ratio moves by up to ~0.26 between interpreters (ast.walk
6110
+ # counts nodes differently on 3.8 vs 3.13 -- the 1.84.0 review measured it); more
6111
+ # decimals would print precision the measurement does not have (ADR-027).
6112
+ rec["intra_over_inter"] = round(rec["intra_mean"] / rec["inter"], 2)
6113
+ rec["class"] = _r2_class(rec["intra_over_inter"])
6114
+ elif rec["intra_mean"] is not None and rec["inter"] == 0.0:
6115
+ rec["reason"] = "inter-compiler distance is 0 (converged run-1) -- ratio undefined"
6116
+ elif rec["intra_mean"] is None:
6117
+ # r2/ exists but no model pair yielded a computable distance (unparseable source,
6118
+ # missing COMPILATION.json in every r2 model): absent, named -- never a silent None
6119
+ rec["reason"] = ("r2/ present but no model pair measurable (no parseable run-1+run-2 "
6120
+ "source for any model) -- intra-model variance not measured")
6121
+ return rec, has_js
6122
+
6123
+
6124
+ _R2_VERDICT_TEXT = {
6125
+ "SIGNAL": "Across the measured entries, same-model reruns differ far less than different "
6126
+ "models do: the inter-compiler variance the program reports is signal, not sampling "
6127
+ "noise -- for these archetypes, at n=2 per model.",
6128
+ "NOISY": "Same-model reruns differ by a sizeable fraction of the inter-compiler distance: "
6129
+ "inter-compiler variance carries information, but a delta smaller than the noise "
6130
+ "floor should not be read as an effect.",
6131
+ "NOISE": "Same-model reruns differ as much as different models do: the inter-compiler "
6132
+ "variance the program reports is dominated by sampling noise for these archetypes -- "
6133
+ "variance-based claims must be re-read.",
6134
+ "UNMEASURED": "No entry carries a second run; the noise floor is absent, not zero.",
6135
+ }
6136
+
6137
+
6138
+ def _render_r2_md(recs, agg, has_js=False):
6139
+ lines = ["<!-- GENERATED by qa_ledger.py bench-r2 (ADR-027) -- measured run; do not hand-edit. -->",
6140
+ "", "# DIAMOND-BENCH-R2 -- intra-model variance, the noise floor under the bench", "",
6141
+ "For every archetype with a second blind run (`r2/`) of the SAME model on the SAME "
6142
+ "canonical package: the structural distance between run 1 and run 2 (the same "
6143
+ "function the bench uses BETWEEN compilers), each run's oracle pass-rate, and "
6144
+ "whether both runs fail the same oracle cases. `intra/inter` below 0.5 reads SIGNAL "
6145
+ "(inter-compiler variance is at least twice the noise floor), 0.5-1.0 NOISY, at or "
6146
+ "above 1.0 NOISE (same-model reruns differ as much as different models). Advisory: "
6147
+ "no bench verdict changes; this qualifies every variance claim the program makes.", "",
6148
+ "| Archetype | intra (mean) | inter | intra/inter | class | per model (run1 -> run2 pass-rate, stable) |",
6149
+ "|-----------|--------------|-------|-------------|-------|--------------------------------------------|"]
6150
+ for r in recs:
6151
+ if not r["has_r2"]:
6152
+ lines.append("| %s | -- | -- | -- | (no r2) | %s |" % (r["archetype"], r["reason"] or ""))
6153
+ continue
6154
+ pm = "; ".join(("%s %s->%s %s" % (m["dir"], m.get("run1_passrate"), m.get("run2_passrate"),
6155
+ "stable" if m.get("behaviour_stable") else "DIFFERS"))
6156
+ if m.get("r2") != "absent" else ("%s (r2 absent)" % m["dir"])
6157
+ for m in r["models"])
6158
+ lines.append("| %s | %s | %s | %s | %s | %s |" % (
6159
+ r["archetype"],
6160
+ "%.4f" % r["intra_mean"] if r["intra_mean"] is not None else "--",
6161
+ "%.4f" % r["inter"] if r["inter"] is not None else "--",
6162
+ "%.2f" % r["intra_over_inter"] if r["intra_over_inter"] is not None else "--",
6163
+ r["class"] or (r["reason"] or "--"), pm))
6164
+ lines += ["", "**Aggregate:** %d entries measured -- SIGNAL %d · NOISY %d · NOISE %d · mean "
6165
+ "intra/inter %s · behaviour-stable reruns %d/%d." % (
6166
+ agg["measured"], agg["signal"], agg["noisy"], agg["noise"],
6167
+ "%.3f" % agg["mean_ratio"] if agg["mean_ratio"] is not None else "n/a",
6168
+ agg["stable"], agg["reruns"]),
6169
+ "", "## Verdict: %s" % agg["verdict"], "", _R2_VERDICT_TEXT[agg["verdict"]],
6170
+ "", "*Structural distance = mean of normalized LOC delta, AST-node delta and "
6171
+ "import-set Jaccard distance -- the one function shared by lang-compare, bench "
6172
+ "and bench-r2. n=2 per model is the minimum that yields a floor at all; it is a "
6173
+ "floor, not a distribution. The ratio is printed to 2 decimals on purpose: the "
6174
+ "AST-node count differs between Python versions (ast.walk on 3.8 vs 3.13), so a "
6175
+ "ratio can move by up to ~0.26 across interpreters without any code or model "
6176
+ "changing -- classes are stable across 3.8/3.13 on this bench, ratios are not "
6177
+ "point-precise, and an entry within ~0.25 of a threshold should be read as "
6178
+ "borderline.*", ""]
6179
+ if has_js:
6180
+ lines += ["*JS pairs use a 2-dimensional distance (LOC + import Jaccard; no stdlib JS "
6181
+ "AST).*", ""]
6182
+ return "\n".join(lines)
6183
+
6184
+
6185
+ def cmd_bench_r2(args):
6186
+ if not os.path.isdir(args.dir):
6187
+ print("[qa_ledger] bench-r2: no directory %s" % args.dir, file=sys.stderr)
6188
+ sys.exit(2)
6189
+ entries = sorted(d for d in os.listdir(args.dir)
6190
+ if os.path.isfile(os.path.join(args.dir, d, IR_FILE)))
6191
+ if not entries:
6192
+ print("[qa_ledger] bench-r2: no entries under %s" % args.dir, file=sys.stderr)
6193
+ sys.exit(2)
6194
+ pairs = [_r2_entry(os.path.join(args.dir, e), e) for e in entries]
6195
+ recs = [p[0] for p in pairs]
6196
+ has_js = any(p[1] for p in pairs)
6197
+ measured = [r for r in recs if r["class"] is not None]
6198
+ ratios = [r["intra_over_inter"] for r in measured]
6199
+ reruns = [m for r in recs if r["has_r2"] for m in r["models"] if m.get("r2") != "absent"]
6200
+ agg = {"entries": len(recs), "with_r2": sum(1 for r in recs if r["has_r2"]),
6201
+ "measured": len(measured),
6202
+ "signal": sum(1 for r in measured if r["class"] == "SIGNAL"),
6203
+ "noisy": sum(1 for r in measured if r["class"] == "NOISY"),
6204
+ "noise": sum(1 for r in measured if r["class"] == "NOISE"),
6205
+ "mean_ratio": round(sum(ratios) / len(ratios), 3) if ratios else None,
6206
+ "reruns": len(reruns),
6207
+ "stable": sum(1 for m in reruns if m.get("behaviour_stable"))}
6208
+ if not measured:
6209
+ agg["verdict"] = "UNMEASURED"
6210
+ else:
6211
+ counts = [(agg["signal"], "SIGNAL"), (agg["noisy"], "NOISY"), (agg["noise"], "NOISE")]
6212
+ top = max(c for c, _ in counts)
6213
+ # ties resolve toward the more cautious reading (NOISE > NOISY > SIGNAL)
6214
+ agg["verdict"] = [n for c, n in reversed(counts) if c == top][0]
6215
+ md = _render_r2_md(recs, agg, has_js)
6216
+ if args.out:
6217
+ with open(args.out, "w", encoding="utf-8", newline="\n") as fh:
6218
+ fh.write(md)
6219
+ if args.json:
6220
+ print(json.dumps({"aggregate": agg, "raw": recs}, indent=2, ensure_ascii=False))
6221
+ else:
6222
+ print("BENCH-R2: %d entries, %d with r2, %d measured -- SIGNAL %d / NOISY %d / NOISE %d"
6223
+ % (agg["entries"], agg["with_r2"], agg["measured"], agg["signal"], agg["noisy"],
6224
+ agg["noise"]))
6225
+ for r in recs:
6226
+ if r["class"]:
6227
+ print(" %-17s intra %.4f inter %.4f ratio %.2f %s"
6228
+ % (r["archetype"], r["intra_mean"], r["inter"], r["intra_over_inter"],
6229
+ r["class"]))
6230
+ elif r["has_r2"]:
6231
+ print(" %-17s %s" % (r["archetype"], r["reason"]))
6232
+ print("VERDICT: %s%s" % (agg["verdict"], (" -> %s" % args.out) if args.out else ""))
6233
+ print(" (ratios are not point-precise: AST-node counts differ across Python versions; "
6234
+ "classes stable, ratios +/-0.25 -- read borderline entries as borderline)")
6235
+ sys.exit(0)
6236
+
6237
+
6238
+ _RT_ID_RE = re.compile(r"(?i)\b(AC|INV|ADR)[-_][A-Z0-9-]*?\d+\b")
6239
+
6240
+
6241
+ def _rt_ids_in(text):
6242
+ """Every canonical id (AC-*, INV-*, ADR-*) literally referenced in a text, upper-cased.
6243
+ ID REFERENCE only (the ADR-013 rule) -- no fuzzy or semantic matching, ever."""
6244
+ out = set()
6245
+ for m in _RT_ID_RE.finditer(text or ""):
6246
+ out.add(m.group(0).upper().replace("_", "-"))
6247
+ return out
6248
+
6249
+
6250
+ def _rt_read_source(cd, unit):
6251
+ try:
6252
+ with open(os.path.join(cd, unit.replace("/", os.sep)), encoding="utf-8",
6253
+ errors="replace") as fh:
6254
+ return fh.read()
6255
+ except OSError:
6256
+ return ""
6257
+
6258
+
6259
+ def _rt_compilation(entry_dir, cd, ir_graph, cases):
6260
+ """Round-trip recoverability of ONE compilation against the pinned IR (ADR-030): for every
6261
+ IR node, whether the mechanical reverse organs find footing for it in the artifact --
6262
+ (a) static: a source unit's text or a static observation literally references the id;
6263
+ (b) manifest: the compiler's validated trace manifest maps the node to a unit that exists;
6264
+ (c) behaviour: for AC nodes, at least one withheld-oracle case tagged with the id passes
6265
+ (UNMEASURED when no case carries any AC tag). It regenerates NOTHING: no IR', no spec --
6266
+ a coverage over the human-authored IR, plus the list of nodes nothing anchors."""
6267
+ cj = os.path.join(cd, "COMPILATION.json")
6268
+ try:
6269
+ with open(cj, encoding="utf-8-sig") as fh:
6270
+ c = json.load(fh)
6271
+ except (OSError, ValueError):
6272
+ return None
6273
+ units = [u.get("unit") for u in (c.get("source") or []) if isinstance(u, dict) and u.get("unit")]
6274
+ units = [u for u in units if os.path.isfile(os.path.join(cd, u.replace("/", os.sep)))]
6275
+ node_ids = [n["id"] for n in (ir_graph.get("nodes") or [])]
6276
+ # (a) static footing: ids referenced in source text (comments/docstrings/names) or in
6277
+ # the static observations' statements/provenance
6278
+ static_ids = set()
6279
+ for u in units:
6280
+ static_ids |= _rt_ids_in(_rt_read_source(cd, u))
6281
+ so, _un = _static_surface_for(cd, u)
6282
+ for o in so:
6283
+ static_ids |= _rt_ids_in(o.get("statement", ""))
6284
+ # (b) manifest footing
6285
+ manifest_ids = set()
6286
+ for e in c.get("trace_manifest") or []:
6287
+ if e.get("unit") in units:
6288
+ for nid in e.get("implements") or []:
6289
+ manifest_ids.add(str(nid).upper())
6290
+ # (c) behaviour footing: oracle cases whose NAME carries an AC id (payload tags are not a
6291
+ # convention the bench oracles use today -- absence named, not faked)
6292
+ tagged = {}
6293
+ for case in cases:
6294
+ for cid in _rt_ids_in(case.get("name", "")):
6295
+ tagged.setdefault(cid, []).append(case)
6296
+ behaviour_measured = bool(tagged)
6297
+ behaviour_ids = set()
6298
+ entry_unit = _entry_unit(c.get("source") or [])
6299
+ impl = os.path.join(cd, entry_unit.replace("/", os.sep)) if entry_unit else None
6300
+ if behaviour_measured and impl and os.path.isfile(impl):
6301
+ for cid, cs in tagged.items():
6302
+ if any(_run_oracle_case(impl, case).get("ok") for case in cs):
6303
+ behaviour_ids.add(cid)
6304
+ per_node = []
6305
+ anchored = 0
6306
+ for nid in node_ids:
6307
+ a_s = nid in static_ids
6308
+ a_m = nid in manifest_ids
6309
+ a_b = (nid in behaviour_ids) if (behaviour_measured and nid.startswith("AC-")) else None
6310
+ anchored_any = a_s or a_m or bool(a_b)
6311
+ # the MEASURED footing excludes the manifest: the manifest is what the compiler
6312
+ # CLAIMED (and the prompt handed it the ids), so counting it as recovered would be
6313
+ # tautological -- the 1.000 that means nothing (found on the first run of this tool)
6314
+ anchored_meas = a_s or bool(a_b)
6315
+ anchored += 1 if anchored_meas else 0
6316
+ per_node.append({"id": nid, "static": a_s, "manifest": a_m,
6317
+ "behaviour": a_b, "anchored": anchored_meas,
6318
+ "claimed": anchored_any})
6319
+ edges = ir_graph.get("edges") or []
6320
+ anch_ids = {n["id"] for n in per_node if n["anchored"]}
6321
+ edges_recovered = sum(1 for e in edges if e.get("from") in anch_ids and e.get("to") in anch_ids)
6322
+ claimed = sum(1 for n in per_node if n["claimed"])
6323
+ return {"dir": os.path.basename(cd), "units": len(units),
6324
+ "ir_nodes": len(node_ids), "anchored": anchored, "claimed": claimed,
6325
+ "recoverability": round(anchored / len(node_ids), 3) if node_ids else None,
6326
+ "claimed_share": round(claimed / len(node_ids), 3) if node_ids else None,
6327
+ "static_anchored": sum(1 for n in per_node if n["static"]),
6328
+ "manifest_anchored": sum(1 for n in per_node if n["manifest"]),
6329
+ "behaviour": ("UNMEASURED" if not behaviour_measured
6330
+ else sum(1 for n in per_node if n["behaviour"])),
6331
+ "edges": len(edges), "edges_recovered": edges_recovered,
6332
+ "unanchored": [n["id"] for n in per_node if not n["anchored"]],
6333
+ "nodes": per_node}
6334
+
6335
+
6336
+ def _rt_entry(entry_dir, name):
6337
+ ir_graph, ir_errors = _load_ir_at(os.path.join(entry_dir, IR_FILE))
6338
+ try:
6339
+ with open(os.path.join(entry_dir, "oracle", "ORACLE.json"), encoding="utf-8-sig") as fh:
6340
+ cases = (json.load(fh) or {}).get("cases") or []
6341
+ except (OSError, ValueError):
6342
+ cases = []
6343
+ rec = {"archetype": name, "compilations": [], "ir_nodes": None, "edges": None,
6344
+ "recoverability_mean": None, "edges_recovered_mean": None, "reason": None}
6345
+ if ir_graph is None or ir_errors:
6346
+ rec["reason"] = "entry incomplete (missing/invalid IR)"
6347
+ return rec
6348
+ rec["ir_nodes"] = len(ir_graph.get("nodes") or [])
6349
+ rec["edges"] = len(ir_graph.get("edges") or [])
6350
+ for d in sorted(os.listdir(entry_dir)):
6351
+ cd = os.path.join(entry_dir, d)
6352
+ if d.startswith("c-") and os.path.isfile(os.path.join(cd, "COMPILATION.json")):
6353
+ r = _rt_compilation(entry_dir, cd, ir_graph, cases)
6354
+ if r:
6355
+ rec["compilations"].append(r)
6356
+ recs = [c["recoverability"] for c in rec["compilations"] if c["recoverability"] is not None]
6357
+ if recs:
6358
+ rec["recoverability_mean"] = round(sum(recs) / len(recs), 3)
6359
+ rec["edges_recovered_mean"] = round(
6360
+ sum(c["edges_recovered"] for c in rec["compilations"]) / len(rec["compilations"]), 2)
6361
+ else:
6362
+ rec["reason"] = "no compilations yet"
6363
+ return rec
6364
+
6365
+
6366
+ def _render_rt_md(recs, agg):
6367
+ lines = ["<!-- GENERATED by qa_ledger.py bench-roundtrip (ADR-030) -- measured run; do not hand-edit. -->",
6368
+ "", "# DIAMOND-ROUNDTRIP -- how much of the pinned IR the reverse organs can anchor in each compiled artifact", "",
6369
+ "**What this is NOT:** it does not regenerate an IR from code, it does not diff specs, "
6370
+ "it does not infer requirements. Reverse discovery produces FACTS (ADR-013); the human "
6371
+ "authors the spec. This instrument joins those facts against the human-authored IR "
6372
+ "and reports, per compilation, which nodes have footing -- (a) static: the id is "
6373
+ "literally referenced in a source unit or a static observation; (b) manifest: the "
6374
+ "compiler's validated trace manifest maps the node to a unit that exists; (c) "
6375
+ "behaviour: a withheld-oracle case tagged with the AC id passes (UNMEASURED where "
6376
+ "the entry's oracle cases carry no AC tag -- absence named). **`recoverability` "
6377
+ "counts ONLY static + behaviour footing** -- the manifest is what the compiler "
6378
+ "CLAIMED (and the blind prompt handed it the node ids), so it is reported apart as "
6379
+ "`claimed` and never counted as recovered: the first run of this instrument read "
6380
+ "1.000 on every entry for exactly that reason. `edges recovered` = edges with both "
6381
+ "endpoints MEASURED-anchored. Advisory: no bench verdict changes. The unanchored "
6382
+ "list IS the honest gap, per compiler.", "",
6383
+ "| Archetype | IR nodes | edges | recoverability (mean, measured) | edges recovered (mean) | per compilation (measured/nodes · static · behaviour · claimed-by-manifest · unanchored) |",
6384
+ "|-----------|----------|-------|---------------------------------|------------------------|--------------------------------------------------------------------------------------------|"]
6385
+ for r in recs:
6386
+ if r["recoverability_mean"] is None:
6387
+ lines.append("| %s | %s | %s | -- | -- | %s |" % (r["archetype"], r["ir_nodes"], r["edges"], r["reason"] or ""))
6388
+ continue
6389
+ pc = "; ".join("%s %d/%d · s%d · b%s · claimed %d · [%s]" % (
6390
+ c["dir"], c["anchored"], c["ir_nodes"], c["static_anchored"],
6391
+ c["behaviour"], c["manifest_anchored"], ",".join(c["unanchored"]) or "none")
6392
+ for c in r["compilations"])
6393
+ lines.append("| %s | %d | %d | %.3f | %.2f | %s |" % (
6394
+ r["archetype"], r["ir_nodes"], r["edges"], r["recoverability_mean"],
6395
+ r["edges_recovered_mean"], pc))
6396
+ lines += ["", "**Aggregate:** %d entries measured · mean recoverability %s · entries with edges %d "
6397
+ "· behaviour dimension measured in %d/%d entries." % (
6398
+ agg["measured"], "%.3f" % agg["mean_recoverability"] if agg["mean_recoverability"] is not None else "n/a",
6399
+ agg["with_edges"], agg["behaviour_measured"], agg["measured"]),
6400
+ "", "*The manifest dimension is what the compiler CLAIMED (validated for shape, "
6401
+ "not truth) and is excluded from recoverability; the static dimension is what the "
6402
+ "artifact literally names; the behaviour dimension is what the withheld oracle can "
6403
+ "attribute per AC -- and it is UNMEASURED wherever oracle cases are not tagged with "
6404
+ "AC ids, which today is every entry: the honest state of reverse discovery is that "
6405
+ "it anchors names, not semantics, until oracles carry per-AC tags. None of the three "
6406
+ "is a spec.*", ""]
6407
+ return "\n".join(lines)
6408
+
6409
+
6410
+ def cmd_bench_roundtrip(args):
6411
+ if not os.path.isdir(args.dir):
6412
+ print("[qa_ledger] bench-roundtrip: no directory %s" % args.dir, file=sys.stderr)
6413
+ sys.exit(2)
6414
+ entries = sorted(d for d in os.listdir(args.dir)
6415
+ if os.path.isfile(os.path.join(args.dir, d, IR_FILE)))
6416
+ if not entries:
6417
+ print("[qa_ledger] bench-roundtrip: no entries under %s" % args.dir, file=sys.stderr)
6418
+ sys.exit(2)
6419
+ recs = [_rt_entry(os.path.join(args.dir, e), e) for e in entries]
6420
+ measured = [r for r in recs if r["recoverability_mean"] is not None]
6421
+ agg = {"entries": len(recs), "measured": len(measured),
6422
+ "mean_recoverability": (round(sum(r["recoverability_mean"] for r in measured) / len(measured), 3)
6423
+ if measured else None),
6424
+ "with_edges": sum(1 for r in measured if (r["edges"] or 0) > 0),
6425
+ "behaviour_measured": sum(1 for r in measured
6426
+ if any(c["behaviour"] != "UNMEASURED" for c in r["compilations"]))}
6427
+ md = _render_rt_md(recs, agg)
6428
+ if args.out:
6429
+ with open(args.out, "w", encoding="utf-8", newline="\n") as fh:
6430
+ fh.write(md)
6431
+ if args.json:
6432
+ print(json.dumps({"aggregate": agg, "raw": recs}, indent=2, ensure_ascii=False))
6433
+ else:
6434
+ print("BENCH-ROUNDTRIP: %d entries, %d measured -- mean recoverability %s, %d with edges, "
6435
+ "behaviour measured in %d" % (agg["entries"], agg["measured"],
6436
+ "%.3f" % agg["mean_recoverability"] if agg["mean_recoverability"] is not None else "n/a",
6437
+ agg["with_edges"], agg["behaviour_measured"]))
6438
+ for r in recs:
6439
+ if r["recoverability_mean"] is not None:
6440
+ print(" %-17s nodes %2d edges %d recover %.3f edges-rec %.2f" % (
6441
+ r["archetype"], r["ir_nodes"], r["edges"], r["recoverability_mean"], r["edges_recovered_mean"]))
6442
+ if args.out:
6443
+ print(" -> %s" % args.out)
6444
+ sys.exit(0)
6445
+
6446
+
5789
6447
  def _bench_curable_obs(bench_dir, entry, cdir):
5790
6448
  """The curable set for one compilation: the SAME observations the M1 static extractor
5791
6449
  produces for the descriptor's static_surface, re-extracted at call time -- a verdict
@@ -5802,13 +6460,20 @@ def _bench_curable_obs(bench_dir, entry, cdir):
5802
6460
  with open(cj, encoding="utf-8-sig") as fh:
5803
6461
  c = json.load(fh)
5804
6462
  src = c.get("source") or []
5805
- unit = src[0].get("unit") if src else None
6463
+ units = [u.get("unit") for u in src if isinstance(u, dict) and u.get("unit")]
5806
6464
  except (OSError, ValueError, AttributeError, IndexError):
5807
- unit = None
5808
- if not unit or not os.path.isfile(os.path.join(cd, unit.replace("/", os.sep))):
6465
+ units = []
6466
+ units = [u for u in units if os.path.isfile(os.path.join(cd, u.replace("/", os.sep)))]
6467
+ if not units:
5809
6468
  return None, ("%s/%s: no resolvable source unit -- nothing to extract a surface "
5810
6469
  "from" % (entry, cdir))
5811
- sobs, _uns = _extract_static_py(cd, [unit])
6470
+ sobs, uns = [], []
6471
+ for u in units:
6472
+ so, un = _static_surface_for(cd, u)
6473
+ sobs.extend(so)
6474
+ uns.extend(un)
6475
+ if not sobs and uns:
6476
+ return None, ("%s/%s: static surface UNMEASURED -- %s" % (entry, cdir, "; ".join(uns)))
5812
6477
  return sobs, None
5813
6478
 
5814
6479
 
@@ -5895,7 +6560,9 @@ _LANG_PR_MARGIN = 0.02 # mean oracle pass-rate drop
5895
6560
 
5896
6561
  def _lang_arm_metrics(arm_dir):
5897
6562
  """Measure ONE arm: per-compiler oracle green + unresolved_intent, and inter-compiler
5898
- variance over the arm's compilations. Reuses the M4/M5 organs unchanged."""
6563
+ variance over the arm's compilations. Reuses the M4/M5 organs unchanged. Returns
6564
+ (rec, errors, has_js) -- has_js is a footer-only signal for _render_lang_md (ADR-028);
6565
+ it is never part of `rec`, so lang-compare's JSON output is untouched by it."""
5899
6566
  ir_graph, ir_errors = _load_ir_at(os.path.join(arm_dir, IR_FILE))
5900
6567
  try:
5901
6568
  with open(os.path.join(arm_dir, "oracle", "ORACLE.json"), encoding="utf-8-sig") as fh:
@@ -5907,7 +6574,7 @@ def _lang_arm_metrics(arm_dir):
5907
6574
  "mean_passrate": None, "ui_count": 0.0, "ui_distinct_regions": 0.0,
5908
6575
  "ui_rationale_len": 0.0}
5909
6576
  if ir_graph is None or ir_errors or not cases:
5910
- return rec, ["arm incomplete (missing/invalid IR or oracle)"]
6577
+ return rec, ["arm incomplete (missing/invalid IR or oracle)"], False
5911
6578
  comp_dirs = sorted(d for d in os.listdir(arm_dir)
5912
6579
  if d.startswith("c-") and
5913
6580
  os.path.isfile(os.path.join(arm_dir, d, "COMPILATION.json")))
@@ -5921,7 +6588,7 @@ def _lang_arm_metrics(arm_dir):
5921
6588
  with open(cj, encoding="utf-8-sig") as fh:
5922
6589
  c = json.load(fh)
5923
6590
  src = c.get("source") or []
5924
- unit = src[0].get("unit") if src else None
6591
+ unit = _entry_unit(src)
5925
6592
  model = (c.get("compilation_report") or {}).get("model")
5926
6593
  uis = c.get("unresolved_intent") or []
5927
6594
  except (OSError, ValueError, AttributeError, IndexError):
@@ -5954,14 +6621,29 @@ def _lang_arm_metrics(arm_dir):
5954
6621
  dists = []
5955
6622
  for i in range(len(good)):
5956
6623
  for j in range(i + 1, len(good)):
5957
- a, b = good[i], good[j]
5958
- ia, ib = set(a["imports"]), set(b["imports"])
5959
- jac = (len(ia & ib) / len(ia | ib)) if (ia or ib) else 1.0
5960
- dloc = abs(a["loc"] - b["loc"]) / max(a["loc"], b["loc"], 1)
5961
- dast = abs(a["ast_nodes"] - b["ast_nodes"]) / max(a["ast_nodes"], b["ast_nodes"], 1)
5962
- dists.append((dloc + dast + (1.0 - jac)) / 3.0)
6624
+ dists.append(_struct_distance(good[i], good[j]))
5963
6625
  rec["variance_score"] = round(sum(dists) / len(dists), 4) if dists else None
5964
- return rec, []
6626
+ has_js = any(p.lower().endswith(".js") for p in impl_paths)
6627
+ return rec, [], has_js
6628
+
6629
+
6630
+ def _struct_distance(a, b):
6631
+ """The ONE structural distance the program uses between two implementations -- mean of
6632
+ normalized LOC delta, AST-node delta and import-set Jaccard distance (0 = identical). Shared
6633
+ by the inter-compiler variance (lang-compare, bench) and the intra-model variance (bench-r2,
6634
+ ADR-027) so their ratio is commensurable: same function, both sides.
6635
+
6636
+ ADR-028: a `.js` metrics dict has `ast_nodes: None` (no stdlib JS AST) -- when either side
6637
+ lacks it, the distance is the mean over the dimensions BOTH sides have (LOC + import
6638
+ Jaccard only). A cross-language pair never occurs (one archetype, one language), so this
6639
+ never mixes a Python 3-dimensional distance with a JS 2-dimensional one within a pair."""
6640
+ ia, ib = set(a["imports"]), set(b["imports"])
6641
+ jac = (len(ia & ib) / len(ia | ib)) if (ia or ib) else 1.0
6642
+ dloc = abs(a["loc"] - b["loc"]) / max(a["loc"], b["loc"], 1)
6643
+ if a.get("ast_nodes") is None or b.get("ast_nodes") is None:
6644
+ return (dloc + (1.0 - jac)) / 2.0
6645
+ dast = abs(a["ast_nodes"] - b["ast_nodes"]) / max(a["ast_nodes"], b["ast_nodes"], 1)
6646
+ return (dloc + dast + (1.0 - jac)) / 3.0
5965
6647
 
5966
6648
 
5967
6649
  def _oracle_hash(arm_dir):
@@ -5987,8 +6669,9 @@ def cmd_lang_compare(args):
5987
6669
  "the comparison requires the SAME withheld oracle; behaviour must be held fixed "
5988
6670
  "while only the authoring changes." % (hf[:12], hc[:12]), file=sys.stderr)
5989
6671
  sys.exit(2)
5990
- free, ef = _lang_arm_metrics(args.free)
5991
- ctrl, ec = _lang_arm_metrics(args.controlled)
6672
+ free, ef, has_js_f = _lang_arm_metrics(args.free)
6673
+ ctrl, ec, has_js_c = _lang_arm_metrics(args.controlled)
6674
+ has_js = has_js_f or has_js_c
5992
6675
  if ef or ec:
5993
6676
  for e in ef + ec:
5994
6677
  print("[qa_ledger] lang-compare: %s" % e, file=sys.stderr)
@@ -6027,7 +6710,7 @@ def cmd_lang_compare(args):
6027
6710
  "margin": _LANG_MARGIN, "passrate_margin": _LANG_PR_MARGIN, "oracle_shared": True}
6028
6711
  if args.out:
6029
6712
  with open(args.out, "w", encoding="utf-8", newline="\n") as fh:
6030
- fh.write(_render_lang_md(report))
6713
+ fh.write(_render_lang_md(report, has_js))
6031
6714
  if args.json:
6032
6715
  print(json.dumps(report, indent=2, ensure_ascii=False))
6033
6716
  else:
@@ -6051,7 +6734,7 @@ def cmd_lang_compare(args):
6051
6734
  sys.exit(0)
6052
6735
 
6053
6736
 
6054
- def _render_lang_md(r):
6737
+ def _render_lang_md(r, has_js=False):
6055
6738
  d = r["delta"]
6056
6739
  lines = ["<!-- GENERATED by qa_ledger.py lang-compare (ADR-019) -- measured run; do not "
6057
6740
  "hand-edit. -->", "", "# CONTROLLED-LANGUAGE-REPORT", "",
@@ -6093,11 +6776,16 @@ def _render_lang_md(r):
6093
6776
  "arm's low variance was convergence on a shared reading (right or "
6094
6777
  "wrong): two compilers resolving the same ambiguity the same way read "
6095
6778
  "as agreement, and a rewrite that separates them raises variance "
6096
- "while moving behaviour."
6779
+ "while moving behaviour. Read any variance number here against the "
6780
+ "entry's intra-model noise floor (bench-r2, ADR-027): an inter-compiler "
6781
+ "distance below that floor is not evidence of convergence."
6097
6782
  % r.get("passrate_margin", 0.02)}[r["verdict"]],
6098
6783
  "", "*The oracle is byte-identical across both arms, so behaviour is held fixed; "
6099
6784
  "the only variable is the authoring discipline. The judgement of \"same semantic "
6100
6785
  "content\" between the two canonical packages is human — a stated limitation.*", ""]
6786
+ if has_js:
6787
+ lines += ["*JS pairs use a 2-dimensional distance (LOC + import Jaccard; no stdlib JS "
6788
+ "AST).*", ""]
6101
6789
  return "\n".join(lines)
6102
6790
 
6103
6791
 
@@ -10246,6 +10934,28 @@ def build_parser():
10246
10934
  "(read-only), including stale verdicts whose obs no longer exists")
10247
10935
  pbc.set_defaults(func=cmd_bench_curate)
10248
10936
 
10937
+ pbr = sub.add_parser(
10938
+ "bench-r2",
10939
+ help="intra-model variance (ADR-027): for every bench entry with an r2/ second run of "
10940
+ "the same models, the run1-vs-run2 structural distance via the SAME function the "
10941
+ "bench uses between compilers, oracle stability, and a per-entry SIGNAL/NOISY/NOISE "
10942
+ "class; the noise floor under every variance claim; advisory, deterministic, no LLM")
10943
+ pbr.add_argument("--dir", required=True, help="the bench directory")
10944
+ pbr.add_argument("--out", default=None, help="write DIAMOND-BENCH-R2.md here")
10945
+ pbr.add_argument("--json", action="store_true")
10946
+ pbr.set_defaults(func=cmd_bench_r2)
10947
+
10948
+ prt = sub.add_parser(
10949
+ "bench-roundtrip",
10950
+ help="round-trip recoverability (ADR-030): per compilation, which pinned-IR nodes the "
10951
+ "mechanical reverse organs can anchor in the compiled artifact -- static id "
10952
+ "reference, validated manifest claim, tagged withheld-oracle behaviour -- and the "
10953
+ "unanchored gap list; regenerates NO IR, infers NO spec; advisory, deterministic")
10954
+ prt.add_argument("--dir", required=True, help="the bench directory")
10955
+ prt.add_argument("--out", default=None, help="write DIAMOND-ROUNDTRIP.md here")
10956
+ prt.add_argument("--json", action="store_true")
10957
+ prt.set_defaults(func=cmd_bench_roundtrip)
10958
+
10249
10959
  plc = sub.add_parser(
10250
10960
  "lang-compare",
10251
10961
  help="controlled-language arm (ADR-019): compare a FREE-prose arm and an EARS+STE arm of "