@andresmassello/uscha 1.83.0 → 1.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +1 -1
- package/uscha-kit/.claude/skills/uscha-devloop/qa_ledger.py +760 -50
- package/uscha-kit/.claude-plugin/plugin.json +2 -2
- package/uscha-kit/.codex-plugin/plugin.json +1 -1
- package/uscha-kit/README.md +1 -1
- package/uscha-kit/VERSION +1 -1
- package/uscha-kit/reports/junit/.js-cases.json +1 -0
- package/uscha-kit/reports/junit/.multi-cases.json +1 -0
- package/uscha-kit/reports/junit/.r2-cases.json +1 -0
- package/uscha-kit/reports/junit/.rt-cases.json +1 -0
- package/uscha-kit/skills/uscha-devloop/qa_ledger.py +760 -50
- package/uscha-kit/uscha.config.json +1 -1
|
@@ -4007,6 +4007,86 @@ def _extract_static_py(repo_path, tracked):
|
|
|
4007
4007
|
return obs, unsupported
|
|
4008
4008
|
|
|
4009
4009
|
|
|
4010
|
+
# Node one-liner (ADR-028): `require`s the impl and reports ITS OWN module.exports --
|
|
4011
|
+
# the target's own runtime reporting its own public surface, never a regex-narrated AST. A
|
|
4012
|
+
# module that guards its entry point with `require.main === module` (the SPEC contract) will
|
|
4013
|
+
# not auto-run under `require()`, since `require.main` here is this -e script, not the module.
|
|
4014
|
+
_NODE_EXPORTS_PROBE = (
|
|
4015
|
+
"const p=process.argv[1]; const m=require(require('path').resolve(p)); "
|
|
4016
|
+
"const ks=Object.keys(m).filter(k=>typeof m[k]==='function'||typeof m[k]==='object'); "
|
|
4017
|
+
"process.stdout.write(JSON.stringify(ks.sort()))"
|
|
4018
|
+
)
|
|
4019
|
+
_STATIC_JS_TIMEOUT = 15
|
|
4020
|
+
|
|
4021
|
+
|
|
4022
|
+
def _extract_static_js(repo_path, tracked):
|
|
4023
|
+
"""v0 static extractor for JS (ADR-028): the target's OWN runtime reports its own public
|
|
4024
|
+
surface via `node -e` + `Object.keys(module.exports)` -- measured, not parsed by a
|
|
4025
|
+
heuristic. Function-vs-object and the observation's line number are best-effort source
|
|
4026
|
+
regexes (Node reports WHICH names are exported; it does not report where they are
|
|
4027
|
+
defined). If node is absent, or a file fails to load/print a clean JSON array, that file's
|
|
4028
|
+
surface is UNMEASURED -- named in `unsupported`, never silently empty-as-measured.
|
|
4029
|
+
Returns (observations, unsupported) where unsupported is a list of "<file>: <reason>"."""
|
|
4030
|
+
obs, unsupported = [], []
|
|
4031
|
+
js_files = sorted(rel for rel in tracked if rel.lower().endswith(".js"))
|
|
4032
|
+
if not js_files:
|
|
4033
|
+
return obs, unsupported
|
|
4034
|
+
node = shutil.which("node")
|
|
4035
|
+
if not node:
|
|
4036
|
+
return obs, ["%s: node not on PATH" % rel for rel in js_files]
|
|
4037
|
+
for rel in js_files:
|
|
4038
|
+
full = os.path.join(repo_path, rel.replace("/", os.sep))
|
|
4039
|
+
abs_path = os.path.abspath(full)
|
|
4040
|
+
try:
|
|
4041
|
+
r = subprocess.run([node, "-e", _NODE_EXPORTS_PROBE, "--", abs_path],
|
|
4042
|
+
capture_output=True, text=True, encoding="utf-8",
|
|
4043
|
+
errors="replace", timeout=_STATIC_JS_TIMEOUT,
|
|
4044
|
+
cwd=os.path.dirname(full))
|
|
4045
|
+
except subprocess.TimeoutExpired:
|
|
4046
|
+
unsupported.append("%s: node timed out extracting exports" % rel)
|
|
4047
|
+
continue
|
|
4048
|
+
except OSError as exc:
|
|
4049
|
+
unsupported.append("%s: could not run node: %s" % (rel, exc))
|
|
4050
|
+
continue
|
|
4051
|
+
if r.returncode != 0:
|
|
4052
|
+
unsupported.append("%s: node exited %d extracting exports (%s)"
|
|
4053
|
+
% (rel, r.returncode, (r.stderr or "").strip()[:200]))
|
|
4054
|
+
continue
|
|
4055
|
+
try:
|
|
4056
|
+
names = json.loads(r.stdout)
|
|
4057
|
+
if not isinstance(names, list):
|
|
4058
|
+
raise ValueError("exports output was not a JSON list")
|
|
4059
|
+
except ValueError as exc:
|
|
4060
|
+
unsupported.append("%s: could not parse exports output: %s" % (rel, exc))
|
|
4061
|
+
continue
|
|
4062
|
+
try:
|
|
4063
|
+
with open(full, encoding="utf-8", errors="replace") as fh:
|
|
4064
|
+
src = fh.read()
|
|
4065
|
+
except OSError:
|
|
4066
|
+
src = ""
|
|
4067
|
+
src_lines = src.splitlines()
|
|
4068
|
+
for name in sorted(names):
|
|
4069
|
+
is_function = bool(re.search(r"\bfunction\s+%s\s*\(" % re.escape(name), src))
|
|
4070
|
+
lineno = 0
|
|
4071
|
+
patterns = (r"\bfunction\s+%s\b" % re.escape(name),
|
|
4072
|
+
r"\b%s\s*=" % re.escape(name),
|
|
4073
|
+
r"exports\.%s\b" % re.escape(name),
|
|
4074
|
+
r"module\.exports\b")
|
|
4075
|
+
for i, ln in enumerate(src_lines, 1):
|
|
4076
|
+
if any(re.search(p, ln) for p in patterns):
|
|
4077
|
+
lineno = i
|
|
4078
|
+
break
|
|
4079
|
+
stmt = ("%s exports function %s" % (rel, name) if is_function
|
|
4080
|
+
else "%s exports object %s" % (rel, name))
|
|
4081
|
+
prov = "%s:%d" % (rel, lineno)
|
|
4082
|
+
obs.append({"id": _obs_id("contract", stmt, prov), "type": "contract",
|
|
4083
|
+
"statement": stmt, "evidence_class": "static",
|
|
4084
|
+
"provenance": {"files": [prov],
|
|
4085
|
+
"derivation": "node -e Object.keys(module.exports)",
|
|
4086
|
+
"tool": "qa_ledger-static-js"}})
|
|
4087
|
+
return obs, unsupported
|
|
4088
|
+
|
|
4089
|
+
|
|
4010
4090
|
def _under_bound(rel, bound):
|
|
4011
4091
|
return bound is None or rel == bound or rel.startswith(bound + "/")
|
|
4012
4092
|
|
|
@@ -5307,26 +5387,69 @@ def cmd_compile_ingest(args):
|
|
|
5307
5387
|
_BOOTSTRAP_CASE_TIMEOUT = 15 # a compiled hook must decide fast
|
|
5308
5388
|
|
|
5309
5389
|
|
|
5390
|
+
def _entry_unit(source_list):
|
|
5391
|
+
"""The unit the oracle runs (ADR-029): the one whose basename starts with `cli.` if any,
|
|
5392
|
+
else the first declared source unit. Single-unit compilations resolve to their only unit,
|
|
5393
|
+
unchanged. Returns None when there is no usable unit."""
|
|
5394
|
+
units = [u.get("unit") for u in (source_list or []) if isinstance(u, dict) and u.get("unit")]
|
|
5395
|
+
if not units:
|
|
5396
|
+
return None
|
|
5397
|
+
for u in units:
|
|
5398
|
+
if os.path.basename(u).lower().startswith("cli."):
|
|
5399
|
+
return u
|
|
5400
|
+
return units[0]
|
|
5401
|
+
|
|
5402
|
+
|
|
5403
|
+
def _static_surface_for(cd, unit):
|
|
5404
|
+
"""Route the reverse-discovery organ by the unit's extension (ADR-028/029)."""
|
|
5405
|
+
if os.path.splitext(unit)[1].lower() == ".js":
|
|
5406
|
+
return _extract_static_js(cd, [unit])
|
|
5407
|
+
so, _n = _extract_static_py(cd, [unit])
|
|
5408
|
+
return so, []
|
|
5409
|
+
|
|
5410
|
+
|
|
5411
|
+
def _impl_interpreter(impl_path):
|
|
5412
|
+
"""Resolve the interpreter argv prefix for one implementation file by extension (ADR-028):
|
|
5413
|
+
`.py` runs under this same Python (unchanged); `.js` runs under `node`, resolved from PATH.
|
|
5414
|
+
Returns None when the extension's interpreter cannot be resolved (today: node absent) --
|
|
5415
|
+
the caller must treat that as UNMEASURED, never a fake red or green."""
|
|
5416
|
+
if impl_path.lower().endswith(".js"):
|
|
5417
|
+
node = shutil.which("node")
|
|
5418
|
+
return [node] if node else None
|
|
5419
|
+
return [sys.executable]
|
|
5420
|
+
|
|
5421
|
+
|
|
5310
5422
|
def _run_oracle_case(impl_path, case):
|
|
5311
5423
|
"""Run ONE withheld oracle case against a compiled implementation: feed the case's stdin
|
|
5312
|
-
(raw_stdin verbatim if present, else json.dumps(payload)) to
|
|
5313
|
-
|
|
5314
|
-
oracle is a `measured` fact, never an LLM
|
|
5424
|
+
(raw_stdin verbatim if present, else json.dumps(payload)) to the impl's interpreter (ADR-028:
|
|
5425
|
+
`python` for `.py`, `node` for `.js`), and check its result against whichever expectations
|
|
5426
|
+
the case declares. Deterministic execution -- the oracle is a `measured` fact, never an LLM
|
|
5427
|
+
judgment. Returns a per-case result dict.
|
|
5315
5428
|
|
|
5316
5429
|
A case may assert any of: `expected_exit` (process exit code -- the M4 guard's contract),
|
|
5317
5430
|
`expected_stdout` (the program's stdout, compared stripped -- for archetypes that COMPUTE an
|
|
5318
5431
|
output, e.g. a parser or transformer), and `expected_json` (stdout parsed as JSON and
|
|
5319
5432
|
compared structurally, so an output whose formatting is free but whose value is fixed is not
|
|
5320
5433
|
penalised for whitespace or key order). The case passes iff EVERY declared expectation holds;
|
|
5321
|
-
a case that declares none proves nothing and fails.
|
|
5434
|
+
a case that declares none proves nothing and fails.
|
|
5435
|
+
|
|
5436
|
+
If the impl's interpreter cannot be resolved (a `.js` impl and no `node` on PATH), the case
|
|
5437
|
+
is UNMEASURED -- never a fake red or green (ADR-028)."""
|
|
5322
5438
|
if "raw_stdin" in case:
|
|
5323
5439
|
stdin = case["raw_stdin"]
|
|
5324
5440
|
else:
|
|
5325
5441
|
stdin = json.dumps(case.get("payload"))
|
|
5442
|
+
interp = _impl_interpreter(impl_path)
|
|
5443
|
+
if interp is None:
|
|
5444
|
+
return {"name": case.get("name"), "expected": case.get("expected_exit"), "got": None,
|
|
5445
|
+
"ok": False, "error": "node not on PATH", "unmeasured": True}
|
|
5326
5446
|
try:
|
|
5327
|
-
|
|
5328
|
-
|
|
5329
|
-
|
|
5447
|
+
# cwd = the impl's own directory (ADR-029): a multi-unit compilation imports its
|
|
5448
|
+
# sibling modules by bare name; single-unit impls are unaffected by their cwd
|
|
5449
|
+
r = subprocess.run(interp + [os.path.abspath(impl_path)], input=stdin,
|
|
5450
|
+
capture_output=True, text=True, encoding="utf-8", errors="replace",
|
|
5451
|
+
timeout=_BOOTSTRAP_CASE_TIMEOUT,
|
|
5452
|
+
cwd=os.path.dirname(os.path.abspath(impl_path)) or None)
|
|
5330
5453
|
got, out, err = r.returncode, r.stdout, None
|
|
5331
5454
|
except subprocess.TimeoutExpired:
|
|
5332
5455
|
got, out, err = None, "", "timeout"
|
|
@@ -5369,22 +5492,30 @@ def cmd_bootstrap_oracle(args):
|
|
|
5369
5492
|
file=sys.stderr)
|
|
5370
5493
|
sys.exit(2)
|
|
5371
5494
|
results = [_run_oracle_case(args.impl, c) for c in cases]
|
|
5495
|
+
unmeasured = bool(results) and all(r.get("unmeasured") for r in results)
|
|
5372
5496
|
passed = sum(1 for r in results if r["ok"])
|
|
5373
5497
|
failed = [r for r in results if not r["ok"]]
|
|
5374
5498
|
report = {"impl": args.impl, "oracle": args.oracle, "total": len(results),
|
|
5375
5499
|
"passed": passed, "failed": len(failed),
|
|
5376
|
-
"oracle_green": not failed, "results": results}
|
|
5500
|
+
"oracle_green": (None if unmeasured else not failed), "results": results}
|
|
5501
|
+
if unmeasured:
|
|
5502
|
+
report["unmeasured"] = results[0]["error"]
|
|
5377
5503
|
if args.ledger and args.repo:
|
|
5378
5504
|
ledger = _load(args.ledger)
|
|
5379
5505
|
_repo_node(ledger, args.repo)
|
|
5380
5506
|
rec = {"impl": os.path.basename(args.impl), "oracle": os.path.basename(args.oracle),
|
|
5381
5507
|
"total": len(results), "passed": passed, "failed": len(failed),
|
|
5382
|
-
"oracle_green": not failed,
|
|
5508
|
+
"oracle_green": (None if unmeasured else not failed),
|
|
5383
5509
|
"failing": [r["name"] for r in failed], "at": _now()}
|
|
5510
|
+
if unmeasured:
|
|
5511
|
+
rec["unmeasured"] = results[0]["error"]
|
|
5384
5512
|
ledger.setdefault("bootstrap_oracle", []).append(rec)
|
|
5385
5513
|
_save(args.ledger, ledger)
|
|
5386
5514
|
if args.json:
|
|
5387
5515
|
print(json.dumps(report, indent=2, ensure_ascii=False))
|
|
5516
|
+
elif unmeasured:
|
|
5517
|
+
print("BOOTSTRAP-ORACLE %s: UNMEASURED -- %s"
|
|
5518
|
+
% (os.path.basename(args.impl), report["unmeasured"]))
|
|
5388
5519
|
else:
|
|
5389
5520
|
print("BOOTSTRAP-ORACLE %s: %d/%d cases pass -- %s"
|
|
5390
5521
|
% (os.path.basename(args.impl), passed, len(results),
|
|
@@ -5394,17 +5525,60 @@ def cmd_bootstrap_oracle(args):
|
|
|
5394
5525
|
print(" x %s: expected exit %s, got %s%s"
|
|
5395
5526
|
% (r["name"], r["expected"], r["got"],
|
|
5396
5527
|
" (%s)" % r["error"] if r["error"] else ""))
|
|
5397
|
-
sys.exit(0 if not failed else 1)
|
|
5528
|
+
sys.exit(2 if unmeasured else (0 if not failed else 1))
|
|
5529
|
+
|
|
5530
|
+
|
|
5531
|
+
_JS_IMPORT_RE = re.compile(
|
|
5532
|
+
r"""require\(\s*['"]([^'"]+)['"]\s*\)"""
|
|
5533
|
+
r"""|import\s+(?:.+?\s+from\s+)?['"]([^'"]+)['"]"""
|
|
5534
|
+
)
|
|
5535
|
+
|
|
5536
|
+
|
|
5537
|
+
def _impl_metrics_js(path, src):
|
|
5538
|
+
"""JS structural fingerprint (ADR-028), honestly NARROWER than Python's: `loc` (non-blank,
|
|
5539
|
+
non-`//` lines, with `/*..*/` blocks toggled by a simple per-line state machine) and
|
|
5540
|
+
`imports` (require()/import specifiers -- a lexical fact over string literals, not a
|
|
5541
|
+
narrated structure) are measured; `ast_nodes`/`functions`/`classes` are UNMEASURED (None) --
|
|
5542
|
+
no stdlib JS AST exists and the engine does not invent one."""
|
|
5543
|
+
loc = 0
|
|
5544
|
+
in_block = False
|
|
5545
|
+
for ln in src.splitlines():
|
|
5546
|
+
s = ln.strip()
|
|
5547
|
+
if in_block:
|
|
5548
|
+
if "*/" in s:
|
|
5549
|
+
in_block = False
|
|
5550
|
+
continue
|
|
5551
|
+
if not s:
|
|
5552
|
+
continue
|
|
5553
|
+
if s.startswith("/*"):
|
|
5554
|
+
if "*/" not in s[2:]:
|
|
5555
|
+
in_block = True
|
|
5556
|
+
continue
|
|
5557
|
+
if s.startswith("//"):
|
|
5558
|
+
continue
|
|
5559
|
+
loc += 1
|
|
5560
|
+
imports = set()
|
|
5561
|
+
for m in _JS_IMPORT_RE.finditer(src):
|
|
5562
|
+
spec = m.group(1) or m.group(2)
|
|
5563
|
+
if spec:
|
|
5564
|
+
imports.add(spec)
|
|
5565
|
+
return {"path": path, "loc": loc, "ast_nodes": None, "functions": None,
|
|
5566
|
+
"classes": None, "imports": sorted(imports),
|
|
5567
|
+
"sha256": hashlib.sha256(src.encode("utf-8")).hexdigest()}
|
|
5398
5568
|
|
|
5399
5569
|
|
|
5400
5570
|
def _impl_metrics(path):
|
|
5401
5571
|
"""Structural fingerprint of one implementation: physical LOC, AST node count, function
|
|
5402
|
-
and class counts, and the set of imported top-level modules. Deterministic; no judgment.
|
|
5572
|
+
and class counts, and the set of imported top-level modules. Deterministic; no judgment.
|
|
5573
|
+
Routed by extension (ADR-028): `.py` unchanged; `.js` delegates to `_impl_metrics_js`,
|
|
5574
|
+
which returns the same keys with `ast_nodes`/`functions`/`classes` UNMEASURED (None)."""
|
|
5403
5575
|
try:
|
|
5404
5576
|
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
5405
5577
|
src = fh.read()
|
|
5406
5578
|
except OSError as exc:
|
|
5407
5579
|
return {"path": path, "error": "unreadable: %s" % exc}
|
|
5580
|
+
if path.lower().endswith(".js"):
|
|
5581
|
+
return _impl_metrics_js(path, src)
|
|
5408
5582
|
loc = sum(1 for ln in src.splitlines() if ln.strip())
|
|
5409
5583
|
try:
|
|
5410
5584
|
tree = ast.parse(src)
|
|
@@ -5531,8 +5705,14 @@ def _bench_curation_map(records):
|
|
|
5531
5705
|
def _bench_oracle_all(impl_path, cases):
|
|
5532
5706
|
results = [_run_oracle_case(impl_path, c) for c in cases]
|
|
5533
5707
|
passed = sum(1 for r in results if r["ok"])
|
|
5534
|
-
|
|
5535
|
-
|
|
5708
|
+
out = {"passed": passed, "total": len(results), "green": passed == len(results),
|
|
5709
|
+
"failing": [r["name"] for r in results if not r["ok"]]}
|
|
5710
|
+
# every case UNMEASURED (ADR-028: a `.js` impl with no `node` on PATH) is NOT a fake red --
|
|
5711
|
+
# name the reason and let the caller treat it as absent, never as a scored FAIL.
|
|
5712
|
+
if results and all(r.get("unmeasured") for r in results):
|
|
5713
|
+
out["unmeasured"] = results[0]["error"]
|
|
5714
|
+
out["green"] = False
|
|
5715
|
+
return out
|
|
5536
5716
|
|
|
5537
5717
|
|
|
5538
5718
|
def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
@@ -5564,15 +5744,16 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
|
5564
5744
|
with open(cj, encoding="utf-8-sig") as fh:
|
|
5565
5745
|
c = json.load(fh)
|
|
5566
5746
|
src = c.get("source") or []
|
|
5567
|
-
unit = src
|
|
5747
|
+
unit = _entry_unit(src)
|
|
5748
|
+
all_units = [u.get("unit") for u in src if isinstance(u, dict) and u.get("unit")]
|
|
5568
5749
|
model = (c.get("compilation_report") or {}).get("model")
|
|
5569
5750
|
except (OSError, ValueError, AttributeError, IndexError):
|
|
5570
|
-
|
|
5751
|
+
all_units = []
|
|
5571
5752
|
impl = os.path.join(cd, unit.replace("/", os.sep)) if unit else None
|
|
5572
5753
|
ores = (_bench_oracle_all(impl, cases) if impl and os.path.isfile(impl)
|
|
5573
5754
|
else {"passed": 0, "total": len(cases), "green": False, "failing": ["<no impl>"]})
|
|
5574
|
-
comp_rec = {"dir": d, "model": model, "impl": impl,
|
|
5575
|
-
"compile_valid": not errors, "oracle": ores}
|
|
5755
|
+
comp_rec = {"dir": d, "model": model, "impl": impl, "entry_unit": unit,
|
|
5756
|
+
"units": len(all_units), "compile_valid": not errors, "oracle": ores}
|
|
5576
5757
|
if fidelity and impl and os.path.isfile(impl) and unit:
|
|
5577
5758
|
# the per-compiler fidelity descriptor (ADR-022): the M1 static extractor applied
|
|
5578
5759
|
# to the compiled artifact -- reverse discovery per compiler. Advisory by
|
|
@@ -5597,20 +5778,36 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
|
5597
5778
|
units_f.add(u2["unit"])
|
|
5598
5779
|
except AttributeError:
|
|
5599
5780
|
pass
|
|
5600
|
-
|
|
5781
|
+
# ADR-028: the static extractor is routed by the source unit's extension -- `.py`
|
|
5782
|
+
# via the AST (unchanged), `.js` via the target's own `node` runtime. `uns_f` is a
|
|
5783
|
+
# list of "<file>: <reason>" for JS (an int count for Python, discarded either way).
|
|
5784
|
+
sobs, uns_f = [], []
|
|
5785
|
+
for u_all in (all_units or [unit]):
|
|
5786
|
+
so, un = _static_surface_for(cd, u_all)
|
|
5787
|
+
sobs.extend(so)
|
|
5788
|
+
uns_f.extend(un)
|
|
5601
5789
|
fn_names, cls_names = [], []
|
|
5602
5790
|
for o2 in sobs:
|
|
5603
|
-
m2 = re.search(r"defines function (\w+)
|
|
5791
|
+
m2 = re.search(r"defines function (\w+)|exports function (\w+)",
|
|
5792
|
+
o2.get("statement", ""))
|
|
5604
5793
|
if m2:
|
|
5605
|
-
fn_names.append(m2.group(1))
|
|
5606
|
-
m2 = re.search(r"defines class (\w+)
|
|
5794
|
+
fn_names.append(m2.group(1) or m2.group(2))
|
|
5795
|
+
m2 = re.search(r"defines class (\w+)|exports object (\w+)",
|
|
5796
|
+
o2.get("statement", ""))
|
|
5607
5797
|
if m2:
|
|
5608
|
-
cls_names.append(m2.group(1))
|
|
5798
|
+
cls_names.append(m2.group(1) or m2.group(2))
|
|
5609
5799
|
unex = sorted(u for u in units_f if u not in traced_f)
|
|
5800
|
+
if not sobs and uns_f:
|
|
5801
|
+
# node absent (or every JS file failed to load): the surface is UNMEASURED,
|
|
5802
|
+
# never a silent "0 functions, 0 classes" that reads as a measured empty
|
|
5803
|
+
# surface (ADR-028 -- absence named, same discipline as curation_closure below).
|
|
5804
|
+
static_surface = "UNMEASURED: %s" % "; ".join(uns_f)
|
|
5805
|
+
else:
|
|
5806
|
+
static_surface = {"functions": len(fn_names), "classes": len(cls_names),
|
|
5807
|
+
"names": sorted(fn_names + cls_names)}
|
|
5610
5808
|
comp_rec["fidelity"] = {
|
|
5611
5809
|
"trace_coverage": round(len(covered_f) / max(len(node_ids_f), 1), 3),
|
|
5612
|
-
"static_surface":
|
|
5613
|
-
"names": sorted(fn_names + cls_names)},
|
|
5810
|
+
"static_surface": static_surface,
|
|
5614
5811
|
"oracle_passrate": (round(ores["passed"] / ores["total"], 3)
|
|
5615
5812
|
if ores["total"] else None),
|
|
5616
5813
|
"unexplained_share": round(len(unex) / max(len(units_f), 1), 3),
|
|
@@ -5634,9 +5831,17 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
|
5634
5831
|
stub_dir = os.path.join(entry_dir, "stub")
|
|
5635
5832
|
stub_green = False
|
|
5636
5833
|
if os.path.isdir(stub_dir):
|
|
5637
|
-
stubs = sorted(f for f in os.listdir(stub_dir) if f.endswith(".py"))
|
|
5638
|
-
if stubs
|
|
5639
|
-
|
|
5834
|
+
stubs = sorted(f for f in os.listdir(stub_dir) if f.endswith((".py", ".js")))
|
|
5835
|
+
stub_path = os.path.join(stub_dir, stubs[0]) if stubs else None
|
|
5836
|
+
if stub_path is None and os.path.isdir(os.path.join(stub_dir, "source")):
|
|
5837
|
+
# multi-unit entries (ADR-029): the stub is a directory shaped like a compilation;
|
|
5838
|
+
# its entry unit is the cli.* under source/
|
|
5839
|
+
cands = sorted(f for f in os.listdir(os.path.join(stub_dir, "source"))
|
|
5840
|
+
if f.lower().startswith("cli.") and f.endswith((".py", ".js")))
|
|
5841
|
+
if cands:
|
|
5842
|
+
stub_path = os.path.join(stub_dir, "source", cands[0])
|
|
5843
|
+
if stub_path:
|
|
5844
|
+
sres = _bench_oracle_all(stub_path, cases)
|
|
5640
5845
|
stub_green = sres["green"]
|
|
5641
5846
|
rec["discrimination"] = {"stub_passed": sres["passed"], "total": sres["total"],
|
|
5642
5847
|
"stub_green": stub_green}
|
|
@@ -5651,7 +5856,14 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
|
5651
5856
|
# distinct is None when variance could not be computed (fewer than 2 resolvable impl files),
|
|
5652
5857
|
# which is NOT the same as a byte-identical convergence -- keep the two reasons apart.
|
|
5653
5858
|
distinct = rec["variance"]["all_distinct"] if rec["variance"] else None
|
|
5654
|
-
|
|
5859
|
+
# ADR-028: node absent for a `.js` entry means EVERY case of that compilation came back
|
|
5860
|
+
# UNMEASURED -- never score that as a FAIL (nor a PASS); the entry stays PENDING with the
|
|
5861
|
+
# reason named, ahead of every other verdict check.
|
|
5862
|
+
unmeasured_js = next((i["oracle"]["unmeasured"] for i in impls
|
|
5863
|
+
if i["oracle"].get("unmeasured")), None)
|
|
5864
|
+
if unmeasured_js:
|
|
5865
|
+
rec["verdict"], rec["reason"] = "PENDING", "%s -- JS entry unmeasured" % unmeasured_js
|
|
5866
|
+
elif stub_green:
|
|
5655
5867
|
rec["verdict"], rec["reason"] = "FAIL", ("oracle satisfied by a degenerate stub -- not "
|
|
5656
5868
|
"discriminating; the entry proves nothing")
|
|
5657
5869
|
elif not all_valid:
|
|
@@ -5720,15 +5932,23 @@ def _render_bench_md(table, anon, recs):
|
|
|
5720
5932
|
if isinstance(cur_v, float):
|
|
5721
5933
|
cur_v = "%.3f (judged %d/%d)" % (cur_v, fd["curation"]["judged"],
|
|
5722
5934
|
fd["curation"]["total"])
|
|
5723
|
-
|
|
5935
|
+
ss = fd["static_surface"]
|
|
5936
|
+
# ADR-028: `static_surface` is UNMEASURED (a named string) when a JS entry's
|
|
5937
|
+
# extractor could not run (node absent) -- never a silent "0 fn / 0 cls".
|
|
5938
|
+
surface_txt = ("%d fn / %d cls" % (ss["functions"], ss["classes"])
|
|
5939
|
+
if isinstance(ss, dict) else ss)
|
|
5940
|
+
lines.append("- fidelity `%s` (%s): trace %.2f · surface %s · "
|
|
5724
5941
|
"oracle %s · unexplained %.2f · curation %s" % (
|
|
5725
5942
|
i["dir"], anon.get(i["model"], i["model"] or "?"),
|
|
5726
|
-
fd["trace_coverage"],
|
|
5727
|
-
fd["static_surface"]["classes"],
|
|
5943
|
+
fd["trace_coverage"], surface_txt,
|
|
5728
5944
|
("%.3f" % fd["oracle_passrate"])
|
|
5729
5945
|
if fd["oracle_passrate"] is not None else "n/a",
|
|
5730
5946
|
fd["unexplained_share"], cur_v))
|
|
5731
5947
|
lines.append("")
|
|
5948
|
+
if any((i.get("impl") or "").lower().endswith(".js")
|
|
5949
|
+
for r in recs for i in r["compilations"]):
|
|
5950
|
+
lines += ["*JS pairs use a 2-dimensional distance (LOC + import Jaccard; no stdlib JS "
|
|
5951
|
+
"AST).*", ""]
|
|
5732
5952
|
return "\n".join(lines)
|
|
5733
5953
|
|
|
5734
5954
|
|
|
@@ -5786,6 +6006,444 @@ def cmd_bench(args):
|
|
|
5786
6006
|
sys.exit(0)
|
|
5787
6007
|
|
|
5788
6008
|
|
|
6009
|
+
_R2_DIR = "r2"
|
|
6010
|
+
_R2_SIGNAL = 0.5 # intra/inter below this: inter-compiler variance is real signal
|
|
6011
|
+
_R2_NOISE = 1.0 # intra/inter at/above this: same-model reruns differ as much as models
|
|
6012
|
+
|
|
6013
|
+
|
|
6014
|
+
def _r2_class(ratio):
|
|
6015
|
+
if ratio is None:
|
|
6016
|
+
return None
|
|
6017
|
+
if ratio < _R2_SIGNAL:
|
|
6018
|
+
return "SIGNAL"
|
|
6019
|
+
if ratio < _R2_NOISE:
|
|
6020
|
+
return "NOISY"
|
|
6021
|
+
return "NOISE"
|
|
6022
|
+
|
|
6023
|
+
|
|
6024
|
+
def _r2_entry(entry_dir, name):
|
|
6025
|
+
"""Intra-model variance for ONE bench entry (ADR-027): for each model with a run-1 (top-level
|
|
6026
|
+
c-<model>) AND a run-2 (r2/c-<model>), the structural distance between the two runs via the
|
|
6027
|
+
SAME _struct_distance the bench uses between compilers, each run's oracle pass-rate, and
|
|
6028
|
+
whether the two runs agree on every oracle case. Entries without r2/ report absent, never 0."""
|
|
6029
|
+
ir_graph, ir_errors = _load_ir_at(os.path.join(entry_dir, IR_FILE))
|
|
6030
|
+
try:
|
|
6031
|
+
with open(os.path.join(entry_dir, "oracle", "ORACLE.json"), encoding="utf-8-sig") as fh:
|
|
6032
|
+
cases = (json.load(fh) or {}).get("cases") or []
|
|
6033
|
+
except (OSError, ValueError):
|
|
6034
|
+
cases = []
|
|
6035
|
+
rec = {"archetype": name, "has_r2": False, "models": [], "intra_mean": None,
|
|
6036
|
+
"inter": None, "intra_over_inter": None, "class": None, "reason": None}
|
|
6037
|
+
r2 = os.path.join(entry_dir, _R2_DIR)
|
|
6038
|
+
if not os.path.isdir(r2):
|
|
6039
|
+
rec["reason"] = "no r2/ directory -- intra-model variance not measured"
|
|
6040
|
+
return rec, False
|
|
6041
|
+
rec["has_r2"] = True
|
|
6042
|
+
if ir_graph is None or ir_errors or not cases:
|
|
6043
|
+
rec["reason"] = "entry incomplete (missing/invalid IR or oracle)"
|
|
6044
|
+
return rec, False
|
|
6045
|
+
|
|
6046
|
+
def load_run(cd):
|
|
6047
|
+
cj = os.path.join(cd, "COMPILATION.json")
|
|
6048
|
+
if not os.path.isfile(cj):
|
|
6049
|
+
return None
|
|
6050
|
+
errors, _adv = _validate_compilation(cj, ir_graph)
|
|
6051
|
+
try:
|
|
6052
|
+
with open(cj, encoding="utf-8-sig") as fh:
|
|
6053
|
+
c = json.load(fh)
|
|
6054
|
+
unit = _entry_unit(c.get("source") or [])
|
|
6055
|
+
model = (c.get("compilation_report") or {}).get("model")
|
|
6056
|
+
except (OSError, ValueError, AttributeError, IndexError):
|
|
6057
|
+
return None
|
|
6058
|
+
impl = os.path.join(cd, unit.replace("/", os.sep)) if unit else None
|
|
6059
|
+
if not impl or not os.path.isfile(impl):
|
|
6060
|
+
return None
|
|
6061
|
+
ores = _bench_oracle_all(impl, cases)
|
|
6062
|
+
m = _impl_metrics(impl)
|
|
6063
|
+
return {"model": model, "compile_valid": not errors, "oracle": ores,
|
|
6064
|
+
"metrics": m if "error" not in m else None,
|
|
6065
|
+
"case_vector": tuple(sorted(ores.get("failing") or []))}
|
|
6066
|
+
|
|
6067
|
+
def rate(o):
|
|
6068
|
+
return round(o["passed"] / o["total"], 3) if o["total"] else None
|
|
6069
|
+
|
|
6070
|
+
intra = []
|
|
6071
|
+
run1_metrics = []
|
|
6072
|
+
has_js = False # footer disclosure only; not serialized
|
|
6073
|
+
for d in sorted(os.listdir(entry_dir)):
|
|
6074
|
+
if not d.startswith("c-") or not os.path.isdir(os.path.join(entry_dir, d)):
|
|
6075
|
+
continue
|
|
6076
|
+
r1 = load_run(os.path.join(entry_dir, d))
|
|
6077
|
+
if r1 is None:
|
|
6078
|
+
continue
|
|
6079
|
+
if r1["metrics"]:
|
|
6080
|
+
run1_metrics.append(r1["metrics"])
|
|
6081
|
+
if r1["metrics"]["path"].lower().endswith(".js"):
|
|
6082
|
+
has_js = True
|
|
6083
|
+
r2run = load_run(os.path.join(r2, d))
|
|
6084
|
+
if r2run is None:
|
|
6085
|
+
rec["models"].append({"dir": d, "model": r1["model"], "r2": "absent"})
|
|
6086
|
+
continue
|
|
6087
|
+
if r2run["metrics"] and r2run["metrics"]["path"].lower().endswith(".js"):
|
|
6088
|
+
has_js = True
|
|
6089
|
+
dist = (_struct_distance(r1["metrics"], r2run["metrics"])
|
|
6090
|
+
if r1["metrics"] and r2run["metrics"] else None)
|
|
6091
|
+
stable = r1["case_vector"] == r2run["case_vector"]
|
|
6092
|
+
rec["models"].append({"dir": d, "model": r1["model"],
|
|
6093
|
+
"intra_distance": round(dist, 4) if dist is not None else None,
|
|
6094
|
+
"run1_passrate": rate(r1["oracle"]),
|
|
6095
|
+
"run2_passrate": rate(r2run["oracle"]),
|
|
6096
|
+
"run2_compile_valid": r2run["compile_valid"],
|
|
6097
|
+
"behaviour_stable": stable})
|
|
6098
|
+
if dist is not None:
|
|
6099
|
+
intra.append(dist)
|
|
6100
|
+
if intra:
|
|
6101
|
+
rec["intra_mean"] = round(sum(intra) / len(intra), 4)
|
|
6102
|
+
inter = []
|
|
6103
|
+
for i in range(len(run1_metrics)):
|
|
6104
|
+
for j in range(i + 1, len(run1_metrics)):
|
|
6105
|
+
inter.append(_struct_distance(run1_metrics[i], run1_metrics[j]))
|
|
6106
|
+
if inter:
|
|
6107
|
+
rec["inter"] = round(sum(inter) / len(inter), 4)
|
|
6108
|
+
if rec["intra_mean"] is not None and rec["inter"]:
|
|
6109
|
+
# 2 dp on purpose: the ratio moves by up to ~0.26 between interpreters (ast.walk
|
|
6110
|
+
# counts nodes differently on 3.8 vs 3.13 -- the 1.84.0 review measured it); more
|
|
6111
|
+
# decimals would print precision the measurement does not have (ADR-027).
|
|
6112
|
+
rec["intra_over_inter"] = round(rec["intra_mean"] / rec["inter"], 2)
|
|
6113
|
+
rec["class"] = _r2_class(rec["intra_over_inter"])
|
|
6114
|
+
elif rec["intra_mean"] is not None and rec["inter"] == 0.0:
|
|
6115
|
+
rec["reason"] = "inter-compiler distance is 0 (converged run-1) -- ratio undefined"
|
|
6116
|
+
elif rec["intra_mean"] is None:
|
|
6117
|
+
# r2/ exists but no model pair yielded a computable distance (unparseable source,
|
|
6118
|
+
# missing COMPILATION.json in every r2 model): absent, named -- never a silent None
|
|
6119
|
+
rec["reason"] = ("r2/ present but no model pair measurable (no parseable run-1+run-2 "
|
|
6120
|
+
"source for any model) -- intra-model variance not measured")
|
|
6121
|
+
return rec, has_js
|
|
6122
|
+
|
|
6123
|
+
|
|
6124
|
+
_R2_VERDICT_TEXT = {
|
|
6125
|
+
"SIGNAL": "Across the measured entries, same-model reruns differ far less than different "
|
|
6126
|
+
"models do: the inter-compiler variance the program reports is signal, not sampling "
|
|
6127
|
+
"noise -- for these archetypes, at n=2 per model.",
|
|
6128
|
+
"NOISY": "Same-model reruns differ by a sizeable fraction of the inter-compiler distance: "
|
|
6129
|
+
"inter-compiler variance carries information, but a delta smaller than the noise "
|
|
6130
|
+
"floor should not be read as an effect.",
|
|
6131
|
+
"NOISE": "Same-model reruns differ as much as different models do: the inter-compiler "
|
|
6132
|
+
"variance the program reports is dominated by sampling noise for these archetypes -- "
|
|
6133
|
+
"variance-based claims must be re-read.",
|
|
6134
|
+
"UNMEASURED": "No entry carries a second run; the noise floor is absent, not zero.",
|
|
6135
|
+
}
|
|
6136
|
+
|
|
6137
|
+
|
|
6138
|
+
def _render_r2_md(recs, agg, has_js=False):
|
|
6139
|
+
lines = ["<!-- GENERATED by qa_ledger.py bench-r2 (ADR-027) -- measured run; do not hand-edit. -->",
|
|
6140
|
+
"", "# DIAMOND-BENCH-R2 -- intra-model variance, the noise floor under the bench", "",
|
|
6141
|
+
"For every archetype with a second blind run (`r2/`) of the SAME model on the SAME "
|
|
6142
|
+
"canonical package: the structural distance between run 1 and run 2 (the same "
|
|
6143
|
+
"function the bench uses BETWEEN compilers), each run's oracle pass-rate, and "
|
|
6144
|
+
"whether both runs fail the same oracle cases. `intra/inter` below 0.5 reads SIGNAL "
|
|
6145
|
+
"(inter-compiler variance is at least twice the noise floor), 0.5-1.0 NOISY, at or "
|
|
6146
|
+
"above 1.0 NOISE (same-model reruns differ as much as different models). Advisory: "
|
|
6147
|
+
"no bench verdict changes; this qualifies every variance claim the program makes.", "",
|
|
6148
|
+
"| Archetype | intra (mean) | inter | intra/inter | class | per model (run1 -> run2 pass-rate, stable) |",
|
|
6149
|
+
"|-----------|--------------|-------|-------------|-------|--------------------------------------------|"]
|
|
6150
|
+
for r in recs:
|
|
6151
|
+
if not r["has_r2"]:
|
|
6152
|
+
lines.append("| %s | -- | -- | -- | (no r2) | %s |" % (r["archetype"], r["reason"] or ""))
|
|
6153
|
+
continue
|
|
6154
|
+
pm = "; ".join(("%s %s->%s %s" % (m["dir"], m.get("run1_passrate"), m.get("run2_passrate"),
|
|
6155
|
+
"stable" if m.get("behaviour_stable") else "DIFFERS"))
|
|
6156
|
+
if m.get("r2") != "absent" else ("%s (r2 absent)" % m["dir"])
|
|
6157
|
+
for m in r["models"])
|
|
6158
|
+
lines.append("| %s | %s | %s | %s | %s | %s |" % (
|
|
6159
|
+
r["archetype"],
|
|
6160
|
+
"%.4f" % r["intra_mean"] if r["intra_mean"] is not None else "--",
|
|
6161
|
+
"%.4f" % r["inter"] if r["inter"] is not None else "--",
|
|
6162
|
+
"%.2f" % r["intra_over_inter"] if r["intra_over_inter"] is not None else "--",
|
|
6163
|
+
r["class"] or (r["reason"] or "--"), pm))
|
|
6164
|
+
lines += ["", "**Aggregate:** %d entries measured -- SIGNAL %d · NOISY %d · NOISE %d · mean "
|
|
6165
|
+
"intra/inter %s · behaviour-stable reruns %d/%d." % (
|
|
6166
|
+
agg["measured"], agg["signal"], agg["noisy"], agg["noise"],
|
|
6167
|
+
"%.3f" % agg["mean_ratio"] if agg["mean_ratio"] is not None else "n/a",
|
|
6168
|
+
agg["stable"], agg["reruns"]),
|
|
6169
|
+
"", "## Verdict: %s" % agg["verdict"], "", _R2_VERDICT_TEXT[agg["verdict"]],
|
|
6170
|
+
"", "*Structural distance = mean of normalized LOC delta, AST-node delta and "
|
|
6171
|
+
"import-set Jaccard distance -- the one function shared by lang-compare, bench "
|
|
6172
|
+
"and bench-r2. n=2 per model is the minimum that yields a floor at all; it is a "
|
|
6173
|
+
"floor, not a distribution. The ratio is printed to 2 decimals on purpose: the "
|
|
6174
|
+
"AST-node count differs between Python versions (ast.walk on 3.8 vs 3.13), so a "
|
|
6175
|
+
"ratio can move by up to ~0.26 across interpreters without any code or model "
|
|
6176
|
+
"changing -- classes are stable across 3.8/3.13 on this bench, ratios are not "
|
|
6177
|
+
"point-precise, and an entry within ~0.25 of a threshold should be read as "
|
|
6178
|
+
"borderline.*", ""]
|
|
6179
|
+
if has_js:
|
|
6180
|
+
lines += ["*JS pairs use a 2-dimensional distance (LOC + import Jaccard; no stdlib JS "
|
|
6181
|
+
"AST).*", ""]
|
|
6182
|
+
return "\n".join(lines)
|
|
6183
|
+
|
|
6184
|
+
|
|
6185
|
+
def cmd_bench_r2(args):
|
|
6186
|
+
if not os.path.isdir(args.dir):
|
|
6187
|
+
print("[qa_ledger] bench-r2: no directory %s" % args.dir, file=sys.stderr)
|
|
6188
|
+
sys.exit(2)
|
|
6189
|
+
entries = sorted(d for d in os.listdir(args.dir)
|
|
6190
|
+
if os.path.isfile(os.path.join(args.dir, d, IR_FILE)))
|
|
6191
|
+
if not entries:
|
|
6192
|
+
print("[qa_ledger] bench-r2: no entries under %s" % args.dir, file=sys.stderr)
|
|
6193
|
+
sys.exit(2)
|
|
6194
|
+
pairs = [_r2_entry(os.path.join(args.dir, e), e) for e in entries]
|
|
6195
|
+
recs = [p[0] for p in pairs]
|
|
6196
|
+
has_js = any(p[1] for p in pairs)
|
|
6197
|
+
measured = [r for r in recs if r["class"] is not None]
|
|
6198
|
+
ratios = [r["intra_over_inter"] for r in measured]
|
|
6199
|
+
reruns = [m for r in recs if r["has_r2"] for m in r["models"] if m.get("r2") != "absent"]
|
|
6200
|
+
agg = {"entries": len(recs), "with_r2": sum(1 for r in recs if r["has_r2"]),
|
|
6201
|
+
"measured": len(measured),
|
|
6202
|
+
"signal": sum(1 for r in measured if r["class"] == "SIGNAL"),
|
|
6203
|
+
"noisy": sum(1 for r in measured if r["class"] == "NOISY"),
|
|
6204
|
+
"noise": sum(1 for r in measured if r["class"] == "NOISE"),
|
|
6205
|
+
"mean_ratio": round(sum(ratios) / len(ratios), 3) if ratios else None,
|
|
6206
|
+
"reruns": len(reruns),
|
|
6207
|
+
"stable": sum(1 for m in reruns if m.get("behaviour_stable"))}
|
|
6208
|
+
if not measured:
|
|
6209
|
+
agg["verdict"] = "UNMEASURED"
|
|
6210
|
+
else:
|
|
6211
|
+
counts = [(agg["signal"], "SIGNAL"), (agg["noisy"], "NOISY"), (agg["noise"], "NOISE")]
|
|
6212
|
+
top = max(c for c, _ in counts)
|
|
6213
|
+
# ties resolve toward the more cautious reading (NOISE > NOISY > SIGNAL)
|
|
6214
|
+
agg["verdict"] = [n for c, n in reversed(counts) if c == top][0]
|
|
6215
|
+
md = _render_r2_md(recs, agg, has_js)
|
|
6216
|
+
if args.out:
|
|
6217
|
+
with open(args.out, "w", encoding="utf-8", newline="\n") as fh:
|
|
6218
|
+
fh.write(md)
|
|
6219
|
+
if args.json:
|
|
6220
|
+
print(json.dumps({"aggregate": agg, "raw": recs}, indent=2, ensure_ascii=False))
|
|
6221
|
+
else:
|
|
6222
|
+
print("BENCH-R2: %d entries, %d with r2, %d measured -- SIGNAL %d / NOISY %d / NOISE %d"
|
|
6223
|
+
% (agg["entries"], agg["with_r2"], agg["measured"], agg["signal"], agg["noisy"],
|
|
6224
|
+
agg["noise"]))
|
|
6225
|
+
for r in recs:
|
|
6226
|
+
if r["class"]:
|
|
6227
|
+
print(" %-17s intra %.4f inter %.4f ratio %.2f %s"
|
|
6228
|
+
% (r["archetype"], r["intra_mean"], r["inter"], r["intra_over_inter"],
|
|
6229
|
+
r["class"]))
|
|
6230
|
+
elif r["has_r2"]:
|
|
6231
|
+
print(" %-17s %s" % (r["archetype"], r["reason"]))
|
|
6232
|
+
print("VERDICT: %s%s" % (agg["verdict"], (" -> %s" % args.out) if args.out else ""))
|
|
6233
|
+
print(" (ratios are not point-precise: AST-node counts differ across Python versions; "
|
|
6234
|
+
"classes stable, ratios +/-0.25 -- read borderline entries as borderline)")
|
|
6235
|
+
sys.exit(0)
|
|
6236
|
+
|
|
6237
|
+
|
|
6238
|
+
_RT_ID_RE = re.compile(r"(?i)\b(AC|INV|ADR)[-_][A-Z0-9-]*?\d+\b")
|
|
6239
|
+
|
|
6240
|
+
|
|
6241
|
+
def _rt_ids_in(text):
|
|
6242
|
+
"""Every canonical id (AC-*, INV-*, ADR-*) literally referenced in a text, upper-cased.
|
|
6243
|
+
ID REFERENCE only (the ADR-013 rule) -- no fuzzy or semantic matching, ever."""
|
|
6244
|
+
out = set()
|
|
6245
|
+
for m in _RT_ID_RE.finditer(text or ""):
|
|
6246
|
+
out.add(m.group(0).upper().replace("_", "-"))
|
|
6247
|
+
return out
|
|
6248
|
+
|
|
6249
|
+
|
|
6250
|
+
def _rt_read_source(cd, unit):
|
|
6251
|
+
try:
|
|
6252
|
+
with open(os.path.join(cd, unit.replace("/", os.sep)), encoding="utf-8",
|
|
6253
|
+
errors="replace") as fh:
|
|
6254
|
+
return fh.read()
|
|
6255
|
+
except OSError:
|
|
6256
|
+
return ""
|
|
6257
|
+
|
|
6258
|
+
|
|
6259
|
+
def _rt_compilation(entry_dir, cd, ir_graph, cases):
|
|
6260
|
+
"""Round-trip recoverability of ONE compilation against the pinned IR (ADR-030): for every
|
|
6261
|
+
IR node, whether the mechanical reverse organs find footing for it in the artifact --
|
|
6262
|
+
(a) static: a source unit's text or a static observation literally references the id;
|
|
6263
|
+
(b) manifest: the compiler's validated trace manifest maps the node to a unit that exists;
|
|
6264
|
+
(c) behaviour: for AC nodes, at least one withheld-oracle case tagged with the id passes
|
|
6265
|
+
(UNMEASURED when no case carries any AC tag). It regenerates NOTHING: no IR', no spec --
|
|
6266
|
+
a coverage over the human-authored IR, plus the list of nodes nothing anchors."""
|
|
6267
|
+
cj = os.path.join(cd, "COMPILATION.json")
|
|
6268
|
+
try:
|
|
6269
|
+
with open(cj, encoding="utf-8-sig") as fh:
|
|
6270
|
+
c = json.load(fh)
|
|
6271
|
+
except (OSError, ValueError):
|
|
6272
|
+
return None
|
|
6273
|
+
units = [u.get("unit") for u in (c.get("source") or []) if isinstance(u, dict) and u.get("unit")]
|
|
6274
|
+
units = [u for u in units if os.path.isfile(os.path.join(cd, u.replace("/", os.sep)))]
|
|
6275
|
+
node_ids = [n["id"] for n in (ir_graph.get("nodes") or [])]
|
|
6276
|
+
# (a) static footing: ids referenced in source text (comments/docstrings/names) or in
|
|
6277
|
+
# the static observations' statements/provenance
|
|
6278
|
+
static_ids = set()
|
|
6279
|
+
for u in units:
|
|
6280
|
+
static_ids |= _rt_ids_in(_rt_read_source(cd, u))
|
|
6281
|
+
so, _un = _static_surface_for(cd, u)
|
|
6282
|
+
for o in so:
|
|
6283
|
+
static_ids |= _rt_ids_in(o.get("statement", ""))
|
|
6284
|
+
# (b) manifest footing
|
|
6285
|
+
manifest_ids = set()
|
|
6286
|
+
for e in c.get("trace_manifest") or []:
|
|
6287
|
+
if e.get("unit") in units:
|
|
6288
|
+
for nid in e.get("implements") or []:
|
|
6289
|
+
manifest_ids.add(str(nid).upper())
|
|
6290
|
+
# (c) behaviour footing: oracle cases whose NAME carries an AC id (payload tags are not a
|
|
6291
|
+
# convention the bench oracles use today -- absence named, not faked)
|
|
6292
|
+
tagged = {}
|
|
6293
|
+
for case in cases:
|
|
6294
|
+
for cid in _rt_ids_in(case.get("name", "")):
|
|
6295
|
+
tagged.setdefault(cid, []).append(case)
|
|
6296
|
+
behaviour_measured = bool(tagged)
|
|
6297
|
+
behaviour_ids = set()
|
|
6298
|
+
entry_unit = _entry_unit(c.get("source") or [])
|
|
6299
|
+
impl = os.path.join(cd, entry_unit.replace("/", os.sep)) if entry_unit else None
|
|
6300
|
+
if behaviour_measured and impl and os.path.isfile(impl):
|
|
6301
|
+
for cid, cs in tagged.items():
|
|
6302
|
+
if any(_run_oracle_case(impl, case).get("ok") for case in cs):
|
|
6303
|
+
behaviour_ids.add(cid)
|
|
6304
|
+
per_node = []
|
|
6305
|
+
anchored = 0
|
|
6306
|
+
for nid in node_ids:
|
|
6307
|
+
a_s = nid in static_ids
|
|
6308
|
+
a_m = nid in manifest_ids
|
|
6309
|
+
a_b = (nid in behaviour_ids) if (behaviour_measured and nid.startswith("AC-")) else None
|
|
6310
|
+
anchored_any = a_s or a_m or bool(a_b)
|
|
6311
|
+
# the MEASURED footing excludes the manifest: the manifest is what the compiler
|
|
6312
|
+
# CLAIMED (and the prompt handed it the ids), so counting it as recovered would be
|
|
6313
|
+
# tautological -- the 1.000 that means nothing (found on the first run of this tool)
|
|
6314
|
+
anchored_meas = a_s or bool(a_b)
|
|
6315
|
+
anchored += 1 if anchored_meas else 0
|
|
6316
|
+
per_node.append({"id": nid, "static": a_s, "manifest": a_m,
|
|
6317
|
+
"behaviour": a_b, "anchored": anchored_meas,
|
|
6318
|
+
"claimed": anchored_any})
|
|
6319
|
+
edges = ir_graph.get("edges") or []
|
|
6320
|
+
anch_ids = {n["id"] for n in per_node if n["anchored"]}
|
|
6321
|
+
edges_recovered = sum(1 for e in edges if e.get("from") in anch_ids and e.get("to") in anch_ids)
|
|
6322
|
+
claimed = sum(1 for n in per_node if n["claimed"])
|
|
6323
|
+
return {"dir": os.path.basename(cd), "units": len(units),
|
|
6324
|
+
"ir_nodes": len(node_ids), "anchored": anchored, "claimed": claimed,
|
|
6325
|
+
"recoverability": round(anchored / len(node_ids), 3) if node_ids else None,
|
|
6326
|
+
"claimed_share": round(claimed / len(node_ids), 3) if node_ids else None,
|
|
6327
|
+
"static_anchored": sum(1 for n in per_node if n["static"]),
|
|
6328
|
+
"manifest_anchored": sum(1 for n in per_node if n["manifest"]),
|
|
6329
|
+
"behaviour": ("UNMEASURED" if not behaviour_measured
|
|
6330
|
+
else sum(1 for n in per_node if n["behaviour"])),
|
|
6331
|
+
"edges": len(edges), "edges_recovered": edges_recovered,
|
|
6332
|
+
"unanchored": [n["id"] for n in per_node if not n["anchored"]],
|
|
6333
|
+
"nodes": per_node}
|
|
6334
|
+
|
|
6335
|
+
|
|
6336
|
+
def _rt_entry(entry_dir, name):
|
|
6337
|
+
ir_graph, ir_errors = _load_ir_at(os.path.join(entry_dir, IR_FILE))
|
|
6338
|
+
try:
|
|
6339
|
+
with open(os.path.join(entry_dir, "oracle", "ORACLE.json"), encoding="utf-8-sig") as fh:
|
|
6340
|
+
cases = (json.load(fh) or {}).get("cases") or []
|
|
6341
|
+
except (OSError, ValueError):
|
|
6342
|
+
cases = []
|
|
6343
|
+
rec = {"archetype": name, "compilations": [], "ir_nodes": None, "edges": None,
|
|
6344
|
+
"recoverability_mean": None, "edges_recovered_mean": None, "reason": None}
|
|
6345
|
+
if ir_graph is None or ir_errors:
|
|
6346
|
+
rec["reason"] = "entry incomplete (missing/invalid IR)"
|
|
6347
|
+
return rec
|
|
6348
|
+
rec["ir_nodes"] = len(ir_graph.get("nodes") or [])
|
|
6349
|
+
rec["edges"] = len(ir_graph.get("edges") or [])
|
|
6350
|
+
for d in sorted(os.listdir(entry_dir)):
|
|
6351
|
+
cd = os.path.join(entry_dir, d)
|
|
6352
|
+
if d.startswith("c-") and os.path.isfile(os.path.join(cd, "COMPILATION.json")):
|
|
6353
|
+
r = _rt_compilation(entry_dir, cd, ir_graph, cases)
|
|
6354
|
+
if r:
|
|
6355
|
+
rec["compilations"].append(r)
|
|
6356
|
+
recs = [c["recoverability"] for c in rec["compilations"] if c["recoverability"] is not None]
|
|
6357
|
+
if recs:
|
|
6358
|
+
rec["recoverability_mean"] = round(sum(recs) / len(recs), 3)
|
|
6359
|
+
rec["edges_recovered_mean"] = round(
|
|
6360
|
+
sum(c["edges_recovered"] for c in rec["compilations"]) / len(rec["compilations"]), 2)
|
|
6361
|
+
else:
|
|
6362
|
+
rec["reason"] = "no compilations yet"
|
|
6363
|
+
return rec
|
|
6364
|
+
|
|
6365
|
+
|
|
6366
|
+
def _render_rt_md(recs, agg):
|
|
6367
|
+
lines = ["<!-- GENERATED by qa_ledger.py bench-roundtrip (ADR-030) -- measured run; do not hand-edit. -->",
|
|
6368
|
+
"", "# DIAMOND-ROUNDTRIP -- how much of the pinned IR the reverse organs can anchor in each compiled artifact", "",
|
|
6369
|
+
"**What this is NOT:** it does not regenerate an IR from code, it does not diff specs, "
|
|
6370
|
+
"it does not infer requirements. Reverse discovery produces FACTS (ADR-013); the human "
|
|
6371
|
+
"authors the spec. This instrument joins those facts against the human-authored IR "
|
|
6372
|
+
"and reports, per compilation, which nodes have footing -- (a) static: the id is "
|
|
6373
|
+
"literally referenced in a source unit or a static observation; (b) manifest: the "
|
|
6374
|
+
"compiler's validated trace manifest maps the node to a unit that exists; (c) "
|
|
6375
|
+
"behaviour: a withheld-oracle case tagged with the AC id passes (UNMEASURED where "
|
|
6376
|
+
"the entry's oracle cases carry no AC tag -- absence named). **`recoverability` "
|
|
6377
|
+
"counts ONLY static + behaviour footing** -- the manifest is what the compiler "
|
|
6378
|
+
"CLAIMED (and the blind prompt handed it the node ids), so it is reported apart as "
|
|
6379
|
+
"`claimed` and never counted as recovered: the first run of this instrument read "
|
|
6380
|
+
"1.000 on every entry for exactly that reason. `edges recovered` = edges with both "
|
|
6381
|
+
"endpoints MEASURED-anchored. Advisory: no bench verdict changes. The unanchored "
|
|
6382
|
+
"list IS the honest gap, per compiler.", "",
|
|
6383
|
+
"| Archetype | IR nodes | edges | recoverability (mean, measured) | edges recovered (mean) | per compilation (measured/nodes · static · behaviour · claimed-by-manifest · unanchored) |",
|
|
6384
|
+
"|-----------|----------|-------|---------------------------------|------------------------|--------------------------------------------------------------------------------------------|"]
|
|
6385
|
+
for r in recs:
|
|
6386
|
+
if r["recoverability_mean"] is None:
|
|
6387
|
+
lines.append("| %s | %s | %s | -- | -- | %s |" % (r["archetype"], r["ir_nodes"], r["edges"], r["reason"] or ""))
|
|
6388
|
+
continue
|
|
6389
|
+
pc = "; ".join("%s %d/%d · s%d · b%s · claimed %d · [%s]" % (
|
|
6390
|
+
c["dir"], c["anchored"], c["ir_nodes"], c["static_anchored"],
|
|
6391
|
+
c["behaviour"], c["manifest_anchored"], ",".join(c["unanchored"]) or "none")
|
|
6392
|
+
for c in r["compilations"])
|
|
6393
|
+
lines.append("| %s | %d | %d | %.3f | %.2f | %s |" % (
|
|
6394
|
+
r["archetype"], r["ir_nodes"], r["edges"], r["recoverability_mean"],
|
|
6395
|
+
r["edges_recovered_mean"], pc))
|
|
6396
|
+
lines += ["", "**Aggregate:** %d entries measured · mean recoverability %s · entries with edges %d "
|
|
6397
|
+
"· behaviour dimension measured in %d/%d entries." % (
|
|
6398
|
+
agg["measured"], "%.3f" % agg["mean_recoverability"] if agg["mean_recoverability"] is not None else "n/a",
|
|
6399
|
+
agg["with_edges"], agg["behaviour_measured"], agg["measured"]),
|
|
6400
|
+
"", "*The manifest dimension is what the compiler CLAIMED (validated for shape, "
|
|
6401
|
+
"not truth) and is excluded from recoverability; the static dimension is what the "
|
|
6402
|
+
"artifact literally names; the behaviour dimension is what the withheld oracle can "
|
|
6403
|
+
"attribute per AC -- and it is UNMEASURED wherever oracle cases are not tagged with "
|
|
6404
|
+
"AC ids, which today is every entry: the honest state of reverse discovery is that "
|
|
6405
|
+
"it anchors names, not semantics, until oracles carry per-AC tags. None of the three "
|
|
6406
|
+
"is a spec.*", ""]
|
|
6407
|
+
return "\n".join(lines)
|
|
6408
|
+
|
|
6409
|
+
|
|
6410
|
+
def cmd_bench_roundtrip(args):
|
|
6411
|
+
if not os.path.isdir(args.dir):
|
|
6412
|
+
print("[qa_ledger] bench-roundtrip: no directory %s" % args.dir, file=sys.stderr)
|
|
6413
|
+
sys.exit(2)
|
|
6414
|
+
entries = sorted(d for d in os.listdir(args.dir)
|
|
6415
|
+
if os.path.isfile(os.path.join(args.dir, d, IR_FILE)))
|
|
6416
|
+
if not entries:
|
|
6417
|
+
print("[qa_ledger] bench-roundtrip: no entries under %s" % args.dir, file=sys.stderr)
|
|
6418
|
+
sys.exit(2)
|
|
6419
|
+
recs = [_rt_entry(os.path.join(args.dir, e), e) for e in entries]
|
|
6420
|
+
measured = [r for r in recs if r["recoverability_mean"] is not None]
|
|
6421
|
+
agg = {"entries": len(recs), "measured": len(measured),
|
|
6422
|
+
"mean_recoverability": (round(sum(r["recoverability_mean"] for r in measured) / len(measured), 3)
|
|
6423
|
+
if measured else None),
|
|
6424
|
+
"with_edges": sum(1 for r in measured if (r["edges"] or 0) > 0),
|
|
6425
|
+
"behaviour_measured": sum(1 for r in measured
|
|
6426
|
+
if any(c["behaviour"] != "UNMEASURED" for c in r["compilations"]))}
|
|
6427
|
+
md = _render_rt_md(recs, agg)
|
|
6428
|
+
if args.out:
|
|
6429
|
+
with open(args.out, "w", encoding="utf-8", newline="\n") as fh:
|
|
6430
|
+
fh.write(md)
|
|
6431
|
+
if args.json:
|
|
6432
|
+
print(json.dumps({"aggregate": agg, "raw": recs}, indent=2, ensure_ascii=False))
|
|
6433
|
+
else:
|
|
6434
|
+
print("BENCH-ROUNDTRIP: %d entries, %d measured -- mean recoverability %s, %d with edges, "
|
|
6435
|
+
"behaviour measured in %d" % (agg["entries"], agg["measured"],
|
|
6436
|
+
"%.3f" % agg["mean_recoverability"] if agg["mean_recoverability"] is not None else "n/a",
|
|
6437
|
+
agg["with_edges"], agg["behaviour_measured"]))
|
|
6438
|
+
for r in recs:
|
|
6439
|
+
if r["recoverability_mean"] is not None:
|
|
6440
|
+
print(" %-17s nodes %2d edges %d recover %.3f edges-rec %.2f" % (
|
|
6441
|
+
r["archetype"], r["ir_nodes"], r["edges"], r["recoverability_mean"], r["edges_recovered_mean"]))
|
|
6442
|
+
if args.out:
|
|
6443
|
+
print(" -> %s" % args.out)
|
|
6444
|
+
sys.exit(0)
|
|
6445
|
+
|
|
6446
|
+
|
|
5789
6447
|
def _bench_curable_obs(bench_dir, entry, cdir):
|
|
5790
6448
|
"""The curable set for one compilation: the SAME observations the M1 static extractor
|
|
5791
6449
|
produces for the descriptor's static_surface, re-extracted at call time -- a verdict
|
|
@@ -5802,13 +6460,20 @@ def _bench_curable_obs(bench_dir, entry, cdir):
|
|
|
5802
6460
|
with open(cj, encoding="utf-8-sig") as fh:
|
|
5803
6461
|
c = json.load(fh)
|
|
5804
6462
|
src = c.get("source") or []
|
|
5805
|
-
|
|
6463
|
+
units = [u.get("unit") for u in src if isinstance(u, dict) and u.get("unit")]
|
|
5806
6464
|
except (OSError, ValueError, AttributeError, IndexError):
|
|
5807
|
-
|
|
5808
|
-
|
|
6465
|
+
units = []
|
|
6466
|
+
units = [u for u in units if os.path.isfile(os.path.join(cd, u.replace("/", os.sep)))]
|
|
6467
|
+
if not units:
|
|
5809
6468
|
return None, ("%s/%s: no resolvable source unit -- nothing to extract a surface "
|
|
5810
6469
|
"from" % (entry, cdir))
|
|
5811
|
-
sobs,
|
|
6470
|
+
sobs, uns = [], []
|
|
6471
|
+
for u in units:
|
|
6472
|
+
so, un = _static_surface_for(cd, u)
|
|
6473
|
+
sobs.extend(so)
|
|
6474
|
+
uns.extend(un)
|
|
6475
|
+
if not sobs and uns:
|
|
6476
|
+
return None, ("%s/%s: static surface UNMEASURED -- %s" % (entry, cdir, "; ".join(uns)))
|
|
5812
6477
|
return sobs, None
|
|
5813
6478
|
|
|
5814
6479
|
|
|
@@ -5895,7 +6560,9 @@ _LANG_PR_MARGIN = 0.02 # mean oracle pass-rate drop
|
|
|
5895
6560
|
|
|
5896
6561
|
def _lang_arm_metrics(arm_dir):
|
|
5897
6562
|
"""Measure ONE arm: per-compiler oracle green + unresolved_intent, and inter-compiler
|
|
5898
|
-
variance over the arm's compilations. Reuses the M4/M5 organs unchanged.
|
|
6563
|
+
variance over the arm's compilations. Reuses the M4/M5 organs unchanged. Returns
|
|
6564
|
+
(rec, errors, has_js) -- has_js is a footer-only signal for _render_lang_md (ADR-028);
|
|
6565
|
+
it is never part of `rec`, so lang-compare's JSON output is untouched by it."""
|
|
5899
6566
|
ir_graph, ir_errors = _load_ir_at(os.path.join(arm_dir, IR_FILE))
|
|
5900
6567
|
try:
|
|
5901
6568
|
with open(os.path.join(arm_dir, "oracle", "ORACLE.json"), encoding="utf-8-sig") as fh:
|
|
@@ -5907,7 +6574,7 @@ def _lang_arm_metrics(arm_dir):
|
|
|
5907
6574
|
"mean_passrate": None, "ui_count": 0.0, "ui_distinct_regions": 0.0,
|
|
5908
6575
|
"ui_rationale_len": 0.0}
|
|
5909
6576
|
if ir_graph is None or ir_errors or not cases:
|
|
5910
|
-
return rec, ["arm incomplete (missing/invalid IR or oracle)"]
|
|
6577
|
+
return rec, ["arm incomplete (missing/invalid IR or oracle)"], False
|
|
5911
6578
|
comp_dirs = sorted(d for d in os.listdir(arm_dir)
|
|
5912
6579
|
if d.startswith("c-") and
|
|
5913
6580
|
os.path.isfile(os.path.join(arm_dir, d, "COMPILATION.json")))
|
|
@@ -5921,7 +6588,7 @@ def _lang_arm_metrics(arm_dir):
|
|
|
5921
6588
|
with open(cj, encoding="utf-8-sig") as fh:
|
|
5922
6589
|
c = json.load(fh)
|
|
5923
6590
|
src = c.get("source") or []
|
|
5924
|
-
unit = src
|
|
6591
|
+
unit = _entry_unit(src)
|
|
5925
6592
|
model = (c.get("compilation_report") or {}).get("model")
|
|
5926
6593
|
uis = c.get("unresolved_intent") or []
|
|
5927
6594
|
except (OSError, ValueError, AttributeError, IndexError):
|
|
@@ -5954,14 +6621,29 @@ def _lang_arm_metrics(arm_dir):
|
|
|
5954
6621
|
dists = []
|
|
5955
6622
|
for i in range(len(good)):
|
|
5956
6623
|
for j in range(i + 1, len(good)):
|
|
5957
|
-
|
|
5958
|
-
ia, ib = set(a["imports"]), set(b["imports"])
|
|
5959
|
-
jac = (len(ia & ib) / len(ia | ib)) if (ia or ib) else 1.0
|
|
5960
|
-
dloc = abs(a["loc"] - b["loc"]) / max(a["loc"], b["loc"], 1)
|
|
5961
|
-
dast = abs(a["ast_nodes"] - b["ast_nodes"]) / max(a["ast_nodes"], b["ast_nodes"], 1)
|
|
5962
|
-
dists.append((dloc + dast + (1.0 - jac)) / 3.0)
|
|
6624
|
+
dists.append(_struct_distance(good[i], good[j]))
|
|
5963
6625
|
rec["variance_score"] = round(sum(dists) / len(dists), 4) if dists else None
|
|
5964
|
-
|
|
6626
|
+
has_js = any(p.lower().endswith(".js") for p in impl_paths)
|
|
6627
|
+
return rec, [], has_js
|
|
6628
|
+
|
|
6629
|
+
|
|
6630
|
+
def _struct_distance(a, b):
|
|
6631
|
+
"""The ONE structural distance the program uses between two implementations -- mean of
|
|
6632
|
+
normalized LOC delta, AST-node delta and import-set Jaccard distance (0 = identical). Shared
|
|
6633
|
+
by the inter-compiler variance (lang-compare, bench) and the intra-model variance (bench-r2,
|
|
6634
|
+
ADR-027) so their ratio is commensurable: same function, both sides.
|
|
6635
|
+
|
|
6636
|
+
ADR-028: a `.js` metrics dict has `ast_nodes: None` (no stdlib JS AST) -- when either side
|
|
6637
|
+
lacks it, the distance is the mean over the dimensions BOTH sides have (LOC + import
|
|
6638
|
+
Jaccard only). A cross-language pair never occurs (one archetype, one language), so this
|
|
6639
|
+
never mixes a Python 3-dimensional distance with a JS 2-dimensional one within a pair."""
|
|
6640
|
+
ia, ib = set(a["imports"]), set(b["imports"])
|
|
6641
|
+
jac = (len(ia & ib) / len(ia | ib)) if (ia or ib) else 1.0
|
|
6642
|
+
dloc = abs(a["loc"] - b["loc"]) / max(a["loc"], b["loc"], 1)
|
|
6643
|
+
if a.get("ast_nodes") is None or b.get("ast_nodes") is None:
|
|
6644
|
+
return (dloc + (1.0 - jac)) / 2.0
|
|
6645
|
+
dast = abs(a["ast_nodes"] - b["ast_nodes"]) / max(a["ast_nodes"], b["ast_nodes"], 1)
|
|
6646
|
+
return (dloc + dast + (1.0 - jac)) / 3.0
|
|
5965
6647
|
|
|
5966
6648
|
|
|
5967
6649
|
def _oracle_hash(arm_dir):
|
|
@@ -5987,8 +6669,9 @@ def cmd_lang_compare(args):
|
|
|
5987
6669
|
"the comparison requires the SAME withheld oracle; behaviour must be held fixed "
|
|
5988
6670
|
"while only the authoring changes." % (hf[:12], hc[:12]), file=sys.stderr)
|
|
5989
6671
|
sys.exit(2)
|
|
5990
|
-
free, ef = _lang_arm_metrics(args.free)
|
|
5991
|
-
ctrl, ec = _lang_arm_metrics(args.controlled)
|
|
6672
|
+
free, ef, has_js_f = _lang_arm_metrics(args.free)
|
|
6673
|
+
ctrl, ec, has_js_c = _lang_arm_metrics(args.controlled)
|
|
6674
|
+
has_js = has_js_f or has_js_c
|
|
5992
6675
|
if ef or ec:
|
|
5993
6676
|
for e in ef + ec:
|
|
5994
6677
|
print("[qa_ledger] lang-compare: %s" % e, file=sys.stderr)
|
|
@@ -6027,7 +6710,7 @@ def cmd_lang_compare(args):
|
|
|
6027
6710
|
"margin": _LANG_MARGIN, "passrate_margin": _LANG_PR_MARGIN, "oracle_shared": True}
|
|
6028
6711
|
if args.out:
|
|
6029
6712
|
with open(args.out, "w", encoding="utf-8", newline="\n") as fh:
|
|
6030
|
-
fh.write(_render_lang_md(report))
|
|
6713
|
+
fh.write(_render_lang_md(report, has_js))
|
|
6031
6714
|
if args.json:
|
|
6032
6715
|
print(json.dumps(report, indent=2, ensure_ascii=False))
|
|
6033
6716
|
else:
|
|
@@ -6051,7 +6734,7 @@ def cmd_lang_compare(args):
|
|
|
6051
6734
|
sys.exit(0)
|
|
6052
6735
|
|
|
6053
6736
|
|
|
6054
|
-
def _render_lang_md(r):
|
|
6737
|
+
def _render_lang_md(r, has_js=False):
|
|
6055
6738
|
d = r["delta"]
|
|
6056
6739
|
lines = ["<!-- GENERATED by qa_ledger.py lang-compare (ADR-019) -- measured run; do not "
|
|
6057
6740
|
"hand-edit. -->", "", "# CONTROLLED-LANGUAGE-REPORT", "",
|
|
@@ -6093,11 +6776,16 @@ def _render_lang_md(r):
|
|
|
6093
6776
|
"arm's low variance was convergence on a shared reading (right or "
|
|
6094
6777
|
"wrong): two compilers resolving the same ambiguity the same way read "
|
|
6095
6778
|
"as agreement, and a rewrite that separates them raises variance "
|
|
6096
|
-
"while moving behaviour."
|
|
6779
|
+
"while moving behaviour. Read any variance number here against the "
|
|
6780
|
+
"entry's intra-model noise floor (bench-r2, ADR-027): an inter-compiler "
|
|
6781
|
+
"distance below that floor is not evidence of convergence."
|
|
6097
6782
|
% r.get("passrate_margin", 0.02)}[r["verdict"]],
|
|
6098
6783
|
"", "*The oracle is byte-identical across both arms, so behaviour is held fixed; "
|
|
6099
6784
|
"the only variable is the authoring discipline. The judgement of \"same semantic "
|
|
6100
6785
|
"content\" between the two canonical packages is human — a stated limitation.*", ""]
|
|
6786
|
+
if has_js:
|
|
6787
|
+
lines += ["*JS pairs use a 2-dimensional distance (LOC + import Jaccard; no stdlib JS "
|
|
6788
|
+
"AST).*", ""]
|
|
6101
6789
|
return "\n".join(lines)
|
|
6102
6790
|
|
|
6103
6791
|
|
|
@@ -10246,6 +10934,28 @@ def build_parser():
|
|
|
10246
10934
|
"(read-only), including stale verdicts whose obs no longer exists")
|
|
10247
10935
|
pbc.set_defaults(func=cmd_bench_curate)
|
|
10248
10936
|
|
|
10937
|
+
pbr = sub.add_parser(
|
|
10938
|
+
"bench-r2",
|
|
10939
|
+
help="intra-model variance (ADR-027): for every bench entry with an r2/ second run of "
|
|
10940
|
+
"the same models, the run1-vs-run2 structural distance via the SAME function the "
|
|
10941
|
+
"bench uses between compilers, oracle stability, and a per-entry SIGNAL/NOISY/NOISE "
|
|
10942
|
+
"class; the noise floor under every variance claim; advisory, deterministic, no LLM")
|
|
10943
|
+
pbr.add_argument("--dir", required=True, help="the bench directory")
|
|
10944
|
+
pbr.add_argument("--out", default=None, help="write DIAMOND-BENCH-R2.md here")
|
|
10945
|
+
pbr.add_argument("--json", action="store_true")
|
|
10946
|
+
pbr.set_defaults(func=cmd_bench_r2)
|
|
10947
|
+
|
|
10948
|
+
prt = sub.add_parser(
|
|
10949
|
+
"bench-roundtrip",
|
|
10950
|
+
help="round-trip recoverability (ADR-030): per compilation, which pinned-IR nodes the "
|
|
10951
|
+
"mechanical reverse organs can anchor in the compiled artifact -- static id "
|
|
10952
|
+
"reference, validated manifest claim, tagged withheld-oracle behaviour -- and the "
|
|
10953
|
+
"unanchored gap list; regenerates NO IR, infers NO spec; advisory, deterministic")
|
|
10954
|
+
prt.add_argument("--dir", required=True, help="the bench directory")
|
|
10955
|
+
prt.add_argument("--out", default=None, help="write DIAMOND-ROUNDTRIP.md here")
|
|
10956
|
+
prt.add_argument("--json", action="store_true")
|
|
10957
|
+
prt.set_defaults(func=cmd_bench_roundtrip)
|
|
10958
|
+
|
|
10249
10959
|
plc = sub.add_parser(
|
|
10250
10960
|
"lang-compare",
|
|
10251
10961
|
help="controlled-language arm (ADR-019): compare a FREE-prose arm and an EARS+STE arm of "
|