@andresmassello/uscha 1.84.0 → 1.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +1 -1
- package/uscha-kit/.claude/skills/uscha-devloop/qa_ledger.py +522 -51
- package/uscha-kit/.claude-plugin/plugin.json +2 -2
- package/uscha-kit/.codex-plugin/plugin.json +1 -1
- package/uscha-kit/README.md +1 -1
- package/uscha-kit/VERSION +1 -1
- package/uscha-kit/reports/junit/.js-cases.json +1 -0
- package/uscha-kit/reports/junit/.multi-cases.json +1 -0
- package/uscha-kit/reports/junit/.rt-cases.json +1 -0
- package/uscha-kit/skills/uscha-devloop/qa_ledger.py +522 -51
- package/uscha-kit/uscha.config.json +1 -1
|
@@ -4007,6 +4007,86 @@ def _extract_static_py(repo_path, tracked):
|
|
|
4007
4007
|
return obs, unsupported
|
|
4008
4008
|
|
|
4009
4009
|
|
|
4010
|
+
# Node one-liner (ADR-028): `require`s the impl and reports ITS OWN module.exports --
|
|
4011
|
+
# the target's own runtime reporting its own public surface, never a regex-narrated AST. A
|
|
4012
|
+
# module that guards its entry point with `require.main === module` (the SPEC contract) will
|
|
4013
|
+
# not auto-run under `require()`, since `require.main` here is this -e script, not the module.
|
|
4014
|
+
_NODE_EXPORTS_PROBE = (
|
|
4015
|
+
"const p=process.argv[1]; const m=require(require('path').resolve(p)); "
|
|
4016
|
+
"const ks=Object.keys(m).filter(k=>typeof m[k]==='function'||typeof m[k]==='object'); "
|
|
4017
|
+
"process.stdout.write(JSON.stringify(ks.sort()))"
|
|
4018
|
+
)
|
|
4019
|
+
_STATIC_JS_TIMEOUT = 15
|
|
4020
|
+
|
|
4021
|
+
|
|
4022
|
+
def _extract_static_js(repo_path, tracked):
|
|
4023
|
+
"""v0 static extractor for JS (ADR-028): the target's OWN runtime reports its own public
|
|
4024
|
+
surface via `node -e` + `Object.keys(module.exports)` -- measured, not parsed by a
|
|
4025
|
+
heuristic. Function-vs-object and the observation's line number are best-effort source
|
|
4026
|
+
regexes (Node reports WHICH names are exported; it does not report where they are
|
|
4027
|
+
defined). If node is absent, or a file fails to load/print a clean JSON array, that file's
|
|
4028
|
+
surface is UNMEASURED -- named in `unsupported`, never silently empty-as-measured.
|
|
4029
|
+
Returns (observations, unsupported) where unsupported is a list of "<file>: <reason>"."""
|
|
4030
|
+
obs, unsupported = [], []
|
|
4031
|
+
js_files = sorted(rel for rel in tracked if rel.lower().endswith(".js"))
|
|
4032
|
+
if not js_files:
|
|
4033
|
+
return obs, unsupported
|
|
4034
|
+
node = shutil.which("node")
|
|
4035
|
+
if not node:
|
|
4036
|
+
return obs, ["%s: node not on PATH" % rel for rel in js_files]
|
|
4037
|
+
for rel in js_files:
|
|
4038
|
+
full = os.path.join(repo_path, rel.replace("/", os.sep))
|
|
4039
|
+
abs_path = os.path.abspath(full)
|
|
4040
|
+
try:
|
|
4041
|
+
r = subprocess.run([node, "-e", _NODE_EXPORTS_PROBE, "--", abs_path],
|
|
4042
|
+
capture_output=True, text=True, encoding="utf-8",
|
|
4043
|
+
errors="replace", timeout=_STATIC_JS_TIMEOUT,
|
|
4044
|
+
cwd=os.path.dirname(full))
|
|
4045
|
+
except subprocess.TimeoutExpired:
|
|
4046
|
+
unsupported.append("%s: node timed out extracting exports" % rel)
|
|
4047
|
+
continue
|
|
4048
|
+
except OSError as exc:
|
|
4049
|
+
unsupported.append("%s: could not run node: %s" % (rel, exc))
|
|
4050
|
+
continue
|
|
4051
|
+
if r.returncode != 0:
|
|
4052
|
+
unsupported.append("%s: node exited %d extracting exports (%s)"
|
|
4053
|
+
% (rel, r.returncode, (r.stderr or "").strip()[:200]))
|
|
4054
|
+
continue
|
|
4055
|
+
try:
|
|
4056
|
+
names = json.loads(r.stdout)
|
|
4057
|
+
if not isinstance(names, list):
|
|
4058
|
+
raise ValueError("exports output was not a JSON list")
|
|
4059
|
+
except ValueError as exc:
|
|
4060
|
+
unsupported.append("%s: could not parse exports output: %s" % (rel, exc))
|
|
4061
|
+
continue
|
|
4062
|
+
try:
|
|
4063
|
+
with open(full, encoding="utf-8", errors="replace") as fh:
|
|
4064
|
+
src = fh.read()
|
|
4065
|
+
except OSError:
|
|
4066
|
+
src = ""
|
|
4067
|
+
src_lines = src.splitlines()
|
|
4068
|
+
for name in sorted(names):
|
|
4069
|
+
is_function = bool(re.search(r"\bfunction\s+%s\s*\(" % re.escape(name), src))
|
|
4070
|
+
lineno = 0
|
|
4071
|
+
patterns = (r"\bfunction\s+%s\b" % re.escape(name),
|
|
4072
|
+
r"\b%s\s*=" % re.escape(name),
|
|
4073
|
+
r"exports\.%s\b" % re.escape(name),
|
|
4074
|
+
r"module\.exports\b")
|
|
4075
|
+
for i, ln in enumerate(src_lines, 1):
|
|
4076
|
+
if any(re.search(p, ln) for p in patterns):
|
|
4077
|
+
lineno = i
|
|
4078
|
+
break
|
|
4079
|
+
stmt = ("%s exports function %s" % (rel, name) if is_function
|
|
4080
|
+
else "%s exports object %s" % (rel, name))
|
|
4081
|
+
prov = "%s:%d" % (rel, lineno)
|
|
4082
|
+
obs.append({"id": _obs_id("contract", stmt, prov), "type": "contract",
|
|
4083
|
+
"statement": stmt, "evidence_class": "static",
|
|
4084
|
+
"provenance": {"files": [prov],
|
|
4085
|
+
"derivation": "node -e Object.keys(module.exports)",
|
|
4086
|
+
"tool": "qa_ledger-static-js"}})
|
|
4087
|
+
return obs, unsupported
|
|
4088
|
+
|
|
4089
|
+
|
|
4010
4090
|
def _under_bound(rel, bound):
|
|
4011
4091
|
return bound is None or rel == bound or rel.startswith(bound + "/")
|
|
4012
4092
|
|
|
@@ -5307,26 +5387,69 @@ def cmd_compile_ingest(args):
|
|
|
5307
5387
|
_BOOTSTRAP_CASE_TIMEOUT = 15 # a compiled hook must decide fast
|
|
5308
5388
|
|
|
5309
5389
|
|
|
5390
|
+
def _entry_unit(source_list):
|
|
5391
|
+
"""The unit the oracle runs (ADR-029): the one whose basename starts with `cli.` if any,
|
|
5392
|
+
else the first declared source unit. Single-unit compilations resolve to their only unit,
|
|
5393
|
+
unchanged. Returns None when there is no usable unit."""
|
|
5394
|
+
units = [u.get("unit") for u in (source_list or []) if isinstance(u, dict) and u.get("unit")]
|
|
5395
|
+
if not units:
|
|
5396
|
+
return None
|
|
5397
|
+
for u in units:
|
|
5398
|
+
if os.path.basename(u).lower().startswith("cli."):
|
|
5399
|
+
return u
|
|
5400
|
+
return units[0]
|
|
5401
|
+
|
|
5402
|
+
|
|
5403
|
+
def _static_surface_for(cd, unit):
|
|
5404
|
+
"""Route the reverse-discovery organ by the unit's extension (ADR-028/029)."""
|
|
5405
|
+
if os.path.splitext(unit)[1].lower() == ".js":
|
|
5406
|
+
return _extract_static_js(cd, [unit])
|
|
5407
|
+
so, _n = _extract_static_py(cd, [unit])
|
|
5408
|
+
return so, []
|
|
5409
|
+
|
|
5410
|
+
|
|
5411
|
+
def _impl_interpreter(impl_path):
|
|
5412
|
+
"""Resolve the interpreter argv prefix for one implementation file by extension (ADR-028):
|
|
5413
|
+
`.py` runs under this same Python (unchanged); `.js` runs under `node`, resolved from PATH.
|
|
5414
|
+
Returns None when the extension's interpreter cannot be resolved (today: node absent) --
|
|
5415
|
+
the caller must treat that as UNMEASURED, never a fake red or green."""
|
|
5416
|
+
if impl_path.lower().endswith(".js"):
|
|
5417
|
+
node = shutil.which("node")
|
|
5418
|
+
return [node] if node else None
|
|
5419
|
+
return [sys.executable]
|
|
5420
|
+
|
|
5421
|
+
|
|
5310
5422
|
def _run_oracle_case(impl_path, case):
|
|
5311
5423
|
"""Run ONE withheld oracle case against a compiled implementation: feed the case's stdin
|
|
5312
|
-
(raw_stdin verbatim if present, else json.dumps(payload)) to
|
|
5313
|
-
|
|
5314
|
-
oracle is a `measured` fact, never an LLM
|
|
5424
|
+
(raw_stdin verbatim if present, else json.dumps(payload)) to the impl's interpreter (ADR-028:
|
|
5425
|
+
`python` for `.py`, `node` for `.js`), and check its result against whichever expectations
|
|
5426
|
+
the case declares. Deterministic execution -- the oracle is a `measured` fact, never an LLM
|
|
5427
|
+
judgment. Returns a per-case result dict.
|
|
5315
5428
|
|
|
5316
5429
|
A case may assert any of: `expected_exit` (process exit code -- the M4 guard's contract),
|
|
5317
5430
|
`expected_stdout` (the program's stdout, compared stripped -- for archetypes that COMPUTE an
|
|
5318
5431
|
output, e.g. a parser or transformer), and `expected_json` (stdout parsed as JSON and
|
|
5319
5432
|
compared structurally, so an output whose formatting is free but whose value is fixed is not
|
|
5320
5433
|
penalised for whitespace or key order). The case passes iff EVERY declared expectation holds;
|
|
5321
|
-
a case that declares none proves nothing and fails.
|
|
5434
|
+
a case that declares none proves nothing and fails.
|
|
5435
|
+
|
|
5436
|
+
If the impl's interpreter cannot be resolved (a `.js` impl and no `node` on PATH), the case
|
|
5437
|
+
is UNMEASURED -- never a fake red or green (ADR-028)."""
|
|
5322
5438
|
if "raw_stdin" in case:
|
|
5323
5439
|
stdin = case["raw_stdin"]
|
|
5324
5440
|
else:
|
|
5325
5441
|
stdin = json.dumps(case.get("payload"))
|
|
5442
|
+
interp = _impl_interpreter(impl_path)
|
|
5443
|
+
if interp is None:
|
|
5444
|
+
return {"name": case.get("name"), "expected": case.get("expected_exit"), "got": None,
|
|
5445
|
+
"ok": False, "error": "node not on PATH", "unmeasured": True}
|
|
5326
5446
|
try:
|
|
5327
|
-
|
|
5328
|
-
|
|
5329
|
-
|
|
5447
|
+
# cwd = the impl's own directory (ADR-029): a multi-unit compilation imports its
|
|
5448
|
+
# sibling modules by bare name; single-unit impls are unaffected by their cwd
|
|
5449
|
+
r = subprocess.run(interp + [os.path.abspath(impl_path)], input=stdin,
|
|
5450
|
+
capture_output=True, text=True, encoding="utf-8", errors="replace",
|
|
5451
|
+
timeout=_BOOTSTRAP_CASE_TIMEOUT,
|
|
5452
|
+
cwd=os.path.dirname(os.path.abspath(impl_path)) or None)
|
|
5330
5453
|
got, out, err = r.returncode, r.stdout, None
|
|
5331
5454
|
except subprocess.TimeoutExpired:
|
|
5332
5455
|
got, out, err = None, "", "timeout"
|
|
@@ -5369,22 +5492,30 @@ def cmd_bootstrap_oracle(args):
|
|
|
5369
5492
|
file=sys.stderr)
|
|
5370
5493
|
sys.exit(2)
|
|
5371
5494
|
results = [_run_oracle_case(args.impl, c) for c in cases]
|
|
5495
|
+
unmeasured = bool(results) and all(r.get("unmeasured") for r in results)
|
|
5372
5496
|
passed = sum(1 for r in results if r["ok"])
|
|
5373
5497
|
failed = [r for r in results if not r["ok"]]
|
|
5374
5498
|
report = {"impl": args.impl, "oracle": args.oracle, "total": len(results),
|
|
5375
5499
|
"passed": passed, "failed": len(failed),
|
|
5376
|
-
"oracle_green": not failed, "results": results}
|
|
5500
|
+
"oracle_green": (None if unmeasured else not failed), "results": results}
|
|
5501
|
+
if unmeasured:
|
|
5502
|
+
report["unmeasured"] = results[0]["error"]
|
|
5377
5503
|
if args.ledger and args.repo:
|
|
5378
5504
|
ledger = _load(args.ledger)
|
|
5379
5505
|
_repo_node(ledger, args.repo)
|
|
5380
5506
|
rec = {"impl": os.path.basename(args.impl), "oracle": os.path.basename(args.oracle),
|
|
5381
5507
|
"total": len(results), "passed": passed, "failed": len(failed),
|
|
5382
|
-
"oracle_green": not failed,
|
|
5508
|
+
"oracle_green": (None if unmeasured else not failed),
|
|
5383
5509
|
"failing": [r["name"] for r in failed], "at": _now()}
|
|
5510
|
+
if unmeasured:
|
|
5511
|
+
rec["unmeasured"] = results[0]["error"]
|
|
5384
5512
|
ledger.setdefault("bootstrap_oracle", []).append(rec)
|
|
5385
5513
|
_save(args.ledger, ledger)
|
|
5386
5514
|
if args.json:
|
|
5387
5515
|
print(json.dumps(report, indent=2, ensure_ascii=False))
|
|
5516
|
+
elif unmeasured:
|
|
5517
|
+
print("BOOTSTRAP-ORACLE %s: UNMEASURED -- %s"
|
|
5518
|
+
% (os.path.basename(args.impl), report["unmeasured"]))
|
|
5388
5519
|
else:
|
|
5389
5520
|
print("BOOTSTRAP-ORACLE %s: %d/%d cases pass -- %s"
|
|
5390
5521
|
% (os.path.basename(args.impl), passed, len(results),
|
|
@@ -5394,17 +5525,60 @@ def cmd_bootstrap_oracle(args):
|
|
|
5394
5525
|
print(" x %s: expected exit %s, got %s%s"
|
|
5395
5526
|
% (r["name"], r["expected"], r["got"],
|
|
5396
5527
|
" (%s)" % r["error"] if r["error"] else ""))
|
|
5397
|
-
sys.exit(0 if not failed else 1)
|
|
5528
|
+
sys.exit(2 if unmeasured else (0 if not failed else 1))
|
|
5529
|
+
|
|
5530
|
+
|
|
5531
|
+
_JS_IMPORT_RE = re.compile(
|
|
5532
|
+
r"""require\(\s*['"]([^'"]+)['"]\s*\)"""
|
|
5533
|
+
r"""|import\s+(?:.+?\s+from\s+)?['"]([^'"]+)['"]"""
|
|
5534
|
+
)
|
|
5535
|
+
|
|
5536
|
+
|
|
5537
|
+
def _impl_metrics_js(path, src):
|
|
5538
|
+
"""JS structural fingerprint (ADR-028), honestly NARROWER than Python's: `loc` (non-blank,
|
|
5539
|
+
non-`//` lines, with `/*..*/` blocks toggled by a simple per-line state machine) and
|
|
5540
|
+
`imports` (require()/import specifiers -- a lexical fact over string literals, not a
|
|
5541
|
+
narrated structure) are measured; `ast_nodes`/`functions`/`classes` are UNMEASURED (None) --
|
|
5542
|
+
no stdlib JS AST exists and the engine does not invent one."""
|
|
5543
|
+
loc = 0
|
|
5544
|
+
in_block = False
|
|
5545
|
+
for ln in src.splitlines():
|
|
5546
|
+
s = ln.strip()
|
|
5547
|
+
if in_block:
|
|
5548
|
+
if "*/" in s:
|
|
5549
|
+
in_block = False
|
|
5550
|
+
continue
|
|
5551
|
+
if not s:
|
|
5552
|
+
continue
|
|
5553
|
+
if s.startswith("/*"):
|
|
5554
|
+
if "*/" not in s[2:]:
|
|
5555
|
+
in_block = True
|
|
5556
|
+
continue
|
|
5557
|
+
if s.startswith("//"):
|
|
5558
|
+
continue
|
|
5559
|
+
loc += 1
|
|
5560
|
+
imports = set()
|
|
5561
|
+
for m in _JS_IMPORT_RE.finditer(src):
|
|
5562
|
+
spec = m.group(1) or m.group(2)
|
|
5563
|
+
if spec:
|
|
5564
|
+
imports.add(spec)
|
|
5565
|
+
return {"path": path, "loc": loc, "ast_nodes": None, "functions": None,
|
|
5566
|
+
"classes": None, "imports": sorted(imports),
|
|
5567
|
+
"sha256": hashlib.sha256(src.encode("utf-8")).hexdigest()}
|
|
5398
5568
|
|
|
5399
5569
|
|
|
5400
5570
|
def _impl_metrics(path):
|
|
5401
5571
|
"""Structural fingerprint of one implementation: physical LOC, AST node count, function
|
|
5402
|
-
and class counts, and the set of imported top-level modules. Deterministic; no judgment.
|
|
5572
|
+
and class counts, and the set of imported top-level modules. Deterministic; no judgment.
|
|
5573
|
+
Routed by extension (ADR-028): `.py` unchanged; `.js` delegates to `_impl_metrics_js`,
|
|
5574
|
+
which returns the same keys with `ast_nodes`/`functions`/`classes` UNMEASURED (None)."""
|
|
5403
5575
|
try:
|
|
5404
5576
|
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
5405
5577
|
src = fh.read()
|
|
5406
5578
|
except OSError as exc:
|
|
5407
5579
|
return {"path": path, "error": "unreadable: %s" % exc}
|
|
5580
|
+
if path.lower().endswith(".js"):
|
|
5581
|
+
return _impl_metrics_js(path, src)
|
|
5408
5582
|
loc = sum(1 for ln in src.splitlines() if ln.strip())
|
|
5409
5583
|
try:
|
|
5410
5584
|
tree = ast.parse(src)
|
|
@@ -5531,8 +5705,14 @@ def _bench_curation_map(records):
|
|
|
5531
5705
|
def _bench_oracle_all(impl_path, cases):
|
|
5532
5706
|
results = [_run_oracle_case(impl_path, c) for c in cases]
|
|
5533
5707
|
passed = sum(1 for r in results if r["ok"])
|
|
5534
|
-
|
|
5535
|
-
|
|
5708
|
+
out = {"passed": passed, "total": len(results), "green": passed == len(results),
|
|
5709
|
+
"failing": [r["name"] for r in results if not r["ok"]]}
|
|
5710
|
+
# every case UNMEASURED (ADR-028: a `.js` impl with no `node` on PATH) is NOT a fake red --
|
|
5711
|
+
# name the reason and let the caller treat it as absent, never as a scored FAIL.
|
|
5712
|
+
if results and all(r.get("unmeasured") for r in results):
|
|
5713
|
+
out["unmeasured"] = results[0]["error"]
|
|
5714
|
+
out["green"] = False
|
|
5715
|
+
return out
|
|
5536
5716
|
|
|
5537
5717
|
|
|
5538
5718
|
def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
@@ -5564,15 +5744,16 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
|
5564
5744
|
with open(cj, encoding="utf-8-sig") as fh:
|
|
5565
5745
|
c = json.load(fh)
|
|
5566
5746
|
src = c.get("source") or []
|
|
5567
|
-
unit = src
|
|
5747
|
+
unit = _entry_unit(src)
|
|
5748
|
+
all_units = [u.get("unit") for u in src if isinstance(u, dict) and u.get("unit")]
|
|
5568
5749
|
model = (c.get("compilation_report") or {}).get("model")
|
|
5569
5750
|
except (OSError, ValueError, AttributeError, IndexError):
|
|
5570
|
-
|
|
5751
|
+
all_units = []
|
|
5571
5752
|
impl = os.path.join(cd, unit.replace("/", os.sep)) if unit else None
|
|
5572
5753
|
ores = (_bench_oracle_all(impl, cases) if impl and os.path.isfile(impl)
|
|
5573
5754
|
else {"passed": 0, "total": len(cases), "green": False, "failing": ["<no impl>"]})
|
|
5574
|
-
comp_rec = {"dir": d, "model": model, "impl": impl,
|
|
5575
|
-
"compile_valid": not errors, "oracle": ores}
|
|
5755
|
+
comp_rec = {"dir": d, "model": model, "impl": impl, "entry_unit": unit,
|
|
5756
|
+
"units": len(all_units), "compile_valid": not errors, "oracle": ores}
|
|
5576
5757
|
if fidelity and impl and os.path.isfile(impl) and unit:
|
|
5577
5758
|
# the per-compiler fidelity descriptor (ADR-022): the M1 static extractor applied
|
|
5578
5759
|
# to the compiled artifact -- reverse discovery per compiler. Advisory by
|
|
@@ -5597,20 +5778,36 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
|
5597
5778
|
units_f.add(u2["unit"])
|
|
5598
5779
|
except AttributeError:
|
|
5599
5780
|
pass
|
|
5600
|
-
|
|
5781
|
+
# ADR-028: the static extractor is routed by the source unit's extension -- `.py`
|
|
5782
|
+
# via the AST (unchanged), `.js` via the target's own `node` runtime. `uns_f` is a
|
|
5783
|
+
# list of "<file>: <reason>" for JS (an int count for Python, discarded either way).
|
|
5784
|
+
sobs, uns_f = [], []
|
|
5785
|
+
for u_all in (all_units or [unit]):
|
|
5786
|
+
so, un = _static_surface_for(cd, u_all)
|
|
5787
|
+
sobs.extend(so)
|
|
5788
|
+
uns_f.extend(un)
|
|
5601
5789
|
fn_names, cls_names = [], []
|
|
5602
5790
|
for o2 in sobs:
|
|
5603
|
-
m2 = re.search(r"defines function (\w+)
|
|
5791
|
+
m2 = re.search(r"defines function (\w+)|exports function (\w+)",
|
|
5792
|
+
o2.get("statement", ""))
|
|
5604
5793
|
if m2:
|
|
5605
|
-
fn_names.append(m2.group(1))
|
|
5606
|
-
m2 = re.search(r"defines class (\w+)
|
|
5794
|
+
fn_names.append(m2.group(1) or m2.group(2))
|
|
5795
|
+
m2 = re.search(r"defines class (\w+)|exports object (\w+)",
|
|
5796
|
+
o2.get("statement", ""))
|
|
5607
5797
|
if m2:
|
|
5608
|
-
cls_names.append(m2.group(1))
|
|
5798
|
+
cls_names.append(m2.group(1) or m2.group(2))
|
|
5609
5799
|
unex = sorted(u for u in units_f if u not in traced_f)
|
|
5800
|
+
if not sobs and uns_f:
|
|
5801
|
+
# node absent (or every JS file failed to load): the surface is UNMEASURED,
|
|
5802
|
+
# never a silent "0 functions, 0 classes" that reads as a measured empty
|
|
5803
|
+
# surface (ADR-028 -- absence named, same discipline as curation_closure below).
|
|
5804
|
+
static_surface = "UNMEASURED: %s" % "; ".join(uns_f)
|
|
5805
|
+
else:
|
|
5806
|
+
static_surface = {"functions": len(fn_names), "classes": len(cls_names),
|
|
5807
|
+
"names": sorted(fn_names + cls_names)}
|
|
5610
5808
|
comp_rec["fidelity"] = {
|
|
5611
5809
|
"trace_coverage": round(len(covered_f) / max(len(node_ids_f), 1), 3),
|
|
5612
|
-
"static_surface":
|
|
5613
|
-
"names": sorted(fn_names + cls_names)},
|
|
5810
|
+
"static_surface": static_surface,
|
|
5614
5811
|
"oracle_passrate": (round(ores["passed"] / ores["total"], 3)
|
|
5615
5812
|
if ores["total"] else None),
|
|
5616
5813
|
"unexplained_share": round(len(unex) / max(len(units_f), 1), 3),
|
|
@@ -5634,9 +5831,17 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
|
5634
5831
|
stub_dir = os.path.join(entry_dir, "stub")
|
|
5635
5832
|
stub_green = False
|
|
5636
5833
|
if os.path.isdir(stub_dir):
|
|
5637
|
-
stubs = sorted(f for f in os.listdir(stub_dir) if f.endswith(".py"))
|
|
5638
|
-
if stubs
|
|
5639
|
-
|
|
5834
|
+
stubs = sorted(f for f in os.listdir(stub_dir) if f.endswith((".py", ".js")))
|
|
5835
|
+
stub_path = os.path.join(stub_dir, stubs[0]) if stubs else None
|
|
5836
|
+
if stub_path is None and os.path.isdir(os.path.join(stub_dir, "source")):
|
|
5837
|
+
# multi-unit entries (ADR-029): the stub is a directory shaped like a compilation;
|
|
5838
|
+
# its entry unit is the cli.* under source/
|
|
5839
|
+
cands = sorted(f for f in os.listdir(os.path.join(stub_dir, "source"))
|
|
5840
|
+
if f.lower().startswith("cli.") and f.endswith((".py", ".js")))
|
|
5841
|
+
if cands:
|
|
5842
|
+
stub_path = os.path.join(stub_dir, "source", cands[0])
|
|
5843
|
+
if stub_path:
|
|
5844
|
+
sres = _bench_oracle_all(stub_path, cases)
|
|
5640
5845
|
stub_green = sres["green"]
|
|
5641
5846
|
rec["discrimination"] = {"stub_passed": sres["passed"], "total": sres["total"],
|
|
5642
5847
|
"stub_green": stub_green}
|
|
@@ -5651,7 +5856,14 @@ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
|
5651
5856
|
# distinct is None when variance could not be computed (fewer than 2 resolvable impl files),
|
|
5652
5857
|
# which is NOT the same as a byte-identical convergence -- keep the two reasons apart.
|
|
5653
5858
|
distinct = rec["variance"]["all_distinct"] if rec["variance"] else None
|
|
5654
|
-
|
|
5859
|
+
# ADR-028: node absent for a `.js` entry means EVERY case of that compilation came back
|
|
5860
|
+
# UNMEASURED -- never score that as a FAIL (nor a PASS); the entry stays PENDING with the
|
|
5861
|
+
# reason named, ahead of every other verdict check.
|
|
5862
|
+
unmeasured_js = next((i["oracle"]["unmeasured"] for i in impls
|
|
5863
|
+
if i["oracle"].get("unmeasured")), None)
|
|
5864
|
+
if unmeasured_js:
|
|
5865
|
+
rec["verdict"], rec["reason"] = "PENDING", "%s -- JS entry unmeasured" % unmeasured_js
|
|
5866
|
+
elif stub_green:
|
|
5655
5867
|
rec["verdict"], rec["reason"] = "FAIL", ("oracle satisfied by a degenerate stub -- not "
|
|
5656
5868
|
"discriminating; the entry proves nothing")
|
|
5657
5869
|
elif not all_valid:
|
|
@@ -5720,15 +5932,23 @@ def _render_bench_md(table, anon, recs):
|
|
|
5720
5932
|
if isinstance(cur_v, float):
|
|
5721
5933
|
cur_v = "%.3f (judged %d/%d)" % (cur_v, fd["curation"]["judged"],
|
|
5722
5934
|
fd["curation"]["total"])
|
|
5723
|
-
|
|
5935
|
+
ss = fd["static_surface"]
|
|
5936
|
+
# ADR-028: `static_surface` is UNMEASURED (a named string) when a JS entry's
|
|
5937
|
+
# extractor could not run (node absent) -- never a silent "0 fn / 0 cls".
|
|
5938
|
+
surface_txt = ("%d fn / %d cls" % (ss["functions"], ss["classes"])
|
|
5939
|
+
if isinstance(ss, dict) else ss)
|
|
5940
|
+
lines.append("- fidelity `%s` (%s): trace %.2f · surface %s · "
|
|
5724
5941
|
"oracle %s · unexplained %.2f · curation %s" % (
|
|
5725
5942
|
i["dir"], anon.get(i["model"], i["model"] or "?"),
|
|
5726
|
-
fd["trace_coverage"],
|
|
5727
|
-
fd["static_surface"]["classes"],
|
|
5943
|
+
fd["trace_coverage"], surface_txt,
|
|
5728
5944
|
("%.3f" % fd["oracle_passrate"])
|
|
5729
5945
|
if fd["oracle_passrate"] is not None else "n/a",
|
|
5730
5946
|
fd["unexplained_share"], cur_v))
|
|
5731
5947
|
lines.append("")
|
|
5948
|
+
if any((i.get("impl") or "").lower().endswith(".js")
|
|
5949
|
+
for r in recs for i in r["compilations"]):
|
|
5950
|
+
lines += ["*JS pairs use a 2-dimensional distance (LOC + import Jaccard; no stdlib JS "
|
|
5951
|
+
"AST).*", ""]
|
|
5732
5952
|
return "\n".join(lines)
|
|
5733
5953
|
|
|
5734
5954
|
|
|
@@ -5817,11 +6037,11 @@ def _r2_entry(entry_dir, name):
|
|
|
5817
6037
|
r2 = os.path.join(entry_dir, _R2_DIR)
|
|
5818
6038
|
if not os.path.isdir(r2):
|
|
5819
6039
|
rec["reason"] = "no r2/ directory -- intra-model variance not measured"
|
|
5820
|
-
return rec
|
|
6040
|
+
return rec, False
|
|
5821
6041
|
rec["has_r2"] = True
|
|
5822
6042
|
if ir_graph is None or ir_errors or not cases:
|
|
5823
6043
|
rec["reason"] = "entry incomplete (missing/invalid IR or oracle)"
|
|
5824
|
-
return rec
|
|
6044
|
+
return rec, False
|
|
5825
6045
|
|
|
5826
6046
|
def load_run(cd):
|
|
5827
6047
|
cj = os.path.join(cd, "COMPILATION.json")
|
|
@@ -5831,7 +6051,7 @@ def _r2_entry(entry_dir, name):
|
|
|
5831
6051
|
try:
|
|
5832
6052
|
with open(cj, encoding="utf-8-sig") as fh:
|
|
5833
6053
|
c = json.load(fh)
|
|
5834
|
-
unit = (c.get("source") or [
|
|
6054
|
+
unit = _entry_unit(c.get("source") or [])
|
|
5835
6055
|
model = (c.get("compilation_report") or {}).get("model")
|
|
5836
6056
|
except (OSError, ValueError, AttributeError, IndexError):
|
|
5837
6057
|
return None
|
|
@@ -5849,6 +6069,7 @@ def _r2_entry(entry_dir, name):
|
|
|
5849
6069
|
|
|
5850
6070
|
intra = []
|
|
5851
6071
|
run1_metrics = []
|
|
6072
|
+
has_js = False # footer disclosure only; not serialized
|
|
5852
6073
|
for d in sorted(os.listdir(entry_dir)):
|
|
5853
6074
|
if not d.startswith("c-") or not os.path.isdir(os.path.join(entry_dir, d)):
|
|
5854
6075
|
continue
|
|
@@ -5857,10 +6078,14 @@ def _r2_entry(entry_dir, name):
|
|
|
5857
6078
|
continue
|
|
5858
6079
|
if r1["metrics"]:
|
|
5859
6080
|
run1_metrics.append(r1["metrics"])
|
|
6081
|
+
if r1["metrics"]["path"].lower().endswith(".js"):
|
|
6082
|
+
has_js = True
|
|
5860
6083
|
r2run = load_run(os.path.join(r2, d))
|
|
5861
6084
|
if r2run is None:
|
|
5862
6085
|
rec["models"].append({"dir": d, "model": r1["model"], "r2": "absent"})
|
|
5863
6086
|
continue
|
|
6087
|
+
if r2run["metrics"] and r2run["metrics"]["path"].lower().endswith(".js"):
|
|
6088
|
+
has_js = True
|
|
5864
6089
|
dist = (_struct_distance(r1["metrics"], r2run["metrics"])
|
|
5865
6090
|
if r1["metrics"] and r2run["metrics"] else None)
|
|
5866
6091
|
stable = r1["case_vector"] == r2run["case_vector"]
|
|
@@ -5893,7 +6118,7 @@ def _r2_entry(entry_dir, name):
|
|
|
5893
6118
|
# missing COMPILATION.json in every r2 model): absent, named -- never a silent None
|
|
5894
6119
|
rec["reason"] = ("r2/ present but no model pair measurable (no parseable run-1+run-2 "
|
|
5895
6120
|
"source for any model) -- intra-model variance not measured")
|
|
5896
|
-
return rec
|
|
6121
|
+
return rec, has_js
|
|
5897
6122
|
|
|
5898
6123
|
|
|
5899
6124
|
_R2_VERDICT_TEXT = {
|
|
@@ -5910,7 +6135,7 @@ _R2_VERDICT_TEXT = {
|
|
|
5910
6135
|
}
|
|
5911
6136
|
|
|
5912
6137
|
|
|
5913
|
-
def _render_r2_md(recs, agg):
|
|
6138
|
+
def _render_r2_md(recs, agg, has_js=False):
|
|
5914
6139
|
lines = ["<!-- GENERATED by qa_ledger.py bench-r2 (ADR-027) -- measured run; do not hand-edit. -->",
|
|
5915
6140
|
"", "# DIAMOND-BENCH-R2 -- intra-model variance, the noise floor under the bench", "",
|
|
5916
6141
|
"For every archetype with a second blind run (`r2/`) of the SAME model on the SAME "
|
|
@@ -5951,6 +6176,9 @@ def _render_r2_md(recs, agg):
|
|
|
5951
6176
|
"changing -- classes are stable across 3.8/3.13 on this bench, ratios are not "
|
|
5952
6177
|
"point-precise, and an entry within ~0.25 of a threshold should be read as "
|
|
5953
6178
|
"borderline.*", ""]
|
|
6179
|
+
if has_js:
|
|
6180
|
+
lines += ["*JS pairs use a 2-dimensional distance (LOC + import Jaccard; no stdlib JS "
|
|
6181
|
+
"AST).*", ""]
|
|
5954
6182
|
return "\n".join(lines)
|
|
5955
6183
|
|
|
5956
6184
|
|
|
@@ -5963,7 +6191,9 @@ def cmd_bench_r2(args):
|
|
|
5963
6191
|
if not entries:
|
|
5964
6192
|
print("[qa_ledger] bench-r2: no entries under %s" % args.dir, file=sys.stderr)
|
|
5965
6193
|
sys.exit(2)
|
|
5966
|
-
|
|
6194
|
+
pairs = [_r2_entry(os.path.join(args.dir, e), e) for e in entries]
|
|
6195
|
+
recs = [p[0] for p in pairs]
|
|
6196
|
+
has_js = any(p[1] for p in pairs)
|
|
5967
6197
|
measured = [r for r in recs if r["class"] is not None]
|
|
5968
6198
|
ratios = [r["intra_over_inter"] for r in measured]
|
|
5969
6199
|
reruns = [m for r in recs if r["has_r2"] for m in r["models"] if m.get("r2") != "absent"]
|
|
@@ -5982,7 +6212,7 @@ def cmd_bench_r2(args):
|
|
|
5982
6212
|
top = max(c for c, _ in counts)
|
|
5983
6213
|
# ties resolve toward the more cautious reading (NOISE > NOISY > SIGNAL)
|
|
5984
6214
|
agg["verdict"] = [n for c, n in reversed(counts) if c == top][0]
|
|
5985
|
-
md = _render_r2_md(recs, agg)
|
|
6215
|
+
md = _render_r2_md(recs, agg, has_js)
|
|
5986
6216
|
if args.out:
|
|
5987
6217
|
with open(args.out, "w", encoding="utf-8", newline="\n") as fh:
|
|
5988
6218
|
fh.write(md)
|
|
@@ -6005,6 +6235,215 @@ def cmd_bench_r2(args):
|
|
|
6005
6235
|
sys.exit(0)
|
|
6006
6236
|
|
|
6007
6237
|
|
|
6238
|
+
_RT_ID_RE = re.compile(r"(?i)\b(AC|INV|ADR)[-_][A-Z0-9-]*?\d+\b")
|
|
6239
|
+
|
|
6240
|
+
|
|
6241
|
+
def _rt_ids_in(text):
|
|
6242
|
+
"""Every canonical id (AC-*, INV-*, ADR-*) literally referenced in a text, upper-cased.
|
|
6243
|
+
ID REFERENCE only (the ADR-013 rule) -- no fuzzy or semantic matching, ever."""
|
|
6244
|
+
out = set()
|
|
6245
|
+
for m in _RT_ID_RE.finditer(text or ""):
|
|
6246
|
+
out.add(m.group(0).upper().replace("_", "-"))
|
|
6247
|
+
return out
|
|
6248
|
+
|
|
6249
|
+
|
|
6250
|
+
def _rt_read_source(cd, unit):
|
|
6251
|
+
try:
|
|
6252
|
+
with open(os.path.join(cd, unit.replace("/", os.sep)), encoding="utf-8",
|
|
6253
|
+
errors="replace") as fh:
|
|
6254
|
+
return fh.read()
|
|
6255
|
+
except OSError:
|
|
6256
|
+
return ""
|
|
6257
|
+
|
|
6258
|
+
|
|
6259
|
+
def _rt_compilation(entry_dir, cd, ir_graph, cases):
|
|
6260
|
+
"""Round-trip recoverability of ONE compilation against the pinned IR (ADR-030): for every
|
|
6261
|
+
IR node, whether the mechanical reverse organs find footing for it in the artifact --
|
|
6262
|
+
(a) static: a source unit's text or a static observation literally references the id;
|
|
6263
|
+
(b) manifest: the compiler's validated trace manifest maps the node to a unit that exists;
|
|
6264
|
+
(c) behaviour: for AC nodes, at least one withheld-oracle case tagged with the id passes
|
|
6265
|
+
(UNMEASURED when no case carries any AC tag). It regenerates NOTHING: no IR', no spec --
|
|
6266
|
+
a coverage over the human-authored IR, plus the list of nodes nothing anchors."""
|
|
6267
|
+
cj = os.path.join(cd, "COMPILATION.json")
|
|
6268
|
+
try:
|
|
6269
|
+
with open(cj, encoding="utf-8-sig") as fh:
|
|
6270
|
+
c = json.load(fh)
|
|
6271
|
+
except (OSError, ValueError):
|
|
6272
|
+
return None
|
|
6273
|
+
units = [u.get("unit") for u in (c.get("source") or []) if isinstance(u, dict) and u.get("unit")]
|
|
6274
|
+
units = [u for u in units if os.path.isfile(os.path.join(cd, u.replace("/", os.sep)))]
|
|
6275
|
+
node_ids = [n["id"] for n in (ir_graph.get("nodes") or [])]
|
|
6276
|
+
# (a) static footing: ids referenced in source text (comments/docstrings/names) or in
|
|
6277
|
+
# the static observations' statements/provenance
|
|
6278
|
+
static_ids = set()
|
|
6279
|
+
for u in units:
|
|
6280
|
+
static_ids |= _rt_ids_in(_rt_read_source(cd, u))
|
|
6281
|
+
so, _un = _static_surface_for(cd, u)
|
|
6282
|
+
for o in so:
|
|
6283
|
+
static_ids |= _rt_ids_in(o.get("statement", ""))
|
|
6284
|
+
# (b) manifest footing
|
|
6285
|
+
manifest_ids = set()
|
|
6286
|
+
for e in c.get("trace_manifest") or []:
|
|
6287
|
+
if e.get("unit") in units:
|
|
6288
|
+
for nid in e.get("implements") or []:
|
|
6289
|
+
manifest_ids.add(str(nid).upper())
|
|
6290
|
+
# (c) behaviour footing: oracle cases whose NAME carries an AC id (payload tags are not a
|
|
6291
|
+
# convention the bench oracles use today -- absence named, not faked)
|
|
6292
|
+
tagged = {}
|
|
6293
|
+
for case in cases:
|
|
6294
|
+
for cid in _rt_ids_in(case.get("name", "")):
|
|
6295
|
+
tagged.setdefault(cid, []).append(case)
|
|
6296
|
+
behaviour_measured = bool(tagged)
|
|
6297
|
+
behaviour_ids = set()
|
|
6298
|
+
entry_unit = _entry_unit(c.get("source") or [])
|
|
6299
|
+
impl = os.path.join(cd, entry_unit.replace("/", os.sep)) if entry_unit else None
|
|
6300
|
+
if behaviour_measured and impl and os.path.isfile(impl):
|
|
6301
|
+
for cid, cs in tagged.items():
|
|
6302
|
+
if any(_run_oracle_case(impl, case).get("ok") for case in cs):
|
|
6303
|
+
behaviour_ids.add(cid)
|
|
6304
|
+
per_node = []
|
|
6305
|
+
anchored = 0
|
|
6306
|
+
for nid in node_ids:
|
|
6307
|
+
a_s = nid in static_ids
|
|
6308
|
+
a_m = nid in manifest_ids
|
|
6309
|
+
a_b = (nid in behaviour_ids) if (behaviour_measured and nid.startswith("AC-")) else None
|
|
6310
|
+
anchored_any = a_s or a_m or bool(a_b)
|
|
6311
|
+
# the MEASURED footing excludes the manifest: the manifest is what the compiler
|
|
6312
|
+
# CLAIMED (and the prompt handed it the ids), so counting it as recovered would be
|
|
6313
|
+
# tautological -- the 1.000 that means nothing (found on the first run of this tool)
|
|
6314
|
+
anchored_meas = a_s or bool(a_b)
|
|
6315
|
+
anchored += 1 if anchored_meas else 0
|
|
6316
|
+
per_node.append({"id": nid, "static": a_s, "manifest": a_m,
|
|
6317
|
+
"behaviour": a_b, "anchored": anchored_meas,
|
|
6318
|
+
"claimed": anchored_any})
|
|
6319
|
+
edges = ir_graph.get("edges") or []
|
|
6320
|
+
anch_ids = {n["id"] for n in per_node if n["anchored"]}
|
|
6321
|
+
edges_recovered = sum(1 for e in edges if e.get("from") in anch_ids and e.get("to") in anch_ids)
|
|
6322
|
+
claimed = sum(1 for n in per_node if n["claimed"])
|
|
6323
|
+
return {"dir": os.path.basename(cd), "units": len(units),
|
|
6324
|
+
"ir_nodes": len(node_ids), "anchored": anchored, "claimed": claimed,
|
|
6325
|
+
"recoverability": round(anchored / len(node_ids), 3) if node_ids else None,
|
|
6326
|
+
"claimed_share": round(claimed / len(node_ids), 3) if node_ids else None,
|
|
6327
|
+
"static_anchored": sum(1 for n in per_node if n["static"]),
|
|
6328
|
+
"manifest_anchored": sum(1 for n in per_node if n["manifest"]),
|
|
6329
|
+
"behaviour": ("UNMEASURED" if not behaviour_measured
|
|
6330
|
+
else sum(1 for n in per_node if n["behaviour"])),
|
|
6331
|
+
"edges": len(edges), "edges_recovered": edges_recovered,
|
|
6332
|
+
"unanchored": [n["id"] for n in per_node if not n["anchored"]],
|
|
6333
|
+
"nodes": per_node}
|
|
6334
|
+
|
|
6335
|
+
|
|
6336
|
+
def _rt_entry(entry_dir, name):
|
|
6337
|
+
ir_graph, ir_errors = _load_ir_at(os.path.join(entry_dir, IR_FILE))
|
|
6338
|
+
try:
|
|
6339
|
+
with open(os.path.join(entry_dir, "oracle", "ORACLE.json"), encoding="utf-8-sig") as fh:
|
|
6340
|
+
cases = (json.load(fh) or {}).get("cases") or []
|
|
6341
|
+
except (OSError, ValueError):
|
|
6342
|
+
cases = []
|
|
6343
|
+
rec = {"archetype": name, "compilations": [], "ir_nodes": None, "edges": None,
|
|
6344
|
+
"recoverability_mean": None, "edges_recovered_mean": None, "reason": None}
|
|
6345
|
+
if ir_graph is None or ir_errors:
|
|
6346
|
+
rec["reason"] = "entry incomplete (missing/invalid IR)"
|
|
6347
|
+
return rec
|
|
6348
|
+
rec["ir_nodes"] = len(ir_graph.get("nodes") or [])
|
|
6349
|
+
rec["edges"] = len(ir_graph.get("edges") or [])
|
|
6350
|
+
for d in sorted(os.listdir(entry_dir)):
|
|
6351
|
+
cd = os.path.join(entry_dir, d)
|
|
6352
|
+
if d.startswith("c-") and os.path.isfile(os.path.join(cd, "COMPILATION.json")):
|
|
6353
|
+
r = _rt_compilation(entry_dir, cd, ir_graph, cases)
|
|
6354
|
+
if r:
|
|
6355
|
+
rec["compilations"].append(r)
|
|
6356
|
+
recs = [c["recoverability"] for c in rec["compilations"] if c["recoverability"] is not None]
|
|
6357
|
+
if recs:
|
|
6358
|
+
rec["recoverability_mean"] = round(sum(recs) / len(recs), 3)
|
|
6359
|
+
rec["edges_recovered_mean"] = round(
|
|
6360
|
+
sum(c["edges_recovered"] for c in rec["compilations"]) / len(rec["compilations"]), 2)
|
|
6361
|
+
else:
|
|
6362
|
+
rec["reason"] = "no compilations yet"
|
|
6363
|
+
return rec
|
|
6364
|
+
|
|
6365
|
+
|
|
6366
|
+
def _render_rt_md(recs, agg):
|
|
6367
|
+
lines = ["<!-- GENERATED by qa_ledger.py bench-roundtrip (ADR-030) -- measured run; do not hand-edit. -->",
|
|
6368
|
+
"", "# DIAMOND-ROUNDTRIP -- how much of the pinned IR the reverse organs can anchor in each compiled artifact", "",
|
|
6369
|
+
"**What this is NOT:** it does not regenerate an IR from code, it does not diff specs, "
|
|
6370
|
+
"it does not infer requirements. Reverse discovery produces FACTS (ADR-013); the human "
|
|
6371
|
+
"authors the spec. This instrument joins those facts against the human-authored IR "
|
|
6372
|
+
"and reports, per compilation, which nodes have footing -- (a) static: the id is "
|
|
6373
|
+
"literally referenced in a source unit or a static observation; (b) manifest: the "
|
|
6374
|
+
"compiler's validated trace manifest maps the node to a unit that exists; (c) "
|
|
6375
|
+
"behaviour: a withheld-oracle case tagged with the AC id passes (UNMEASURED where "
|
|
6376
|
+
"the entry's oracle cases carry no AC tag -- absence named). **`recoverability` "
|
|
6377
|
+
"counts ONLY static + behaviour footing** -- the manifest is what the compiler "
|
|
6378
|
+
"CLAIMED (and the blind prompt handed it the node ids), so it is reported apart as "
|
|
6379
|
+
"`claimed` and never counted as recovered: the first run of this instrument read "
|
|
6380
|
+
"1.000 on every entry for exactly that reason. `edges recovered` = edges with both "
|
|
6381
|
+
"endpoints MEASURED-anchored. Advisory: no bench verdict changes. The unanchored "
|
|
6382
|
+
"list IS the honest gap, per compiler.", "",
|
|
6383
|
+
"| Archetype | IR nodes | edges | recoverability (mean, measured) | edges recovered (mean) | per compilation (measured/nodes · static · behaviour · claimed-by-manifest · unanchored) |",
|
|
6384
|
+
"|-----------|----------|-------|---------------------------------|------------------------|--------------------------------------------------------------------------------------------|"]
|
|
6385
|
+
for r in recs:
|
|
6386
|
+
if r["recoverability_mean"] is None:
|
|
6387
|
+
lines.append("| %s | %s | %s | -- | -- | %s |" % (r["archetype"], r["ir_nodes"], r["edges"], r["reason"] or ""))
|
|
6388
|
+
continue
|
|
6389
|
+
pc = "; ".join("%s %d/%d · s%d · b%s · claimed %d · [%s]" % (
|
|
6390
|
+
c["dir"], c["anchored"], c["ir_nodes"], c["static_anchored"],
|
|
6391
|
+
c["behaviour"], c["manifest_anchored"], ",".join(c["unanchored"]) or "none")
|
|
6392
|
+
for c in r["compilations"])
|
|
6393
|
+
lines.append("| %s | %d | %d | %.3f | %.2f | %s |" % (
|
|
6394
|
+
r["archetype"], r["ir_nodes"], r["edges"], r["recoverability_mean"],
|
|
6395
|
+
r["edges_recovered_mean"], pc))
|
|
6396
|
+
lines += ["", "**Aggregate:** %d entries measured · mean recoverability %s · entries with edges %d "
|
|
6397
|
+
"· behaviour dimension measured in %d/%d entries." % (
|
|
6398
|
+
agg["measured"], "%.3f" % agg["mean_recoverability"] if agg["mean_recoverability"] is not None else "n/a",
|
|
6399
|
+
agg["with_edges"], agg["behaviour_measured"], agg["measured"]),
|
|
6400
|
+
"", "*The manifest dimension is what the compiler CLAIMED (validated for shape, "
|
|
6401
|
+
"not truth) and is excluded from recoverability; the static dimension is what the "
|
|
6402
|
+
"artifact literally names; the behaviour dimension is what the withheld oracle can "
|
|
6403
|
+
"attribute per AC -- and it is UNMEASURED wherever oracle cases are not tagged with "
|
|
6404
|
+
"AC ids, which today is every entry: the honest state of reverse discovery is that "
|
|
6405
|
+
"it anchors names, not semantics, until oracles carry per-AC tags. None of the three "
|
|
6406
|
+
"is a spec.*", ""]
|
|
6407
|
+
return "\n".join(lines)
|
|
6408
|
+
|
|
6409
|
+
|
|
6410
|
+
def cmd_bench_roundtrip(args):
|
|
6411
|
+
if not os.path.isdir(args.dir):
|
|
6412
|
+
print("[qa_ledger] bench-roundtrip: no directory %s" % args.dir, file=sys.stderr)
|
|
6413
|
+
sys.exit(2)
|
|
6414
|
+
entries = sorted(d for d in os.listdir(args.dir)
|
|
6415
|
+
if os.path.isfile(os.path.join(args.dir, d, IR_FILE)))
|
|
6416
|
+
if not entries:
|
|
6417
|
+
print("[qa_ledger] bench-roundtrip: no entries under %s" % args.dir, file=sys.stderr)
|
|
6418
|
+
sys.exit(2)
|
|
6419
|
+
recs = [_rt_entry(os.path.join(args.dir, e), e) for e in entries]
|
|
6420
|
+
measured = [r for r in recs if r["recoverability_mean"] is not None]
|
|
6421
|
+
agg = {"entries": len(recs), "measured": len(measured),
|
|
6422
|
+
"mean_recoverability": (round(sum(r["recoverability_mean"] for r in measured) / len(measured), 3)
|
|
6423
|
+
if measured else None),
|
|
6424
|
+
"with_edges": sum(1 for r in measured if (r["edges"] or 0) > 0),
|
|
6425
|
+
"behaviour_measured": sum(1 for r in measured
|
|
6426
|
+
if any(c["behaviour"] != "UNMEASURED" for c in r["compilations"]))}
|
|
6427
|
+
md = _render_rt_md(recs, agg)
|
|
6428
|
+
if args.out:
|
|
6429
|
+
with open(args.out, "w", encoding="utf-8", newline="\n") as fh:
|
|
6430
|
+
fh.write(md)
|
|
6431
|
+
if args.json:
|
|
6432
|
+
print(json.dumps({"aggregate": agg, "raw": recs}, indent=2, ensure_ascii=False))
|
|
6433
|
+
else:
|
|
6434
|
+
print("BENCH-ROUNDTRIP: %d entries, %d measured -- mean recoverability %s, %d with edges, "
|
|
6435
|
+
"behaviour measured in %d" % (agg["entries"], agg["measured"],
|
|
6436
|
+
"%.3f" % agg["mean_recoverability"] if agg["mean_recoverability"] is not None else "n/a",
|
|
6437
|
+
agg["with_edges"], agg["behaviour_measured"]))
|
|
6438
|
+
for r in recs:
|
|
6439
|
+
if r["recoverability_mean"] is not None:
|
|
6440
|
+
print(" %-17s nodes %2d edges %d recover %.3f edges-rec %.2f" % (
|
|
6441
|
+
r["archetype"], r["ir_nodes"], r["edges"], r["recoverability_mean"], r["edges_recovered_mean"]))
|
|
6442
|
+
if args.out:
|
|
6443
|
+
print(" -> %s" % args.out)
|
|
6444
|
+
sys.exit(0)
|
|
6445
|
+
|
|
6446
|
+
|
|
6008
6447
|
def _bench_curable_obs(bench_dir, entry, cdir):
|
|
6009
6448
|
"""The curable set for one compilation: the SAME observations the M1 static extractor
|
|
6010
6449
|
produces for the descriptor's static_surface, re-extracted at call time -- a verdict
|
|
@@ -6021,13 +6460,20 @@ def _bench_curable_obs(bench_dir, entry, cdir):
|
|
|
6021
6460
|
with open(cj, encoding="utf-8-sig") as fh:
|
|
6022
6461
|
c = json.load(fh)
|
|
6023
6462
|
src = c.get("source") or []
|
|
6024
|
-
|
|
6463
|
+
units = [u.get("unit") for u in src if isinstance(u, dict) and u.get("unit")]
|
|
6025
6464
|
except (OSError, ValueError, AttributeError, IndexError):
|
|
6026
|
-
|
|
6027
|
-
|
|
6465
|
+
units = []
|
|
6466
|
+
units = [u for u in units if os.path.isfile(os.path.join(cd, u.replace("/", os.sep)))]
|
|
6467
|
+
if not units:
|
|
6028
6468
|
return None, ("%s/%s: no resolvable source unit -- nothing to extract a surface "
|
|
6029
6469
|
"from" % (entry, cdir))
|
|
6030
|
-
sobs,
|
|
6470
|
+
sobs, uns = [], []
|
|
6471
|
+
for u in units:
|
|
6472
|
+
so, un = _static_surface_for(cd, u)
|
|
6473
|
+
sobs.extend(so)
|
|
6474
|
+
uns.extend(un)
|
|
6475
|
+
if not sobs and uns:
|
|
6476
|
+
return None, ("%s/%s: static surface UNMEASURED -- %s" % (entry, cdir, "; ".join(uns)))
|
|
6031
6477
|
return sobs, None
|
|
6032
6478
|
|
|
6033
6479
|
|
|
@@ -6114,7 +6560,9 @@ _LANG_PR_MARGIN = 0.02 # mean oracle pass-rate drop
|
|
|
6114
6560
|
|
|
6115
6561
|
def _lang_arm_metrics(arm_dir):
|
|
6116
6562
|
"""Measure ONE arm: per-compiler oracle green + unresolved_intent, and inter-compiler
|
|
6117
|
-
variance over the arm's compilations. Reuses the M4/M5 organs unchanged.
|
|
6563
|
+
variance over the arm's compilations. Reuses the M4/M5 organs unchanged. Returns
|
|
6564
|
+
(rec, errors, has_js) -- has_js is a footer-only signal for _render_lang_md (ADR-028);
|
|
6565
|
+
it is never part of `rec`, so lang-compare's JSON output is untouched by it."""
|
|
6118
6566
|
ir_graph, ir_errors = _load_ir_at(os.path.join(arm_dir, IR_FILE))
|
|
6119
6567
|
try:
|
|
6120
6568
|
with open(os.path.join(arm_dir, "oracle", "ORACLE.json"), encoding="utf-8-sig") as fh:
|
|
@@ -6126,7 +6574,7 @@ def _lang_arm_metrics(arm_dir):
|
|
|
6126
6574
|
"mean_passrate": None, "ui_count": 0.0, "ui_distinct_regions": 0.0,
|
|
6127
6575
|
"ui_rationale_len": 0.0}
|
|
6128
6576
|
if ir_graph is None or ir_errors or not cases:
|
|
6129
|
-
return rec, ["arm incomplete (missing/invalid IR or oracle)"]
|
|
6577
|
+
return rec, ["arm incomplete (missing/invalid IR or oracle)"], False
|
|
6130
6578
|
comp_dirs = sorted(d for d in os.listdir(arm_dir)
|
|
6131
6579
|
if d.startswith("c-") and
|
|
6132
6580
|
os.path.isfile(os.path.join(arm_dir, d, "COMPILATION.json")))
|
|
@@ -6140,7 +6588,7 @@ def _lang_arm_metrics(arm_dir):
|
|
|
6140
6588
|
with open(cj, encoding="utf-8-sig") as fh:
|
|
6141
6589
|
c = json.load(fh)
|
|
6142
6590
|
src = c.get("source") or []
|
|
6143
|
-
unit = src
|
|
6591
|
+
unit = _entry_unit(src)
|
|
6144
6592
|
model = (c.get("compilation_report") or {}).get("model")
|
|
6145
6593
|
uis = c.get("unresolved_intent") or []
|
|
6146
6594
|
except (OSError, ValueError, AttributeError, IndexError):
|
|
@@ -6175,17 +6623,25 @@ def _lang_arm_metrics(arm_dir):
|
|
|
6175
6623
|
for j in range(i + 1, len(good)):
|
|
6176
6624
|
dists.append(_struct_distance(good[i], good[j]))
|
|
6177
6625
|
rec["variance_score"] = round(sum(dists) / len(dists), 4) if dists else None
|
|
6178
|
-
|
|
6626
|
+
has_js = any(p.lower().endswith(".js") for p in impl_paths)
|
|
6627
|
+
return rec, [], has_js
|
|
6179
6628
|
|
|
6180
6629
|
|
|
6181
6630
|
def _struct_distance(a, b):
|
|
6182
6631
|
"""The ONE structural distance the program uses between two implementations -- mean of
|
|
6183
6632
|
normalized LOC delta, AST-node delta and import-set Jaccard distance (0 = identical). Shared
|
|
6184
6633
|
by the inter-compiler variance (lang-compare, bench) and the intra-model variance (bench-r2,
|
|
6185
|
-
ADR-027) so their ratio is commensurable: same function, both sides.
|
|
6634
|
+
ADR-027) so their ratio is commensurable: same function, both sides.
|
|
6635
|
+
|
|
6636
|
+
ADR-028: a `.js` metrics dict has `ast_nodes: None` (no stdlib JS AST) -- when either side
|
|
6637
|
+
lacks it, the distance is the mean over the dimensions BOTH sides have (LOC + import
|
|
6638
|
+
Jaccard only). A cross-language pair never occurs (one archetype, one language), so this
|
|
6639
|
+
never mixes a Python 3-dimensional distance with a JS 2-dimensional one within a pair."""
|
|
6186
6640
|
ia, ib = set(a["imports"]), set(b["imports"])
|
|
6187
6641
|
jac = (len(ia & ib) / len(ia | ib)) if (ia or ib) else 1.0
|
|
6188
6642
|
dloc = abs(a["loc"] - b["loc"]) / max(a["loc"], b["loc"], 1)
|
|
6643
|
+
if a.get("ast_nodes") is None or b.get("ast_nodes") is None:
|
|
6644
|
+
return (dloc + (1.0 - jac)) / 2.0
|
|
6189
6645
|
dast = abs(a["ast_nodes"] - b["ast_nodes"]) / max(a["ast_nodes"], b["ast_nodes"], 1)
|
|
6190
6646
|
return (dloc + dast + (1.0 - jac)) / 3.0
|
|
6191
6647
|
|
|
@@ -6213,8 +6669,9 @@ def cmd_lang_compare(args):
|
|
|
6213
6669
|
"the comparison requires the SAME withheld oracle; behaviour must be held fixed "
|
|
6214
6670
|
"while only the authoring changes." % (hf[:12], hc[:12]), file=sys.stderr)
|
|
6215
6671
|
sys.exit(2)
|
|
6216
|
-
free, ef = _lang_arm_metrics(args.free)
|
|
6217
|
-
ctrl, ec = _lang_arm_metrics(args.controlled)
|
|
6672
|
+
free, ef, has_js_f = _lang_arm_metrics(args.free)
|
|
6673
|
+
ctrl, ec, has_js_c = _lang_arm_metrics(args.controlled)
|
|
6674
|
+
has_js = has_js_f or has_js_c
|
|
6218
6675
|
if ef or ec:
|
|
6219
6676
|
for e in ef + ec:
|
|
6220
6677
|
print("[qa_ledger] lang-compare: %s" % e, file=sys.stderr)
|
|
@@ -6253,7 +6710,7 @@ def cmd_lang_compare(args):
|
|
|
6253
6710
|
"margin": _LANG_MARGIN, "passrate_margin": _LANG_PR_MARGIN, "oracle_shared": True}
|
|
6254
6711
|
if args.out:
|
|
6255
6712
|
with open(args.out, "w", encoding="utf-8", newline="\n") as fh:
|
|
6256
|
-
fh.write(_render_lang_md(report))
|
|
6713
|
+
fh.write(_render_lang_md(report, has_js))
|
|
6257
6714
|
if args.json:
|
|
6258
6715
|
print(json.dumps(report, indent=2, ensure_ascii=False))
|
|
6259
6716
|
else:
|
|
@@ -6277,7 +6734,7 @@ def cmd_lang_compare(args):
|
|
|
6277
6734
|
sys.exit(0)
|
|
6278
6735
|
|
|
6279
6736
|
|
|
6280
|
-
def _render_lang_md(r):
|
|
6737
|
+
def _render_lang_md(r, has_js=False):
|
|
6281
6738
|
d = r["delta"]
|
|
6282
6739
|
lines = ["<!-- GENERATED by qa_ledger.py lang-compare (ADR-019) -- measured run; do not "
|
|
6283
6740
|
"hand-edit. -->", "", "# CONTROLLED-LANGUAGE-REPORT", "",
|
|
@@ -6326,6 +6783,9 @@ def _render_lang_md(r):
|
|
|
6326
6783
|
"", "*The oracle is byte-identical across both arms, so behaviour is held fixed; "
|
|
6327
6784
|
"the only variable is the authoring discipline. The judgement of \"same semantic "
|
|
6328
6785
|
"content\" between the two canonical packages is human — a stated limitation.*", ""]
|
|
6786
|
+
if has_js:
|
|
6787
|
+
lines += ["*JS pairs use a 2-dimensional distance (LOC + import Jaccard; no stdlib JS "
|
|
6788
|
+
"AST).*", ""]
|
|
6329
6789
|
return "\n".join(lines)
|
|
6330
6790
|
|
|
6331
6791
|
|
|
@@ -10485,6 +10945,17 @@ def build_parser():
|
|
|
10485
10945
|
pbr.add_argument("--json", action="store_true")
|
|
10486
10946
|
pbr.set_defaults(func=cmd_bench_r2)
|
|
10487
10947
|
|
|
10948
|
+
prt = sub.add_parser(
|
|
10949
|
+
"bench-roundtrip",
|
|
10950
|
+
help="round-trip recoverability (ADR-030): per compilation, which pinned-IR nodes the "
|
|
10951
|
+
"mechanical reverse organs can anchor in the compiled artifact -- static id "
|
|
10952
|
+
"reference, validated manifest claim, tagged withheld-oracle behaviour -- and the "
|
|
10953
|
+
"unanchored gap list; regenerates NO IR, infers NO spec; advisory, deterministic")
|
|
10954
|
+
prt.add_argument("--dir", required=True, help="the bench directory")
|
|
10955
|
+
prt.add_argument("--out", default=None, help="write DIAMOND-ROUNDTRIP.md here")
|
|
10956
|
+
prt.add_argument("--json", action="store_true")
|
|
10957
|
+
prt.set_defaults(func=cmd_bench_roundtrip)
|
|
10958
|
+
|
|
10488
10959
|
plc = sub.add_parser(
|
|
10489
10960
|
"lang-compare",
|
|
10490
10961
|
help="controlled-language arm (ADR-019): compare a FREE-prose arm and an EARS+STE arm of "
|