@andresmassello/uscha 1.78.0 → 1.80.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -40,7 +40,7 @@ Requires **Python 3.8+** on the machine (the engine is Python stdlib — no pip
40
40
  runtime dependencies). The npm package is a thin router; the canonical installer is
41
41
  `uscha-kit/install-uscha.py`.
42
42
 
43
- **Kit v1.78.0** <!-- uscha:version --> · [uscha.dev](https://uscha.dev) ·
43
+ **Kit v1.80.0** <!-- uscha:version --> · [uscha.dev](https://uscha.dev) ·
44
44
  [changelog](https://github.com/andresmassello/uscha/blob/main/uscha-kit/CHANGELOG.md)
45
45
  (the per-release changelogs live in the repo, not in the npm tarball)
46
46
 
@@ -76,7 +76,7 @@ and see which file, which test, and when.
76
76
  | `/uscha-mirador` | Bird's-eye HTML dashboard: readiness, trail, acceptance, loops |
77
77
  | `/uscha-status` | One-line progress readout, in chat |
78
78
 
79
- **A measurement engine** (`qa_ledger.py`, 48 subcommands, Python stdlib) that ingests
79
+ **A measurement engine** (`qa_ledger.py`, 49 subcommands, Python stdlib) that ingests
80
80
  evidence from **11 language stacks** — maven, gradle, ant, python, node, go, rust, dotnet,
81
81
  cpp, swift, flutter — and computes a readiness score with hard caps and visible provenance.
82
82
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@andresmassello/uscha",
3
- "version": "1.78.0",
3
+ "version": "1.80.0",
4
4
  "description": "Spec-driven development for LLM coding agents: 9 skills + a stdlib evidence engine. Facts block, guesses advise; the human approves.",
5
5
  "author": {
6
6
  "name": "Andres Massello",
@@ -5480,6 +5480,52 @@ def cmd_bootstrap_variance(args):
5480
5480
  # --------------------------------------------------------------------------- #
5481
5481
  # min oracle pass-rate for a non-green compilation to still count as PARTIAL (core identity).
5482
5482
  _BENCH_PARTIAL_FLOOR = 0.8
5483
+ _BENCH_CURATION_FILE = "BENCH-CURATION.json"
5484
+ _BENCH_CURATION_VERDICTS = ("preserve", "fix", "undefined")
5485
+
5486
+
5487
+ def _load_bench_curation(bench_dir):
5488
+ """Load the append-only bench curation store. Returns (records, errors): a missing
5489
+ file is a legitimate absence ([], None); a malformed one is (None, errors) and every
5490
+ caller fails closed on it -- a broken verdict store must never silently degrade to
5491
+ UNMEASURED, which would hide tampering behind the honest word (ADR-023)."""
5492
+ path = os.path.join(bench_dir, _BENCH_CURATION_FILE)
5493
+ if os.path.exists(path) and not os.path.isfile(path):
5494
+ # a directory (editor swap, broken merge) occupying the store's path must not
5495
+ # read as "no curation exists" -- that is the silent-degrade this loader forbids
5496
+ return None, ["%s exists but is not a regular file" % _BENCH_CURATION_FILE]
5497
+ if not os.path.isfile(path):
5498
+ return [], None
5499
+ try:
5500
+ with open(path, encoding="utf-8-sig") as fh:
5501
+ data = json.load(fh)
5502
+ except (OSError, ValueError) as exc:
5503
+ return None, ["%s unreadable or not JSON: %s" % (_BENCH_CURATION_FILE, exc)]
5504
+ recs = data.get("records") if isinstance(data, dict) else None
5505
+ if not isinstance(recs, list):
5506
+ return None, ["%s has no 'records' list" % _BENCH_CURATION_FILE]
5507
+ errs = []
5508
+ for i, r in enumerate(recs):
5509
+ if not isinstance(r, dict) or any(
5510
+ not (isinstance(r.get(k), str) and r.get(k))
5511
+ for k in ("obs_id", "verdict", "human", "at", "entry", "dir")):
5512
+ errs.append("record %d malformed (obs_id/verdict/human/at/entry/dir "
5513
+ "must be non-empty strings)" % i)
5514
+ elif r["verdict"] not in _BENCH_CURATION_VERDICTS:
5515
+ errs.append("record %d: verdict %r is not one of %s"
5516
+ % (i, r["verdict"], "|".join(_BENCH_CURATION_VERDICTS)))
5517
+ if errs:
5518
+ return None, errs
5519
+ return recs, None
5520
+
5521
+
5522
+ def _bench_curation_map(records):
5523
+ """(entry, dir, obs_id) -> latest verdict. Append-only store: the LAST record for a
5524
+ key wins, earlier ones stay in the file as history (same discipline as `curate`)."""
5525
+ m = {}
5526
+ for r in records:
5527
+ m[(r["entry"], r["dir"], r["obs_id"])] = r["verdict"]
5528
+ return m
5483
5529
 
5484
5530
 
5485
5531
  def _bench_oracle_all(impl_path, cases):
@@ -5489,7 +5535,7 @@ def _bench_oracle_all(impl_path, cases):
5489
5535
  "failing": [r["name"] for r in results if not r["ok"]]}
5490
5536
 
5491
5537
 
5492
- def _bench_entry(entry_dir, name):
5538
+ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
5493
5539
  """Run compile-validate + the withheld oracle + variance over ONE bench entry and compute
5494
5540
  its verdict. Reuses the M3/M4 organs unchanged; consults no model. A PASS is >=3 oracle-green
5495
5541
  compilations that genuinely differ; PARTIAL is core identity with the divergence isolated;
@@ -5525,8 +5571,58 @@ def _bench_entry(entry_dir, name):
5525
5571
  impl = os.path.join(cd, unit.replace("/", os.sep)) if unit else None
5526
5572
  ores = (_bench_oracle_all(impl, cases) if impl and os.path.isfile(impl)
5527
5573
  else {"passed": 0, "total": len(cases), "green": False, "failing": ["<no impl>"]})
5528
- rec["compilations"].append({"dir": d, "model": model, "impl": impl,
5529
- "compile_valid": not errors, "oracle": ores})
5574
+ comp_rec = {"dir": d, "model": model, "impl": impl,
5575
+ "compile_valid": not errors, "oracle": ores}
5576
+ if fidelity and impl and os.path.isfile(impl) and unit:
5577
+ # the per-compiler fidelity descriptor (ADR-022): the M1 static extractor applied
5578
+ # to the compiled artifact -- reverse discovery per compiler. Advisory by
5579
+ # construction; curation_closure is UNMEASURED unless a human has judged at least
5580
+ # one of this compilation's observations via bench-curate (ADR-023) -- absence
5581
+ # named, never faked, and zero verdicts is an absence, not a 0.0.
5582
+ node_ids_f = {nd["id"] for nd in ir_graph.get("nodes") or []}
5583
+ covered_f = set()
5584
+ units_f, traced_f = set(), set()
5585
+ try:
5586
+ with open(cj, encoding="utf-8-sig") as fh2:
5587
+ c2 = json.load(fh2)
5588
+ for e2 in c2.get("trace_manifest") or []:
5589
+ traced_f.add(e2.get("unit"))
5590
+ for nid in e2.get("implements") or []:
5591
+ if nid in node_ids_f:
5592
+ covered_f.add(nid)
5593
+ for sec in ("source", "tests"):
5594
+ for u2 in c2.get(sec) or []:
5595
+ if u2.get("unit"):
5596
+ units_f.add(u2["unit"])
5597
+ except (OSError, ValueError, AttributeError):
5598
+ pass
5599
+ sobs, _uns = _extract_static_py(cd, [unit])
5600
+ fn_names, cls_names = [], []
5601
+ for o2 in sobs:
5602
+ m2 = re.search(r"defines function (\w+)", o2.get("statement", ""))
5603
+ if m2:
5604
+ fn_names.append(m2.group(1))
5605
+ m2 = re.search(r"defines class (\w+)", o2.get("statement", ""))
5606
+ if m2:
5607
+ cls_names.append(m2.group(1))
5608
+ unex = sorted(u for u in units_f if u not in traced_f)
5609
+ comp_rec["fidelity"] = {
5610
+ "trace_coverage": round(len(covered_f) / max(len(node_ids_f), 1), 3),
5611
+ "static_surface": {"functions": len(fn_names), "classes": len(cls_names),
5612
+ "names": sorted(fn_names + cls_names)},
5613
+ "oracle_passrate": (round(ores["passed"] / ores["total"], 3)
5614
+ if ores["total"] else None),
5615
+ "unexplained_share": round(len(unex) / max(len(units_f), 1), 3),
5616
+ "curation_closure": "UNMEASURED"}
5617
+ if curation:
5618
+ obs_ids_f = [o2["id"] for o2 in sobs]
5619
+ judged_f = sum(1 for oid in obs_ids_f if (name, d, oid) in curation)
5620
+ if judged_f:
5621
+ comp_rec["fidelity"]["curation_closure"] = round(
5622
+ judged_f / max(len(obs_ids_f), 1), 3)
5623
+ comp_rec["fidelity"]["curation"] = {"judged": judged_f,
5624
+ "total": len(obs_ids_f)}
5625
+ rec["compilations"].append(comp_rec)
5530
5626
  impls = rec["compilations"]
5531
5627
  impl_paths = [i["impl"] for i in impls if i["impl"] and os.path.isfile(i["impl"])]
5532
5628
  if len(impl_paths) >= 2:
@@ -5616,6 +5712,21 @@ def _render_bench_md(table, anon, recs):
5616
5712
  lines.append("- discrimination stub: %d/%d (%s)" % (
5617
5713
  dsc["stub_passed"], dsc["total"],
5618
5714
  "NON-DISCRIMINATING" if dsc["stub_green"] else "oracle rejects the stub"))
5715
+ for i in r["compilations"]:
5716
+ fd = i.get("fidelity")
5717
+ if fd:
5718
+ cur_v = fd["curation_closure"]
5719
+ if isinstance(cur_v, float):
5720
+ cur_v = "%.3f (judged %d/%d)" % (cur_v, fd["curation"]["judged"],
5721
+ fd["curation"]["total"])
5722
+ lines.append("- fidelity `%s` (%s): trace %.2f · surface %d fn / %d cls · "
5723
+ "oracle %s · unexplained %.2f · curation %s" % (
5724
+ i["dir"], anon.get(i["model"], i["model"] or "?"),
5725
+ fd["trace_coverage"], fd["static_surface"]["functions"],
5726
+ fd["static_surface"]["classes"],
5727
+ ("%.3f" % fd["oracle_passrate"])
5728
+ if fd["oracle_passrate"] is not None else "n/a",
5729
+ fd["unexplained_share"], cur_v))
5619
5730
  lines.append("")
5620
5731
  return "\n".join(lines)
5621
5732
 
@@ -5630,7 +5741,22 @@ def cmd_bench(args):
5630
5741
  print("[qa_ledger] bench: no entries under %s (an entry is a subdir with %s)"
5631
5742
  % (args.dir, IR_FILE), file=sys.stderr)
5632
5743
  sys.exit(2)
5633
- recs = [_bench_entry(os.path.join(args.dir, e), e) for e in entries]
5744
+ # the store is only ever consumed by the fidelity descriptor (ADR-023): a plain bench
5745
+ # run must not be blocked by a stray/corrupt store it was never going to read
5746
+ cur_map = {}
5747
+ if getattr(args, "fidelity", False):
5748
+ cur_recs, cur_errs = _load_bench_curation(args.dir)
5749
+ if cur_errs:
5750
+ for e in cur_errs:
5751
+ print("[qa_ledger] bench: %s" % e, file=sys.stderr)
5752
+ print("[qa_ledger] bench: a malformed verdict store must not silently degrade "
5753
+ "to UNMEASURED -- fix or remove %s." % _BENCH_CURATION_FILE,
5754
+ file=sys.stderr)
5755
+ sys.exit(2)
5756
+ cur_map = _bench_curation_map(cur_recs)
5757
+ recs = [_bench_entry(os.path.join(args.dir, e), e,
5758
+ fidelity=getattr(args, "fidelity", False), curation=cur_map)
5759
+ for e in entries]
5634
5760
  models = sorted({i["model"] for r in recs for i in r["compilations"] if i.get("model")})
5635
5761
  anon = {m: "M%d" % (k + 1) for k, m in enumerate(models)}
5636
5762
  table = []
@@ -5659,6 +5785,102 @@ def cmd_bench(args):
5659
5785
  sys.exit(0)
5660
5786
 
5661
5787
 
5788
+ def _bench_curable_obs(bench_dir, entry, cdir):
5789
+ """The curable set for one compilation: the SAME observations the M1 static extractor
5790
+ produces for the descriptor's static_surface, re-extracted at call time -- a verdict
5791
+ must judge a real observation, and a fixture edited since --list invalidates the old
5792
+ content-addressed id instead of silently carrying the stale judgment (ADR-023).
5793
+ Returns (obs_list, error_string)."""
5794
+ cd = os.path.join(bench_dir, entry, cdir)
5795
+ cj = os.path.join(cd, "COMPILATION.json")
5796
+ if not os.path.isdir(os.path.join(bench_dir, entry)):
5797
+ return None, "entry %r is not a directory under %s" % (entry, bench_dir)
5798
+ if not os.path.isfile(cj):
5799
+ return None, "%s/%s has no COMPILATION.json -- not a compilation" % (entry, cdir)
5800
+ try:
5801
+ with open(cj, encoding="utf-8-sig") as fh:
5802
+ c = json.load(fh)
5803
+ src = c.get("source") or []
5804
+ unit = src[0].get("unit") if src else None
5805
+ except (OSError, ValueError, AttributeError, IndexError):
5806
+ unit = None
5807
+ if not unit or not os.path.isfile(os.path.join(cd, unit.replace("/", os.sep))):
5808
+ return None, ("%s/%s: no resolvable source unit -- nothing to extract a surface "
5809
+ "from" % (entry, cdir))
5810
+ sobs, _uns = _extract_static_py(cd, [unit])
5811
+ return sobs, None
5812
+
5813
+
5814
+ def cmd_bench_curate(args):
5815
+ """ONE human verdict for ONE observation of ONE bench compilation, appended to the
5816
+ bench-root store (ADR-023). Inherits cmd_curate's refusals verbatim: no batch path
5817
+ exists and will not; an unknown observation is refused, never recorded."""
5818
+ if not os.path.isdir(args.bench):
5819
+ print("[qa_ledger] bench-curate: no directory %s" % args.bench, file=sys.stderr)
5820
+ sys.exit(2)
5821
+ obs_list, err = _bench_curable_obs(args.bench, args.entry, args.dir)
5822
+ if err:
5823
+ print("[qa_ledger] bench-curate: %s" % err, file=sys.stderr)
5824
+ sys.exit(2)
5825
+ records, errors = _load_bench_curation(args.bench)
5826
+ if errors:
5827
+ for e in errors:
5828
+ print("[qa_ledger] bench-curate: %s" % e, file=sys.stderr)
5829
+ sys.exit(2) # fail-closed, never degrade
5830
+ cur_map = _bench_curation_map(records)
5831
+ known = {o["id"] for o in obs_list}
5832
+ if args.list:
5833
+ print("BENCH-CURATE %s/%s: %d curable observation(s)"
5834
+ % (args.entry, args.dir, len(obs_list)))
5835
+ for o in obs_list:
5836
+ v = cur_map.get((args.entry, args.dir, o["id"]))
5837
+ print(" %s %-9s %s" % (o["id"], v or "unjudged", o["statement"]))
5838
+ stale = sorted(r["obs_id"] for r in records
5839
+ if r["entry"] == args.entry and r["dir"] == args.dir
5840
+ and r["obs_id"] not in known)
5841
+ for sid in stale:
5842
+ print(" %s STALE no longer in the extracted surface -- the fixture "
5843
+ "changed after this verdict" % sid)
5844
+ sys.exit(0)
5845
+ if not args.obs or not args.verdict:
5846
+ print("[qa_ledger] bench-curate: --obs and --verdict are required (or --list to "
5847
+ "see what awaits judgment).", file=sys.stderr)
5848
+ sys.exit(2)
5849
+ if args.obs != args.obs.strip() and not re.search(r"[,\s*]", args.obs.strip()):
5850
+ print("[qa_ledger] bench-curate: %r has leading/trailing whitespace -- pass the "
5851
+ "bare OBS id." % args.obs, file=sys.stderr)
5852
+ sys.exit(2)
5853
+ if re.search(r"[,\s*]", args.obs) or args.obs.lower() in ("all", "*"):
5854
+ print("[qa_ledger] bench-curate: %r -- one OBS, one human verdict. A batch-accept "
5855
+ "path does not exist and will not (ADR-023, INV-CURATION-01)." % args.obs,
5856
+ file=sys.stderr)
5857
+ sys.exit(2)
5858
+ if args.obs not in known:
5859
+ print("[qa_ledger] bench-curate: %s is not in the current extracted surface of "
5860
+ "%s/%s -- a verdict must judge a real observation (if the fixture changed, "
5861
+ "re-run --list)." % (args.obs, args.entry, args.dir), file=sys.stderr)
5862
+ sys.exit(2)
5863
+ prev = cur_map.get((args.entry, args.dir, args.obs))
5864
+ human = args.human or os.environ.get("USERNAME") or os.environ.get("USER") or "unknown"
5865
+ records.append({"obs_id": args.obs, "verdict": args.verdict, "human": human,
5866
+ "at": _now(), "note": args.note, "entry": args.entry, "dir": args.dir})
5867
+ store = {"_generated_by": "qa_ledger.py bench-curate (ADR-023) -- append-only human "
5868
+ "verdicts over bench compilations; the LAST record per "
5869
+ "(entry, dir, obs) wins, earlier ones stay as history",
5870
+ "records": records}
5871
+ with open(os.path.join(args.bench, _BENCH_CURATION_FILE), "w", encoding="utf-8",
5872
+ newline="\n") as fh:
5873
+ fh.write(json.dumps(store, indent=2, ensure_ascii=False) + "\n")
5874
+ if prev and prev != args.verdict:
5875
+ print("[qa_ledger] bench-curate: %s/%s %s = %s (supersedes %r -- the earlier "
5876
+ "record stays; append-only, never deleted)"
5877
+ % (args.entry, args.dir, args.obs, args.verdict, prev))
5878
+ else:
5879
+ print("[qa_ledger] bench-curate: %s/%s %s = %s (by %s)"
5880
+ % (args.entry, args.dir, args.obs, args.verdict, human))
5881
+ sys.exit(0)
5882
+
5883
+
5662
5884
  # --------------------------------------------------------------------------- #
5663
5885
  # lang-compare (Diamond controlled-language arm: the SAME canonical package in
5664
5886
  # free prose vs EARS+STE, judged by the SAME withheld oracle, compiled by the
@@ -9982,9 +10204,29 @@ def build_parser():
9982
10204
  pbn.add_argument("--dir", required=True,
9983
10205
  help="the bench directory; each subdir with an IR.json is an entry")
9984
10206
  pbn.add_argument("--out", default=None, help="write DIAMOND-BENCH.md here")
10207
+ pbn.add_argument("--fidelity", action="store_true",
10208
+ help="append the per-compiler fidelity descriptor (ADR-022): the M1 static extractor over each compiled source; advisory, never changes a verdict")
9985
10209
  pbn.add_argument("--json", action="store_true")
9986
10210
  pbn.set_defaults(func=cmd_bench)
9987
10211
 
10212
+ pbc = sub.add_parser(
10213
+ "bench-curate",
10214
+ help="record ONE human verdict (preserve|fix|undefined) for ONE observation of ONE "
10215
+ "bench compilation, appended to BENCH-CURATION.json; no batch path exists "
10216
+ "(ADR-023, INV-CURATION-01)")
10217
+ pbc.add_argument("--bench", required=True,
10218
+ help="the bench directory (the store lives at its root)")
10219
+ pbc.add_argument("--entry", required=True, help="the archetype entry")
10220
+ pbc.add_argument("--dir", required=True, help="the compilation dir (e.g. c-opus)")
10221
+ pbc.add_argument("--obs", default=None, help="a single OBS id from --list")
10222
+ pbc.add_argument("--verdict", default=None, choices=_BENCH_CURATION_VERDICTS)
10223
+ pbc.add_argument("--note", default=None)
10224
+ pbc.add_argument("--human", default=None, help="who judged (default: the OS user)")
10225
+ pbc.add_argument("--list", action="store_true",
10226
+ help="print the curable observations with their current verdicts "
10227
+ "(read-only), including stale verdicts whose obs no longer exists")
10228
+ pbc.set_defaults(func=cmd_bench_curate)
10229
+
9988
10230
  plc = sub.add_parser(
9989
10231
  "lang-compare",
9990
10232
  help="controlled-language arm (ADR-019): compare a FREE-prose arm and an EARS+STE arm of "
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
3
3
  "name": "uscha",
4
- "version": "1.78.0",
4
+ "version": "1.80.0",
5
5
  "displayName": "Uscha",
6
- "description": "Spec-driven development for LLM coding agents: 9 skills (discovery, adr-refine, reverse-discovery, characterize, devloop, sysdoc, rubric, mirador, status) + a stdlib measurement engine (qa_ledger.py, 48 subcommands + universal installer + npm/npx router). Facts block, guesses advise; the human approves.",
6
+ "description": "Spec-driven development for LLM coding agents: 9 skills (discovery, adr-refine, reverse-discovery, characterize, devloop, sysdoc, rubric, mirador, status) + a stdlib measurement engine (qa_ledger.py, 49 subcommands + universal installer + npm/npx router). Facts block, guesses advise; the human approves.",
7
7
  "author": {
8
8
  "name": "Andres Massello",
9
9
  "url": "https://github.com/andresmassello"
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "uscha",
3
- "version": "1.78.0",
3
+ "version": "1.80.0",
4
4
  "description": "Uscha spec-driven development methodology for coding agents. Includes npm/npx router.",
5
5
  "author": {
6
6
  "name": "Andres Massello",
@@ -1,6 +1,6 @@
1
1
  # uscha-kit
2
2
 
3
- **Kit version:** v1.78.0 <!-- uscha:version --> · **[uscha.dev](https://uscha.dev)**
3
+ **Kit version:** v1.80.0 <!-- uscha:version --> · **[uscha.dev](https://uscha.dev)**
4
4
 
5
5
  Spec-driven orchestrator + multi-repo QA for Claude Code, with a deterministic ledger.
6
6
  **Nine skills** (`uscha-discovery`, `uscha-adr-refine`, `uscha-devloop`, `uscha-sysdoc`, `uscha-reverse-discovery`,
package/uscha-kit/VERSION CHANGED
@@ -1 +1 @@
1
- uscha-kit 1.78.0
1
+ uscha-kit 1.80.0
@@ -1 +1 @@
1
- {"AC-DB-01": true, "AC-DB-02": true, "AC-DB-03": true, "AC-DB-05": true, "AC-DB-04": true, "AC-DB-06": true, "AC-BG-01": true, "AC-BG-02": true, "AC-BG-03": true, "AC-BG-04": true, "AC-BG-05": true}
1
+ {"AC-DB-01": true, "AC-DB-02": true, "AC-DB-03": true, "AC-DB-05": true, "AC-DB-04": true, "AC-DB-06": true, "AC-BG-01": true, "AC-BG-02": true, "AC-BG-03": true, "AC-BG-04": true, "AC-BG-05": true, "AC-FC-01": true, "AC-FC-02": true, "AC-FC-03": true}
@@ -0,0 +1 @@
1
+ {"AC-BC-01": true, "AC-BC-02": true, "AC-BC-03": true}
@@ -5480,6 +5480,52 @@ def cmd_bootstrap_variance(args):
5480
5480
  # --------------------------------------------------------------------------- #
5481
5481
  # min oracle pass-rate for a non-green compilation to still count as PARTIAL (core identity).
5482
5482
  _BENCH_PARTIAL_FLOOR = 0.8
5483
+ _BENCH_CURATION_FILE = "BENCH-CURATION.json"
5484
+ _BENCH_CURATION_VERDICTS = ("preserve", "fix", "undefined")
5485
+
5486
+
5487
+ def _load_bench_curation(bench_dir):
5488
+ """Load the append-only bench curation store. Returns (records, errors): a missing
5489
+ file is a legitimate absence ([], None); a malformed one is (None, errors) and every
5490
+ caller fails closed on it -- a broken verdict store must never silently degrade to
5491
+ UNMEASURED, which would hide tampering behind the honest word (ADR-023)."""
5492
+ path = os.path.join(bench_dir, _BENCH_CURATION_FILE)
5493
+ if os.path.exists(path) and not os.path.isfile(path):
5494
+ # a directory (editor swap, broken merge) occupying the store's path must not
5495
+ # read as "no curation exists" -- that is the silent-degrade this loader forbids
5496
+ return None, ["%s exists but is not a regular file" % _BENCH_CURATION_FILE]
5497
+ if not os.path.isfile(path):
5498
+ return [], None
5499
+ try:
5500
+ with open(path, encoding="utf-8-sig") as fh:
5501
+ data = json.load(fh)
5502
+ except (OSError, ValueError) as exc:
5503
+ return None, ["%s unreadable or not JSON: %s" % (_BENCH_CURATION_FILE, exc)]
5504
+ recs = data.get("records") if isinstance(data, dict) else None
5505
+ if not isinstance(recs, list):
5506
+ return None, ["%s has no 'records' list" % _BENCH_CURATION_FILE]
5507
+ errs = []
5508
+ for i, r in enumerate(recs):
5509
+ if not isinstance(r, dict) or any(
5510
+ not (isinstance(r.get(k), str) and r.get(k))
5511
+ for k in ("obs_id", "verdict", "human", "at", "entry", "dir")):
5512
+ errs.append("record %d malformed (obs_id/verdict/human/at/entry/dir "
5513
+ "must be non-empty strings)" % i)
5514
+ elif r["verdict"] not in _BENCH_CURATION_VERDICTS:
5515
+ errs.append("record %d: verdict %r is not one of %s"
5516
+ % (i, r["verdict"], "|".join(_BENCH_CURATION_VERDICTS)))
5517
+ if errs:
5518
+ return None, errs
5519
+ return recs, None
5520
+
5521
+
5522
+ def _bench_curation_map(records):
5523
+ """(entry, dir, obs_id) -> latest verdict. Append-only store: the LAST record for a
5524
+ key wins, earlier ones stay in the file as history (same discipline as `curate`)."""
5525
+ m = {}
5526
+ for r in records:
5527
+ m[(r["entry"], r["dir"], r["obs_id"])] = r["verdict"]
5528
+ return m
5483
5529
 
5484
5530
 
5485
5531
  def _bench_oracle_all(impl_path, cases):
@@ -5489,7 +5535,7 @@ def _bench_oracle_all(impl_path, cases):
5489
5535
  "failing": [r["name"] for r in results if not r["ok"]]}
5490
5536
 
5491
5537
 
5492
- def _bench_entry(entry_dir, name):
5538
+ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
5493
5539
  """Run compile-validate + the withheld oracle + variance over ONE bench entry and compute
5494
5540
  its verdict. Reuses the M3/M4 organs unchanged; consults no model. A PASS is >=3 oracle-green
5495
5541
  compilations that genuinely differ; PARTIAL is core identity with the divergence isolated;
@@ -5525,8 +5571,58 @@ def _bench_entry(entry_dir, name):
5525
5571
  impl = os.path.join(cd, unit.replace("/", os.sep)) if unit else None
5526
5572
  ores = (_bench_oracle_all(impl, cases) if impl and os.path.isfile(impl)
5527
5573
  else {"passed": 0, "total": len(cases), "green": False, "failing": ["<no impl>"]})
5528
- rec["compilations"].append({"dir": d, "model": model, "impl": impl,
5529
- "compile_valid": not errors, "oracle": ores})
5574
+ comp_rec = {"dir": d, "model": model, "impl": impl,
5575
+ "compile_valid": not errors, "oracle": ores}
5576
+ if fidelity and impl and os.path.isfile(impl) and unit:
5577
+ # the per-compiler fidelity descriptor (ADR-022): the M1 static extractor applied
5578
+ # to the compiled artifact -- reverse discovery per compiler. Advisory by
5579
+ # construction; curation_closure is UNMEASURED unless a human has judged at least
5580
+ # one of this compilation's observations via bench-curate (ADR-023) -- absence
5581
+ # named, never faked, and zero verdicts is an absence, not a 0.0.
5582
+ node_ids_f = {nd["id"] for nd in ir_graph.get("nodes") or []}
5583
+ covered_f = set()
5584
+ units_f, traced_f = set(), set()
5585
+ try:
5586
+ with open(cj, encoding="utf-8-sig") as fh2:
5587
+ c2 = json.load(fh2)
5588
+ for e2 in c2.get("trace_manifest") or []:
5589
+ traced_f.add(e2.get("unit"))
5590
+ for nid in e2.get("implements") or []:
5591
+ if nid in node_ids_f:
5592
+ covered_f.add(nid)
5593
+ for sec in ("source", "tests"):
5594
+ for u2 in c2.get(sec) or []:
5595
+ if u2.get("unit"):
5596
+ units_f.add(u2["unit"])
5597
+ except (OSError, ValueError, AttributeError):
5598
+ pass
5599
+ sobs, _uns = _extract_static_py(cd, [unit])
5600
+ fn_names, cls_names = [], []
5601
+ for o2 in sobs:
5602
+ m2 = re.search(r"defines function (\w+)", o2.get("statement", ""))
5603
+ if m2:
5604
+ fn_names.append(m2.group(1))
5605
+ m2 = re.search(r"defines class (\w+)", o2.get("statement", ""))
5606
+ if m2:
5607
+ cls_names.append(m2.group(1))
5608
+ unex = sorted(u for u in units_f if u not in traced_f)
5609
+ comp_rec["fidelity"] = {
5610
+ "trace_coverage": round(len(covered_f) / max(len(node_ids_f), 1), 3),
5611
+ "static_surface": {"functions": len(fn_names), "classes": len(cls_names),
5612
+ "names": sorted(fn_names + cls_names)},
5613
+ "oracle_passrate": (round(ores["passed"] / ores["total"], 3)
5614
+ if ores["total"] else None),
5615
+ "unexplained_share": round(len(unex) / max(len(units_f), 1), 3),
5616
+ "curation_closure": "UNMEASURED"}
5617
+ if curation:
5618
+ obs_ids_f = [o2["id"] for o2 in sobs]
5619
+ judged_f = sum(1 for oid in obs_ids_f if (name, d, oid) in curation)
5620
+ if judged_f:
5621
+ comp_rec["fidelity"]["curation_closure"] = round(
5622
+ judged_f / max(len(obs_ids_f), 1), 3)
5623
+ comp_rec["fidelity"]["curation"] = {"judged": judged_f,
5624
+ "total": len(obs_ids_f)}
5625
+ rec["compilations"].append(comp_rec)
5530
5626
  impls = rec["compilations"]
5531
5627
  impl_paths = [i["impl"] for i in impls if i["impl"] and os.path.isfile(i["impl"])]
5532
5628
  if len(impl_paths) >= 2:
@@ -5616,6 +5712,21 @@ def _render_bench_md(table, anon, recs):
5616
5712
  lines.append("- discrimination stub: %d/%d (%s)" % (
5617
5713
  dsc["stub_passed"], dsc["total"],
5618
5714
  "NON-DISCRIMINATING" if dsc["stub_green"] else "oracle rejects the stub"))
5715
+ for i in r["compilations"]:
5716
+ fd = i.get("fidelity")
5717
+ if fd:
5718
+ cur_v = fd["curation_closure"]
5719
+ if isinstance(cur_v, float):
5720
+ cur_v = "%.3f (judged %d/%d)" % (cur_v, fd["curation"]["judged"],
5721
+ fd["curation"]["total"])
5722
+ lines.append("- fidelity `%s` (%s): trace %.2f · surface %d fn / %d cls · "
5723
+ "oracle %s · unexplained %.2f · curation %s" % (
5724
+ i["dir"], anon.get(i["model"], i["model"] or "?"),
5725
+ fd["trace_coverage"], fd["static_surface"]["functions"],
5726
+ fd["static_surface"]["classes"],
5727
+ ("%.3f" % fd["oracle_passrate"])
5728
+ if fd["oracle_passrate"] is not None else "n/a",
5729
+ fd["unexplained_share"], cur_v))
5619
5730
  lines.append("")
5620
5731
  return "\n".join(lines)
5621
5732
 
@@ -5630,7 +5741,22 @@ def cmd_bench(args):
5630
5741
  print("[qa_ledger] bench: no entries under %s (an entry is a subdir with %s)"
5631
5742
  % (args.dir, IR_FILE), file=sys.stderr)
5632
5743
  sys.exit(2)
5633
- recs = [_bench_entry(os.path.join(args.dir, e), e) for e in entries]
5744
+ # the store is only ever consumed by the fidelity descriptor (ADR-023): a plain bench
5745
+ # run must not be blocked by a stray/corrupt store it was never going to read
5746
+ cur_map = {}
5747
+ if getattr(args, "fidelity", False):
5748
+ cur_recs, cur_errs = _load_bench_curation(args.dir)
5749
+ if cur_errs:
5750
+ for e in cur_errs:
5751
+ print("[qa_ledger] bench: %s" % e, file=sys.stderr)
5752
+ print("[qa_ledger] bench: a malformed verdict store must not silently degrade "
5753
+ "to UNMEASURED -- fix or remove %s." % _BENCH_CURATION_FILE,
5754
+ file=sys.stderr)
5755
+ sys.exit(2)
5756
+ cur_map = _bench_curation_map(cur_recs)
5757
+ recs = [_bench_entry(os.path.join(args.dir, e), e,
5758
+ fidelity=getattr(args, "fidelity", False), curation=cur_map)
5759
+ for e in entries]
5634
5760
  models = sorted({i["model"] for r in recs for i in r["compilations"] if i.get("model")})
5635
5761
  anon = {m: "M%d" % (k + 1) for k, m in enumerate(models)}
5636
5762
  table = []
@@ -5659,6 +5785,102 @@ def cmd_bench(args):
5659
5785
  sys.exit(0)
5660
5786
 
5661
5787
 
5788
+ def _bench_curable_obs(bench_dir, entry, cdir):
5789
+ """The curable set for one compilation: the SAME observations the M1 static extractor
5790
+ produces for the descriptor's static_surface, re-extracted at call time -- a verdict
5791
+ must judge a real observation, and a fixture edited since --list invalidates the old
5792
+ content-addressed id instead of silently carrying the stale judgment (ADR-023).
5793
+ Returns (obs_list, error_string)."""
5794
+ cd = os.path.join(bench_dir, entry, cdir)
5795
+ cj = os.path.join(cd, "COMPILATION.json")
5796
+ if not os.path.isdir(os.path.join(bench_dir, entry)):
5797
+ return None, "entry %r is not a directory under %s" % (entry, bench_dir)
5798
+ if not os.path.isfile(cj):
5799
+ return None, "%s/%s has no COMPILATION.json -- not a compilation" % (entry, cdir)
5800
+ try:
5801
+ with open(cj, encoding="utf-8-sig") as fh:
5802
+ c = json.load(fh)
5803
+ src = c.get("source") or []
5804
+ unit = src[0].get("unit") if src else None
5805
+ except (OSError, ValueError, AttributeError, IndexError):
5806
+ unit = None
5807
+ if not unit or not os.path.isfile(os.path.join(cd, unit.replace("/", os.sep))):
5808
+ return None, ("%s/%s: no resolvable source unit -- nothing to extract a surface "
5809
+ "from" % (entry, cdir))
5810
+ sobs, _uns = _extract_static_py(cd, [unit])
5811
+ return sobs, None
5812
+
5813
+
5814
+ def cmd_bench_curate(args):
5815
+ """ONE human verdict for ONE observation of ONE bench compilation, appended to the
5816
+ bench-root store (ADR-023). Inherits cmd_curate's refusals verbatim: no batch path
5817
+ exists and will not; an unknown observation is refused, never recorded."""
5818
+ if not os.path.isdir(args.bench):
5819
+ print("[qa_ledger] bench-curate: no directory %s" % args.bench, file=sys.stderr)
5820
+ sys.exit(2)
5821
+ obs_list, err = _bench_curable_obs(args.bench, args.entry, args.dir)
5822
+ if err:
5823
+ print("[qa_ledger] bench-curate: %s" % err, file=sys.stderr)
5824
+ sys.exit(2)
5825
+ records, errors = _load_bench_curation(args.bench)
5826
+ if errors:
5827
+ for e in errors:
5828
+ print("[qa_ledger] bench-curate: %s" % e, file=sys.stderr)
5829
+ sys.exit(2) # fail-closed, never degrade
5830
+ cur_map = _bench_curation_map(records)
5831
+ known = {o["id"] for o in obs_list}
5832
+ if args.list:
5833
+ print("BENCH-CURATE %s/%s: %d curable observation(s)"
5834
+ % (args.entry, args.dir, len(obs_list)))
5835
+ for o in obs_list:
5836
+ v = cur_map.get((args.entry, args.dir, o["id"]))
5837
+ print(" %s %-9s %s" % (o["id"], v or "unjudged", o["statement"]))
5838
+ stale = sorted(r["obs_id"] for r in records
5839
+ if r["entry"] == args.entry and r["dir"] == args.dir
5840
+ and r["obs_id"] not in known)
5841
+ for sid in stale:
5842
+ print(" %s STALE no longer in the extracted surface -- the fixture "
5843
+ "changed after this verdict" % sid)
5844
+ sys.exit(0)
5845
+ if not args.obs or not args.verdict:
5846
+ print("[qa_ledger] bench-curate: --obs and --verdict are required (or --list to "
5847
+ "see what awaits judgment).", file=sys.stderr)
5848
+ sys.exit(2)
5849
+ if args.obs != args.obs.strip() and not re.search(r"[,\s*]", args.obs.strip()):
5850
+ print("[qa_ledger] bench-curate: %r has leading/trailing whitespace -- pass the "
5851
+ "bare OBS id." % args.obs, file=sys.stderr)
5852
+ sys.exit(2)
5853
+ if re.search(r"[,\s*]", args.obs) or args.obs.lower() in ("all", "*"):
5854
+ print("[qa_ledger] bench-curate: %r -- one OBS, one human verdict. A batch-accept "
5855
+ "path does not exist and will not (ADR-023, INV-CURATION-01)." % args.obs,
5856
+ file=sys.stderr)
5857
+ sys.exit(2)
5858
+ if args.obs not in known:
5859
+ print("[qa_ledger] bench-curate: %s is not in the current extracted surface of "
5860
+ "%s/%s -- a verdict must judge a real observation (if the fixture changed, "
5861
+ "re-run --list)." % (args.obs, args.entry, args.dir), file=sys.stderr)
5862
+ sys.exit(2)
5863
+ prev = cur_map.get((args.entry, args.dir, args.obs))
5864
+ human = args.human or os.environ.get("USERNAME") or os.environ.get("USER") or "unknown"
5865
+ records.append({"obs_id": args.obs, "verdict": args.verdict, "human": human,
5866
+ "at": _now(), "note": args.note, "entry": args.entry, "dir": args.dir})
5867
+ store = {"_generated_by": "qa_ledger.py bench-curate (ADR-023) -- append-only human "
5868
+ "verdicts over bench compilations; the LAST record per "
5869
+ "(entry, dir, obs) wins, earlier ones stay as history",
5870
+ "records": records}
5871
+ with open(os.path.join(args.bench, _BENCH_CURATION_FILE), "w", encoding="utf-8",
5872
+ newline="\n") as fh:
5873
+ fh.write(json.dumps(store, indent=2, ensure_ascii=False) + "\n")
5874
+ if prev and prev != args.verdict:
5875
+ print("[qa_ledger] bench-curate: %s/%s %s = %s (supersedes %r -- the earlier "
5876
+ "record stays; append-only, never deleted)"
5877
+ % (args.entry, args.dir, args.obs, args.verdict, prev))
5878
+ else:
5879
+ print("[qa_ledger] bench-curate: %s/%s %s = %s (by %s)"
5880
+ % (args.entry, args.dir, args.obs, args.verdict, human))
5881
+ sys.exit(0)
5882
+
5883
+
5662
5884
  # --------------------------------------------------------------------------- #
5663
5885
  # lang-compare (Diamond controlled-language arm: the SAME canonical package in
5664
5886
  # free prose vs EARS+STE, judged by the SAME withheld oracle, compiled by the
@@ -9982,9 +10204,29 @@ def build_parser():
9982
10204
  pbn.add_argument("--dir", required=True,
9983
10205
  help="the bench directory; each subdir with an IR.json is an entry")
9984
10206
  pbn.add_argument("--out", default=None, help="write DIAMOND-BENCH.md here")
10207
+ pbn.add_argument("--fidelity", action="store_true",
10208
+ help="append the per-compiler fidelity descriptor (ADR-022): the M1 static extractor over each compiled source; advisory, never changes a verdict")
9985
10209
  pbn.add_argument("--json", action="store_true")
9986
10210
  pbn.set_defaults(func=cmd_bench)
9987
10211
 
10212
+ pbc = sub.add_parser(
10213
+ "bench-curate",
10214
+ help="record ONE human verdict (preserve|fix|undefined) for ONE observation of ONE "
10215
+ "bench compilation, appended to BENCH-CURATION.json; no batch path exists "
10216
+ "(ADR-023, INV-CURATION-01)")
10217
+ pbc.add_argument("--bench", required=True,
10218
+ help="the bench directory (the store lives at its root)")
10219
+ pbc.add_argument("--entry", required=True, help="the archetype entry")
10220
+ pbc.add_argument("--dir", required=True, help="the compilation dir (e.g. c-opus)")
10221
+ pbc.add_argument("--obs", default=None, help="a single OBS id from --list")
10222
+ pbc.add_argument("--verdict", default=None, choices=_BENCH_CURATION_VERDICTS)
10223
+ pbc.add_argument("--note", default=None)
10224
+ pbc.add_argument("--human", default=None, help="who judged (default: the OS user)")
10225
+ pbc.add_argument("--list", action="store_true",
10226
+ help="print the curable observations with their current verdicts "
10227
+ "(read-only), including stale verdicts whose obs no longer exists")
10228
+ pbc.set_defaults(func=cmd_bench_curate)
10229
+
9988
10230
  plc = sub.add_parser(
9989
10231
  "lang-compare",
9990
10232
  help="controlled-language arm (ADR-019): compare a FREE-prose arm and an EARS+STE arm of "
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "1.78.0",
2
+ "version": "1.80.0",
3
3
  "project": null,
4
4
  "defaults": {
5
5
  "coverage_threshold": 60,