@andresmassello/uscha 1.79.0 → 1.80.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -40,7 +40,7 @@ Requires **Python 3.8+** on the machine (the engine is Python stdlib — no pip
40
40
  runtime dependencies). The npm package is a thin router; the canonical installer is
41
41
  `uscha-kit/install-uscha.py`.
42
42
 
43
- **Kit v1.79.0** <!-- uscha:version --> · [uscha.dev](https://uscha.dev) ·
43
+ **Kit v1.80.0** <!-- uscha:version --> · [uscha.dev](https://uscha.dev) ·
44
44
  [changelog](https://github.com/andresmassello/uscha/blob/main/uscha-kit/CHANGELOG.md)
45
45
  (the per-release changelogs live in the repo, not in the npm tarball)
46
46
 
@@ -76,7 +76,7 @@ and see which file, which test, and when.
76
76
  | `/uscha-mirador` | Bird's-eye HTML dashboard: readiness, trail, acceptance, loops |
77
77
  | `/uscha-status` | One-line progress readout, in chat |
78
78
 
79
- **A measurement engine** (`qa_ledger.py`, 48 subcommands, Python stdlib) that ingests
79
+ **A measurement engine** (`qa_ledger.py`, 49 subcommands, Python stdlib) that ingests
80
80
  evidence from **11 language stacks** — maven, gradle, ant, python, node, go, rust, dotnet,
81
81
  cpp, swift, flutter — and computes a readiness score with hard caps and visible provenance.
82
82
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@andresmassello/uscha",
3
- "version": "1.79.0",
3
+ "version": "1.80.0",
4
4
  "description": "Spec-driven development for LLM coding agents: 9 skills + a stdlib evidence engine. Facts block, guesses advise; the human approves.",
5
5
  "author": {
6
6
  "name": "Andres Massello",
@@ -5480,6 +5480,52 @@ def cmd_bootstrap_variance(args):
5480
5480
  # --------------------------------------------------------------------------- #
5481
5481
  # min oracle pass-rate for a non-green compilation to still count as PARTIAL (core identity).
5482
5482
  _BENCH_PARTIAL_FLOOR = 0.8
5483
+ _BENCH_CURATION_FILE = "BENCH-CURATION.json"
5484
+ _BENCH_CURATION_VERDICTS = ("preserve", "fix", "undefined")
5485
+
5486
+
5487
+ def _load_bench_curation(bench_dir):
5488
+ """Load the append-only bench curation store. Returns (records, errors): a missing
5489
+ file is a legitimate absence ([], None); a malformed one is (None, errors) and every
5490
+ caller fails closed on it -- a broken verdict store must never silently degrade to
5491
+ UNMEASURED, which would hide tampering behind the honest word (ADR-023)."""
5492
+ path = os.path.join(bench_dir, _BENCH_CURATION_FILE)
5493
+ if os.path.exists(path) and not os.path.isfile(path):
5494
+ # a directory (editor swap, broken merge) occupying the store's path must not
5495
+ # read as "no curation exists" -- that is the silent-degrade this loader forbids
5496
+ return None, ["%s exists but is not a regular file" % _BENCH_CURATION_FILE]
5497
+ if not os.path.isfile(path):
5498
+ return [], None
5499
+ try:
5500
+ with open(path, encoding="utf-8-sig") as fh:
5501
+ data = json.load(fh)
5502
+ except (OSError, ValueError) as exc:
5503
+ return None, ["%s unreadable or not JSON: %s" % (_BENCH_CURATION_FILE, exc)]
5504
+ recs = data.get("records") if isinstance(data, dict) else None
5505
+ if not isinstance(recs, list):
5506
+ return None, ["%s has no 'records' list" % _BENCH_CURATION_FILE]
5507
+ errs = []
5508
+ for i, r in enumerate(recs):
5509
+ if not isinstance(r, dict) or any(
5510
+ not (isinstance(r.get(k), str) and r.get(k))
5511
+ for k in ("obs_id", "verdict", "human", "at", "entry", "dir")):
5512
+ errs.append("record %d malformed (obs_id/verdict/human/at/entry/dir "
5513
+ "must be non-empty strings)" % i)
5514
+ elif r["verdict"] not in _BENCH_CURATION_VERDICTS:
5515
+ errs.append("record %d: verdict %r is not one of %s"
5516
+ % (i, r["verdict"], "|".join(_BENCH_CURATION_VERDICTS)))
5517
+ if errs:
5518
+ return None, errs
5519
+ return recs, None
5520
+
5521
+
5522
+ def _bench_curation_map(records):
5523
+ """(entry, dir, obs_id) -> latest verdict. Append-only store: the LAST record for a
5524
+ key wins, earlier ones stay in the file as history (same discipline as `curate`)."""
5525
+ m = {}
5526
+ for r in records:
5527
+ m[(r["entry"], r["dir"], r["obs_id"])] = r["verdict"]
5528
+ return m
5483
5529
 
5484
5530
 
5485
5531
  def _bench_oracle_all(impl_path, cases):
@@ -5489,7 +5535,7 @@ def _bench_oracle_all(impl_path, cases):
5489
5535
  "failing": [r["name"] for r in results if not r["ok"]]}
5490
5536
 
5491
5537
 
5492
- def _bench_entry(entry_dir, name, fidelity=False):
5538
+ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
5493
5539
  """Run compile-validate + the withheld oracle + variance over ONE bench entry and compute
5494
5540
  its verdict. Reuses the M3/M4 organs unchanged; consults no model. A PASS is >=3 oracle-green
5495
5541
  compilations that genuinely differ; PARTIAL is core identity with the divergence isolated;
@@ -5530,8 +5576,9 @@ def _bench_entry(entry_dir, name, fidelity=False):
5530
5576
  if fidelity and impl and os.path.isfile(impl) and unit:
5531
5577
  # the per-compiler fidelity descriptor (ADR-022): the M1 static extractor applied
5532
5578
  # to the compiled artifact -- reverse discovery per compiler. Advisory by
5533
- # construction; curation_closure is UNMEASURED (no human curates fixture code --
5534
- # absence named, never faked).
5579
+ # construction; curation_closure is UNMEASURED unless a human has judged at least
5580
+ # one of this compilation's observations via bench-curate (ADR-023) -- absence
5581
+ # named, never faked, and zero verdicts is an absence, not a 0.0.
5535
5582
  node_ids_f = {nd["id"] for nd in ir_graph.get("nodes") or []}
5536
5583
  covered_f = set()
5537
5584
  units_f, traced_f = set(), set()
@@ -5567,6 +5614,14 @@ def _bench_entry(entry_dir, name, fidelity=False):
5567
5614
  if ores["total"] else None),
5568
5615
  "unexplained_share": round(len(unex) / max(len(units_f), 1), 3),
5569
5616
  "curation_closure": "UNMEASURED"}
5617
+ if curation:
5618
+ obs_ids_f = [o2["id"] for o2 in sobs]
5619
+ judged_f = sum(1 for oid in obs_ids_f if (name, d, oid) in curation)
5620
+ if judged_f:
5621
+ comp_rec["fidelity"]["curation_closure"] = round(
5622
+ judged_f / max(len(obs_ids_f), 1), 3)
5623
+ comp_rec["fidelity"]["curation"] = {"judged": judged_f,
5624
+ "total": len(obs_ids_f)}
5570
5625
  rec["compilations"].append(comp_rec)
5571
5626
  impls = rec["compilations"]
5572
5627
  impl_paths = [i["impl"] for i in impls if i["impl"] and os.path.isfile(i["impl"])]
@@ -5660,6 +5715,10 @@ def _render_bench_md(table, anon, recs):
5660
5715
  for i in r["compilations"]:
5661
5716
  fd = i.get("fidelity")
5662
5717
  if fd:
5718
+ cur_v = fd["curation_closure"]
5719
+ if isinstance(cur_v, float):
5720
+ cur_v = "%.3f (judged %d/%d)" % (cur_v, fd["curation"]["judged"],
5721
+ fd["curation"]["total"])
5663
5722
  lines.append("- fidelity `%s` (%s): trace %.2f · surface %d fn / %d cls · "
5664
5723
  "oracle %s · unexplained %.2f · curation %s" % (
5665
5724
  i["dir"], anon.get(i["model"], i["model"] or "?"),
@@ -5667,7 +5726,7 @@ def _render_bench_md(table, anon, recs):
5667
5726
  fd["static_surface"]["classes"],
5668
5727
  ("%.3f" % fd["oracle_passrate"])
5669
5728
  if fd["oracle_passrate"] is not None else "n/a",
5670
- fd["unexplained_share"], fd["curation_closure"]))
5729
+ fd["unexplained_share"], cur_v))
5671
5730
  lines.append("")
5672
5731
  return "\n".join(lines)
5673
5732
 
@@ -5682,7 +5741,22 @@ def cmd_bench(args):
5682
5741
  print("[qa_ledger] bench: no entries under %s (an entry is a subdir with %s)"
5683
5742
  % (args.dir, IR_FILE), file=sys.stderr)
5684
5743
  sys.exit(2)
5685
- recs = [_bench_entry(os.path.join(args.dir, e), e, fidelity=getattr(args, "fidelity", False)) for e in entries]
5744
+ # the store is only ever consumed by the fidelity descriptor (ADR-023): a plain bench
5745
+ # run must not be blocked by a stray/corrupt store it was never going to read
5746
+ cur_map = {}
5747
+ if getattr(args, "fidelity", False):
5748
+ cur_recs, cur_errs = _load_bench_curation(args.dir)
5749
+ if cur_errs:
5750
+ for e in cur_errs:
5751
+ print("[qa_ledger] bench: %s" % e, file=sys.stderr)
5752
+ print("[qa_ledger] bench: a malformed verdict store must not silently degrade "
5753
+ "to UNMEASURED -- fix or remove %s." % _BENCH_CURATION_FILE,
5754
+ file=sys.stderr)
5755
+ sys.exit(2)
5756
+ cur_map = _bench_curation_map(cur_recs)
5757
+ recs = [_bench_entry(os.path.join(args.dir, e), e,
5758
+ fidelity=getattr(args, "fidelity", False), curation=cur_map)
5759
+ for e in entries]
5686
5760
  models = sorted({i["model"] for r in recs for i in r["compilations"] if i.get("model")})
5687
5761
  anon = {m: "M%d" % (k + 1) for k, m in enumerate(models)}
5688
5762
  table = []
@@ -5711,6 +5785,102 @@ def cmd_bench(args):
5711
5785
  sys.exit(0)
5712
5786
 
5713
5787
 
5788
+ def _bench_curable_obs(bench_dir, entry, cdir):
5789
+ """The curable set for one compilation: the SAME observations the M1 static extractor
5790
+ produces for the descriptor's static_surface, re-extracted at call time -- a verdict
5791
+ must judge a real observation, and a fixture edited since --list invalidates the old
5792
+ content-addressed id instead of silently carrying the stale judgment (ADR-023).
5793
+ Returns (obs_list, error_string)."""
5794
+ cd = os.path.join(bench_dir, entry, cdir)
5795
+ cj = os.path.join(cd, "COMPILATION.json")
5796
+ if not os.path.isdir(os.path.join(bench_dir, entry)):
5797
+ return None, "entry %r is not a directory under %s" % (entry, bench_dir)
5798
+ if not os.path.isfile(cj):
5799
+ return None, "%s/%s has no COMPILATION.json -- not a compilation" % (entry, cdir)
5800
+ try:
5801
+ with open(cj, encoding="utf-8-sig") as fh:
5802
+ c = json.load(fh)
5803
+ src = c.get("source") or []
5804
+ unit = src[0].get("unit") if src else None
5805
+ except (OSError, ValueError, AttributeError, IndexError):
5806
+ unit = None
5807
+ if not unit or not os.path.isfile(os.path.join(cd, unit.replace("/", os.sep))):
5808
+ return None, ("%s/%s: no resolvable source unit -- nothing to extract a surface "
5809
+ "from" % (entry, cdir))
5810
+ sobs, _uns = _extract_static_py(cd, [unit])
5811
+ return sobs, None
5812
+
5813
+
5814
+ def cmd_bench_curate(args):
5815
+ """ONE human verdict for ONE observation of ONE bench compilation, appended to the
5816
+ bench-root store (ADR-023). Inherits cmd_curate's refusals verbatim: no batch path
5817
+ exists and will not; an unknown observation is refused, never recorded."""
5818
+ if not os.path.isdir(args.bench):
5819
+ print("[qa_ledger] bench-curate: no directory %s" % args.bench, file=sys.stderr)
5820
+ sys.exit(2)
5821
+ obs_list, err = _bench_curable_obs(args.bench, args.entry, args.dir)
5822
+ if err:
5823
+ print("[qa_ledger] bench-curate: %s" % err, file=sys.stderr)
5824
+ sys.exit(2)
5825
+ records, errors = _load_bench_curation(args.bench)
5826
+ if errors:
5827
+ for e in errors:
5828
+ print("[qa_ledger] bench-curate: %s" % e, file=sys.stderr)
5829
+ sys.exit(2) # fail-closed, never degrade
5830
+ cur_map = _bench_curation_map(records)
5831
+ known = {o["id"] for o in obs_list}
5832
+ if args.list:
5833
+ print("BENCH-CURATE %s/%s: %d curable observation(s)"
5834
+ % (args.entry, args.dir, len(obs_list)))
5835
+ for o in obs_list:
5836
+ v = cur_map.get((args.entry, args.dir, o["id"]))
5837
+ print(" %s %-9s %s" % (o["id"], v or "unjudged", o["statement"]))
5838
+ stale = sorted(r["obs_id"] for r in records
5839
+ if r["entry"] == args.entry and r["dir"] == args.dir
5840
+ and r["obs_id"] not in known)
5841
+ for sid in stale:
5842
+ print(" %s STALE no longer in the extracted surface -- the fixture "
5843
+ "changed after this verdict" % sid)
5844
+ sys.exit(0)
5845
+ if not args.obs or not args.verdict:
5846
+ print("[qa_ledger] bench-curate: --obs and --verdict are required (or --list to "
5847
+ "see what awaits judgment).", file=sys.stderr)
5848
+ sys.exit(2)
5849
+ if args.obs != args.obs.strip() and not re.search(r"[,\s*]", args.obs.strip()):
5850
+ print("[qa_ledger] bench-curate: %r has leading/trailing whitespace -- pass the "
5851
+ "bare OBS id." % args.obs, file=sys.stderr)
5852
+ sys.exit(2)
5853
+ if re.search(r"[,\s*]", args.obs) or args.obs.lower() in ("all", "*"):
5854
+ print("[qa_ledger] bench-curate: %r -- one OBS, one human verdict. A batch-accept "
5855
+ "path does not exist and will not (ADR-023, INV-CURATION-01)." % args.obs,
5856
+ file=sys.stderr)
5857
+ sys.exit(2)
5858
+ if args.obs not in known:
5859
+ print("[qa_ledger] bench-curate: %s is not in the current extracted surface of "
5860
+ "%s/%s -- a verdict must judge a real observation (if the fixture changed, "
5861
+ "re-run --list)." % (args.obs, args.entry, args.dir), file=sys.stderr)
5862
+ sys.exit(2)
5863
+ prev = cur_map.get((args.entry, args.dir, args.obs))
5864
+ human = args.human or os.environ.get("USERNAME") or os.environ.get("USER") or "unknown"
5865
+ records.append({"obs_id": args.obs, "verdict": args.verdict, "human": human,
5866
+ "at": _now(), "note": args.note, "entry": args.entry, "dir": args.dir})
5867
+ store = {"_generated_by": "qa_ledger.py bench-curate (ADR-023) -- append-only human "
5868
+ "verdicts over bench compilations; the LAST record per "
5869
+ "(entry, dir, obs) wins, earlier ones stay as history",
5870
+ "records": records}
5871
+ with open(os.path.join(args.bench, _BENCH_CURATION_FILE), "w", encoding="utf-8",
5872
+ newline="\n") as fh:
5873
+ fh.write(json.dumps(store, indent=2, ensure_ascii=False) + "\n")
5874
+ if prev and prev != args.verdict:
5875
+ print("[qa_ledger] bench-curate: %s/%s %s = %s (supersedes %r -- the earlier "
5876
+ "record stays; append-only, never deleted)"
5877
+ % (args.entry, args.dir, args.obs, args.verdict, prev))
5878
+ else:
5879
+ print("[qa_ledger] bench-curate: %s/%s %s = %s (by %s)"
5880
+ % (args.entry, args.dir, args.obs, args.verdict, human))
5881
+ sys.exit(0)
5882
+
5883
+
5714
5884
  # --------------------------------------------------------------------------- #
5715
5885
  # lang-compare (Diamond controlled-language arm: the SAME canonical package in
5716
5886
  # free prose vs EARS+STE, judged by the SAME withheld oracle, compiled by the
@@ -10039,6 +10209,24 @@ def build_parser():
10039
10209
  pbn.add_argument("--json", action="store_true")
10040
10210
  pbn.set_defaults(func=cmd_bench)
10041
10211
 
10212
+ pbc = sub.add_parser(
10213
+ "bench-curate",
10214
+ help="record ONE human verdict (preserve|fix|undefined) for ONE observation of ONE "
10215
+ "bench compilation, appended to BENCH-CURATION.json; no batch path exists "
10216
+ "(ADR-023, INV-CURATION-01)")
10217
+ pbc.add_argument("--bench", required=True,
10218
+ help="the bench directory (the store lives at its root)")
10219
+ pbc.add_argument("--entry", required=True, help="the archetype entry")
10220
+ pbc.add_argument("--dir", required=True, help="the compilation dir (e.g. c-opus)")
10221
+ pbc.add_argument("--obs", default=None, help="a single OBS id from --list")
10222
+ pbc.add_argument("--verdict", default=None, choices=_BENCH_CURATION_VERDICTS)
10223
+ pbc.add_argument("--note", default=None)
10224
+ pbc.add_argument("--human", default=None, help="who judged (default: the OS user)")
10225
+ pbc.add_argument("--list", action="store_true",
10226
+ help="print the curable observations with their current verdicts "
10227
+ "(read-only), including stale verdicts whose obs no longer exists")
10228
+ pbc.set_defaults(func=cmd_bench_curate)
10229
+
10042
10230
  plc = sub.add_parser(
10043
10231
  "lang-compare",
10044
10232
  help="controlled-language arm (ADR-019): compare a FREE-prose arm and an EARS+STE arm of "
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
3
3
  "name": "uscha",
4
- "version": "1.79.0",
4
+ "version": "1.80.0",
5
5
  "displayName": "Uscha",
6
- "description": "Spec-driven development for LLM coding agents: 9 skills (discovery, adr-refine, reverse-discovery, characterize, devloop, sysdoc, rubric, mirador, status) + a stdlib measurement engine (qa_ledger.py, 48 subcommands + universal installer + npm/npx router). Facts block, guesses advise; the human approves.",
6
+ "description": "Spec-driven development for LLM coding agents: 9 skills (discovery, adr-refine, reverse-discovery, characterize, devloop, sysdoc, rubric, mirador, status) + a stdlib measurement engine (qa_ledger.py, 49 subcommands + universal installer + npm/npx router). Facts block, guesses advise; the human approves.",
7
7
  "author": {
8
8
  "name": "Andres Massello",
9
9
  "url": "https://github.com/andresmassello"
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "uscha",
3
- "version": "1.79.0",
3
+ "version": "1.80.0",
4
4
  "description": "Uscha spec-driven development methodology for coding agents. Includes npm/npx router.",
5
5
  "author": {
6
6
  "name": "Andres Massello",
@@ -1,6 +1,6 @@
1
1
  # uscha-kit
2
2
 
3
- **Kit version:** v1.79.0 <!-- uscha:version --> · **[uscha.dev](https://uscha.dev)**
3
+ **Kit version:** v1.80.0 <!-- uscha:version --> · **[uscha.dev](https://uscha.dev)**
4
4
 
5
5
  Spec-driven orchestrator + multi-repo QA for Claude Code, with a deterministic ledger.
6
6
  **Nine skills** (`uscha-discovery`, `uscha-adr-refine`, `uscha-devloop`, `uscha-sysdoc`, `uscha-reverse-discovery`,
package/uscha-kit/VERSION CHANGED
@@ -1 +1 @@
1
- uscha-kit 1.79.0
1
+ uscha-kit 1.80.0
@@ -0,0 +1 @@
1
+ {"AC-BC-01": true, "AC-BC-02": true, "AC-BC-03": true}
@@ -5480,6 +5480,52 @@ def cmd_bootstrap_variance(args):
5480
5480
  # --------------------------------------------------------------------------- #
5481
5481
  # min oracle pass-rate for a non-green compilation to still count as PARTIAL (core identity).
5482
5482
  _BENCH_PARTIAL_FLOOR = 0.8
5483
+ _BENCH_CURATION_FILE = "BENCH-CURATION.json"
5484
+ _BENCH_CURATION_VERDICTS = ("preserve", "fix", "undefined")
5485
+
5486
+
5487
+ def _load_bench_curation(bench_dir):
5488
+ """Load the append-only bench curation store. Returns (records, errors): a missing
5489
+ file is a legitimate absence ([], None); a malformed one is (None, errors) and every
5490
+ caller fails closed on it -- a broken verdict store must never silently degrade to
5491
+ UNMEASURED, which would hide tampering behind the honest word (ADR-023)."""
5492
+ path = os.path.join(bench_dir, _BENCH_CURATION_FILE)
5493
+ if os.path.exists(path) and not os.path.isfile(path):
5494
+ # a directory (editor swap, broken merge) occupying the store's path must not
5495
+ # read as "no curation exists" -- that is the silent-degrade this loader forbids
5496
+ return None, ["%s exists but is not a regular file" % _BENCH_CURATION_FILE]
5497
+ if not os.path.isfile(path):
5498
+ return [], None
5499
+ try:
5500
+ with open(path, encoding="utf-8-sig") as fh:
5501
+ data = json.load(fh)
5502
+ except (OSError, ValueError) as exc:
5503
+ return None, ["%s unreadable or not JSON: %s" % (_BENCH_CURATION_FILE, exc)]
5504
+ recs = data.get("records") if isinstance(data, dict) else None
5505
+ if not isinstance(recs, list):
5506
+ return None, ["%s has no 'records' list" % _BENCH_CURATION_FILE]
5507
+ errs = []
5508
+ for i, r in enumerate(recs):
5509
+ if not isinstance(r, dict) or any(
5510
+ not (isinstance(r.get(k), str) and r.get(k))
5511
+ for k in ("obs_id", "verdict", "human", "at", "entry", "dir")):
5512
+ errs.append("record %d malformed (obs_id/verdict/human/at/entry/dir "
5513
+ "must be non-empty strings)" % i)
5514
+ elif r["verdict"] not in _BENCH_CURATION_VERDICTS:
5515
+ errs.append("record %d: verdict %r is not one of %s"
5516
+ % (i, r["verdict"], "|".join(_BENCH_CURATION_VERDICTS)))
5517
+ if errs:
5518
+ return None, errs
5519
+ return recs, None
5520
+
5521
+
5522
+ def _bench_curation_map(records):
5523
+ """(entry, dir, obs_id) -> latest verdict. Append-only store: the LAST record for a
5524
+ key wins, earlier ones stay in the file as history (same discipline as `curate`)."""
5525
+ m = {}
5526
+ for r in records:
5527
+ m[(r["entry"], r["dir"], r["obs_id"])] = r["verdict"]
5528
+ return m
5483
5529
 
5484
5530
 
5485
5531
  def _bench_oracle_all(impl_path, cases):
@@ -5489,7 +5535,7 @@ def _bench_oracle_all(impl_path, cases):
5489
5535
  "failing": [r["name"] for r in results if not r["ok"]]}
5490
5536
 
5491
5537
 
5492
- def _bench_entry(entry_dir, name, fidelity=False):
5538
+ def _bench_entry(entry_dir, name, fidelity=False, curation=None):
5493
5539
  """Run compile-validate + the withheld oracle + variance over ONE bench entry and compute
5494
5540
  its verdict. Reuses the M3/M4 organs unchanged; consults no model. A PASS is >=3 oracle-green
5495
5541
  compilations that genuinely differ; PARTIAL is core identity with the divergence isolated;
@@ -5530,8 +5576,9 @@ def _bench_entry(entry_dir, name, fidelity=False):
5530
5576
  if fidelity and impl and os.path.isfile(impl) and unit:
5531
5577
  # the per-compiler fidelity descriptor (ADR-022): the M1 static extractor applied
5532
5578
  # to the compiled artifact -- reverse discovery per compiler. Advisory by
5533
- # construction; curation_closure is UNMEASURED (no human curates fixture code --
5534
- # absence named, never faked).
5579
+ # construction; curation_closure is UNMEASURED unless a human has judged at least
5580
+ # one of this compilation's observations via bench-curate (ADR-023) -- absence
5581
+ # named, never faked, and zero verdicts is an absence, not a 0.0.
5535
5582
  node_ids_f = {nd["id"] for nd in ir_graph.get("nodes") or []}
5536
5583
  covered_f = set()
5537
5584
  units_f, traced_f = set(), set()
@@ -5567,6 +5614,14 @@ def _bench_entry(entry_dir, name, fidelity=False):
5567
5614
  if ores["total"] else None),
5568
5615
  "unexplained_share": round(len(unex) / max(len(units_f), 1), 3),
5569
5616
  "curation_closure": "UNMEASURED"}
5617
+ if curation:
5618
+ obs_ids_f = [o2["id"] for o2 in sobs]
5619
+ judged_f = sum(1 for oid in obs_ids_f if (name, d, oid) in curation)
5620
+ if judged_f:
5621
+ comp_rec["fidelity"]["curation_closure"] = round(
5622
+ judged_f / max(len(obs_ids_f), 1), 3)
5623
+ comp_rec["fidelity"]["curation"] = {"judged": judged_f,
5624
+ "total": len(obs_ids_f)}
5570
5625
  rec["compilations"].append(comp_rec)
5571
5626
  impls = rec["compilations"]
5572
5627
  impl_paths = [i["impl"] for i in impls if i["impl"] and os.path.isfile(i["impl"])]
@@ -5660,6 +5715,10 @@ def _render_bench_md(table, anon, recs):
5660
5715
  for i in r["compilations"]:
5661
5716
  fd = i.get("fidelity")
5662
5717
  if fd:
5718
+ cur_v = fd["curation_closure"]
5719
+ if isinstance(cur_v, float):
5720
+ cur_v = "%.3f (judged %d/%d)" % (cur_v, fd["curation"]["judged"],
5721
+ fd["curation"]["total"])
5663
5722
  lines.append("- fidelity `%s` (%s): trace %.2f · surface %d fn / %d cls · "
5664
5723
  "oracle %s · unexplained %.2f · curation %s" % (
5665
5724
  i["dir"], anon.get(i["model"], i["model"] or "?"),
@@ -5667,7 +5726,7 @@ def _render_bench_md(table, anon, recs):
5667
5726
  fd["static_surface"]["classes"],
5668
5727
  ("%.3f" % fd["oracle_passrate"])
5669
5728
  if fd["oracle_passrate"] is not None else "n/a",
5670
- fd["unexplained_share"], fd["curation_closure"]))
5729
+ fd["unexplained_share"], cur_v))
5671
5730
  lines.append("")
5672
5731
  return "\n".join(lines)
5673
5732
 
@@ -5682,7 +5741,22 @@ def cmd_bench(args):
5682
5741
  print("[qa_ledger] bench: no entries under %s (an entry is a subdir with %s)"
5683
5742
  % (args.dir, IR_FILE), file=sys.stderr)
5684
5743
  sys.exit(2)
5685
- recs = [_bench_entry(os.path.join(args.dir, e), e, fidelity=getattr(args, "fidelity", False)) for e in entries]
5744
+ # the store is only ever consumed by the fidelity descriptor (ADR-023): a plain bench
5745
+ # run must not be blocked by a stray/corrupt store it was never going to read
5746
+ cur_map = {}
5747
+ if getattr(args, "fidelity", False):
5748
+ cur_recs, cur_errs = _load_bench_curation(args.dir)
5749
+ if cur_errs:
5750
+ for e in cur_errs:
5751
+ print("[qa_ledger] bench: %s" % e, file=sys.stderr)
5752
+ print("[qa_ledger] bench: a malformed verdict store must not silently degrade "
5753
+ "to UNMEASURED -- fix or remove %s." % _BENCH_CURATION_FILE,
5754
+ file=sys.stderr)
5755
+ sys.exit(2)
5756
+ cur_map = _bench_curation_map(cur_recs)
5757
+ recs = [_bench_entry(os.path.join(args.dir, e), e,
5758
+ fidelity=getattr(args, "fidelity", False), curation=cur_map)
5759
+ for e in entries]
5686
5760
  models = sorted({i["model"] for r in recs for i in r["compilations"] if i.get("model")})
5687
5761
  anon = {m: "M%d" % (k + 1) for k, m in enumerate(models)}
5688
5762
  table = []
@@ -5711,6 +5785,102 @@ def cmd_bench(args):
5711
5785
  sys.exit(0)
5712
5786
 
5713
5787
 
5788
+ def _bench_curable_obs(bench_dir, entry, cdir):
5789
+ """The curable set for one compilation: the SAME observations the M1 static extractor
5790
+ produces for the descriptor's static_surface, re-extracted at call time -- a verdict
5791
+ must judge a real observation, and a fixture edited since --list invalidates the old
5792
+ content-addressed id instead of silently carrying the stale judgment (ADR-023).
5793
+ Returns (obs_list, error_string)."""
5794
+ cd = os.path.join(bench_dir, entry, cdir)
5795
+ cj = os.path.join(cd, "COMPILATION.json")
5796
+ if not os.path.isdir(os.path.join(bench_dir, entry)):
5797
+ return None, "entry %r is not a directory under %s" % (entry, bench_dir)
5798
+ if not os.path.isfile(cj):
5799
+ return None, "%s/%s has no COMPILATION.json -- not a compilation" % (entry, cdir)
5800
+ try:
5801
+ with open(cj, encoding="utf-8-sig") as fh:
5802
+ c = json.load(fh)
5803
+ src = c.get("source") or []
5804
+ unit = src[0].get("unit") if src else None
5805
+ except (OSError, ValueError, AttributeError, IndexError):
5806
+ unit = None
5807
+ if not unit or not os.path.isfile(os.path.join(cd, unit.replace("/", os.sep))):
5808
+ return None, ("%s/%s: no resolvable source unit -- nothing to extract a surface "
5809
+ "from" % (entry, cdir))
5810
+ sobs, _uns = _extract_static_py(cd, [unit])
5811
+ return sobs, None
5812
+
5813
+
5814
+ def cmd_bench_curate(args):
5815
+ """ONE human verdict for ONE observation of ONE bench compilation, appended to the
5816
+ bench-root store (ADR-023). Inherits cmd_curate's refusals verbatim: no batch path
5817
+ exists and will not; an unknown observation is refused, never recorded."""
5818
+ if not os.path.isdir(args.bench):
5819
+ print("[qa_ledger] bench-curate: no directory %s" % args.bench, file=sys.stderr)
5820
+ sys.exit(2)
5821
+ obs_list, err = _bench_curable_obs(args.bench, args.entry, args.dir)
5822
+ if err:
5823
+ print("[qa_ledger] bench-curate: %s" % err, file=sys.stderr)
5824
+ sys.exit(2)
5825
+ records, errors = _load_bench_curation(args.bench)
5826
+ if errors:
5827
+ for e in errors:
5828
+ print("[qa_ledger] bench-curate: %s" % e, file=sys.stderr)
5829
+ sys.exit(2) # fail-closed, never degrade
5830
+ cur_map = _bench_curation_map(records)
5831
+ known = {o["id"] for o in obs_list}
5832
+ if args.list:
5833
+ print("BENCH-CURATE %s/%s: %d curable observation(s)"
5834
+ % (args.entry, args.dir, len(obs_list)))
5835
+ for o in obs_list:
5836
+ v = cur_map.get((args.entry, args.dir, o["id"]))
5837
+ print(" %s %-9s %s" % (o["id"], v or "unjudged", o["statement"]))
5838
+ stale = sorted(r["obs_id"] for r in records
5839
+ if r["entry"] == args.entry and r["dir"] == args.dir
5840
+ and r["obs_id"] not in known)
5841
+ for sid in stale:
5842
+ print(" %s STALE no longer in the extracted surface -- the fixture "
5843
+ "changed after this verdict" % sid)
5844
+ sys.exit(0)
5845
+ if not args.obs or not args.verdict:
5846
+ print("[qa_ledger] bench-curate: --obs and --verdict are required (or --list to "
5847
+ "see what awaits judgment).", file=sys.stderr)
5848
+ sys.exit(2)
5849
+ if args.obs != args.obs.strip() and not re.search(r"[,\s*]", args.obs.strip()):
5850
+ print("[qa_ledger] bench-curate: %r has leading/trailing whitespace -- pass the "
5851
+ "bare OBS id." % args.obs, file=sys.stderr)
5852
+ sys.exit(2)
5853
+ if re.search(r"[,\s*]", args.obs) or args.obs.lower() in ("all", "*"):
5854
+ print("[qa_ledger] bench-curate: %r -- one OBS, one human verdict. A batch-accept "
5855
+ "path does not exist and will not (ADR-023, INV-CURATION-01)." % args.obs,
5856
+ file=sys.stderr)
5857
+ sys.exit(2)
5858
+ if args.obs not in known:
5859
+ print("[qa_ledger] bench-curate: %s is not in the current extracted surface of "
5860
+ "%s/%s -- a verdict must judge a real observation (if the fixture changed, "
5861
+ "re-run --list)." % (args.obs, args.entry, args.dir), file=sys.stderr)
5862
+ sys.exit(2)
5863
+ prev = cur_map.get((args.entry, args.dir, args.obs))
5864
+ human = args.human or os.environ.get("USERNAME") or os.environ.get("USER") or "unknown"
5865
+ records.append({"obs_id": args.obs, "verdict": args.verdict, "human": human,
5866
+ "at": _now(), "note": args.note, "entry": args.entry, "dir": args.dir})
5867
+ store = {"_generated_by": "qa_ledger.py bench-curate (ADR-023) -- append-only human "
5868
+ "verdicts over bench compilations; the LAST record per "
5869
+ "(entry, dir, obs) wins, earlier ones stay as history",
5870
+ "records": records}
5871
+ with open(os.path.join(args.bench, _BENCH_CURATION_FILE), "w", encoding="utf-8",
5872
+ newline="\n") as fh:
5873
+ fh.write(json.dumps(store, indent=2, ensure_ascii=False) + "\n")
5874
+ if prev and prev != args.verdict:
5875
+ print("[qa_ledger] bench-curate: %s/%s %s = %s (supersedes %r -- the earlier "
5876
+ "record stays; append-only, never deleted)"
5877
+ % (args.entry, args.dir, args.obs, args.verdict, prev))
5878
+ else:
5879
+ print("[qa_ledger] bench-curate: %s/%s %s = %s (by %s)"
5880
+ % (args.entry, args.dir, args.obs, args.verdict, human))
5881
+ sys.exit(0)
5882
+
5883
+
5714
5884
  # --------------------------------------------------------------------------- #
5715
5885
  # lang-compare (Diamond controlled-language arm: the SAME canonical package in
5716
5886
  # free prose vs EARS+STE, judged by the SAME withheld oracle, compiled by the
@@ -10039,6 +10209,24 @@ def build_parser():
10039
10209
  pbn.add_argument("--json", action="store_true")
10040
10210
  pbn.set_defaults(func=cmd_bench)
10041
10211
 
10212
+ pbc = sub.add_parser(
10213
+ "bench-curate",
10214
+ help="record ONE human verdict (preserve|fix|undefined) for ONE observation of ONE "
10215
+ "bench compilation, appended to BENCH-CURATION.json; no batch path exists "
10216
+ "(ADR-023, INV-CURATION-01)")
10217
+ pbc.add_argument("--bench", required=True,
10218
+ help="the bench directory (the store lives at its root)")
10219
+ pbc.add_argument("--entry", required=True, help="the archetype entry")
10220
+ pbc.add_argument("--dir", required=True, help="the compilation dir (e.g. c-opus)")
10221
+ pbc.add_argument("--obs", default=None, help="a single OBS id from --list")
10222
+ pbc.add_argument("--verdict", default=None, choices=_BENCH_CURATION_VERDICTS)
10223
+ pbc.add_argument("--note", default=None)
10224
+ pbc.add_argument("--human", default=None, help="who judged (default: the OS user)")
10225
+ pbc.add_argument("--list", action="store_true",
10226
+ help="print the curable observations with their current verdicts "
10227
+ "(read-only), including stale verdicts whose obs no longer exists")
10228
+ pbc.set_defaults(func=cmd_bench_curate)
10229
+
10042
10230
  plc = sub.add_parser(
10043
10231
  "lang-compare",
10044
10232
  help="controlled-language arm (ADR-019): compare a FREE-prose arm and an EARS+STE arm of "
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "1.79.0",
2
+ "version": "1.80.0",
3
3
  "project": null,
4
4
  "defaults": {
5
5
  "coverage_threshold": 60,