@andresmassello/uscha 1.78.0 → 1.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +1 -1
- package/uscha-kit/.claude/skills/uscha-devloop/qa_ledger.py +246 -4
- package/uscha-kit/.claude-plugin/plugin.json +2 -2
- package/uscha-kit/.codex-plugin/plugin.json +1 -1
- package/uscha-kit/README.md +1 -1
- package/uscha-kit/VERSION +1 -1
- package/uscha-kit/reports/junit/.bench-cases.json +1 -1
- package/uscha-kit/reports/junit/.bench-curate-cases.json +1 -0
- package/uscha-kit/skills/uscha-devloop/qa_ledger.py +246 -4
- package/uscha-kit/uscha.config.json +1 -1
package/README.md
CHANGED
|
@@ -40,7 +40,7 @@ Requires **Python 3.8+** on the machine (the engine is Python stdlib — no pip
|
|
|
40
40
|
runtime dependencies). The npm package is a thin router; the canonical installer is
|
|
41
41
|
`uscha-kit/install-uscha.py`.
|
|
42
42
|
|
|
43
|
-
**Kit v1.
|
|
43
|
+
**Kit v1.80.0** <!-- uscha:version --> · [uscha.dev](https://uscha.dev) ·
|
|
44
44
|
[changelog](https://github.com/andresmassello/uscha/blob/main/uscha-kit/CHANGELOG.md)
|
|
45
45
|
(the per-release changelogs live in the repo, not in the npm tarball)
|
|
46
46
|
|
|
@@ -76,7 +76,7 @@ and see which file, which test, and when.
|
|
|
76
76
|
| `/uscha-mirador` | Bird's-eye HTML dashboard: readiness, trail, acceptance, loops |
|
|
77
77
|
| `/uscha-status` | One-line progress readout, in chat |
|
|
78
78
|
|
|
79
|
-
**A measurement engine** (`qa_ledger.py`,
|
|
79
|
+
**A measurement engine** (`qa_ledger.py`, 49 subcommands, Python stdlib) that ingests
|
|
80
80
|
evidence from **11 language stacks** — maven, gradle, ant, python, node, go, rust, dotnet,
|
|
81
81
|
cpp, swift, flutter — and computes a readiness score with hard caps and visible provenance.
|
|
82
82
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@andresmassello/uscha",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.80.0",
|
|
4
4
|
"description": "Spec-driven development for LLM coding agents: 9 skills + a stdlib evidence engine. Facts block, guesses advise; the human approves.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Andres Massello",
|
|
@@ -5480,6 +5480,52 @@ def cmd_bootstrap_variance(args):
|
|
|
5480
5480
|
# --------------------------------------------------------------------------- #
|
|
5481
5481
|
# min oracle pass-rate for a non-green compilation to still count as PARTIAL (core identity).
|
|
5482
5482
|
_BENCH_PARTIAL_FLOOR = 0.8
|
|
5483
|
+
_BENCH_CURATION_FILE = "BENCH-CURATION.json"
|
|
5484
|
+
_BENCH_CURATION_VERDICTS = ("preserve", "fix", "undefined")
|
|
5485
|
+
|
|
5486
|
+
|
|
5487
|
+
def _load_bench_curation(bench_dir):
|
|
5488
|
+
"""Load the append-only bench curation store. Returns (records, errors): a missing
|
|
5489
|
+
file is a legitimate absence ([], None); a malformed one is (None, errors) and every
|
|
5490
|
+
caller fails closed on it -- a broken verdict store must never silently degrade to
|
|
5491
|
+
UNMEASURED, which would hide tampering behind the honest word (ADR-023)."""
|
|
5492
|
+
path = os.path.join(bench_dir, _BENCH_CURATION_FILE)
|
|
5493
|
+
if os.path.exists(path) and not os.path.isfile(path):
|
|
5494
|
+
# a directory (editor swap, broken merge) occupying the store's path must not
|
|
5495
|
+
# read as "no curation exists" -- that is the silent-degrade this loader forbids
|
|
5496
|
+
return None, ["%s exists but is not a regular file" % _BENCH_CURATION_FILE]
|
|
5497
|
+
if not os.path.isfile(path):
|
|
5498
|
+
return [], None
|
|
5499
|
+
try:
|
|
5500
|
+
with open(path, encoding="utf-8-sig") as fh:
|
|
5501
|
+
data = json.load(fh)
|
|
5502
|
+
except (OSError, ValueError) as exc:
|
|
5503
|
+
return None, ["%s unreadable or not JSON: %s" % (_BENCH_CURATION_FILE, exc)]
|
|
5504
|
+
recs = data.get("records") if isinstance(data, dict) else None
|
|
5505
|
+
if not isinstance(recs, list):
|
|
5506
|
+
return None, ["%s has no 'records' list" % _BENCH_CURATION_FILE]
|
|
5507
|
+
errs = []
|
|
5508
|
+
for i, r in enumerate(recs):
|
|
5509
|
+
if not isinstance(r, dict) or any(
|
|
5510
|
+
not (isinstance(r.get(k), str) and r.get(k))
|
|
5511
|
+
for k in ("obs_id", "verdict", "human", "at", "entry", "dir")):
|
|
5512
|
+
errs.append("record %d malformed (obs_id/verdict/human/at/entry/dir "
|
|
5513
|
+
"must be non-empty strings)" % i)
|
|
5514
|
+
elif r["verdict"] not in _BENCH_CURATION_VERDICTS:
|
|
5515
|
+
errs.append("record %d: verdict %r is not one of %s"
|
|
5516
|
+
% (i, r["verdict"], "|".join(_BENCH_CURATION_VERDICTS)))
|
|
5517
|
+
if errs:
|
|
5518
|
+
return None, errs
|
|
5519
|
+
return recs, None
|
|
5520
|
+
|
|
5521
|
+
|
|
5522
|
+
def _bench_curation_map(records):
|
|
5523
|
+
"""(entry, dir, obs_id) -> latest verdict. Append-only store: the LAST record for a
|
|
5524
|
+
key wins, earlier ones stay in the file as history (same discipline as `curate`)."""
|
|
5525
|
+
m = {}
|
|
5526
|
+
for r in records:
|
|
5527
|
+
m[(r["entry"], r["dir"], r["obs_id"])] = r["verdict"]
|
|
5528
|
+
return m
|
|
5483
5529
|
|
|
5484
5530
|
|
|
5485
5531
|
def _bench_oracle_all(impl_path, cases):
|
|
@@ -5489,7 +5535,7 @@ def _bench_oracle_all(impl_path, cases):
|
|
|
5489
5535
|
"failing": [r["name"] for r in results if not r["ok"]]}
|
|
5490
5536
|
|
|
5491
5537
|
|
|
5492
|
-
def _bench_entry(entry_dir, name):
|
|
5538
|
+
def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
5493
5539
|
"""Run compile-validate + the withheld oracle + variance over ONE bench entry and compute
|
|
5494
5540
|
its verdict. Reuses the M3/M4 organs unchanged; consults no model. A PASS is >=3 oracle-green
|
|
5495
5541
|
compilations that genuinely differ; PARTIAL is core identity with the divergence isolated;
|
|
@@ -5525,8 +5571,58 @@ def _bench_entry(entry_dir, name):
|
|
|
5525
5571
|
impl = os.path.join(cd, unit.replace("/", os.sep)) if unit else None
|
|
5526
5572
|
ores = (_bench_oracle_all(impl, cases) if impl and os.path.isfile(impl)
|
|
5527
5573
|
else {"passed": 0, "total": len(cases), "green": False, "failing": ["<no impl>"]})
|
|
5528
|
-
|
|
5529
|
-
|
|
5574
|
+
comp_rec = {"dir": d, "model": model, "impl": impl,
|
|
5575
|
+
"compile_valid": not errors, "oracle": ores}
|
|
5576
|
+
if fidelity and impl and os.path.isfile(impl) and unit:
|
|
5577
|
+
# the per-compiler fidelity descriptor (ADR-022): the M1 static extractor applied
|
|
5578
|
+
# to the compiled artifact -- reverse discovery per compiler. Advisory by
|
|
5579
|
+
# construction; curation_closure is UNMEASURED unless a human has judged at least
|
|
5580
|
+
# one of this compilation's observations via bench-curate (ADR-023) -- absence
|
|
5581
|
+
# named, never faked, and zero verdicts is an absence, not a 0.0.
|
|
5582
|
+
node_ids_f = {nd["id"] for nd in ir_graph.get("nodes") or []}
|
|
5583
|
+
covered_f = set()
|
|
5584
|
+
units_f, traced_f = set(), set()
|
|
5585
|
+
try:
|
|
5586
|
+
with open(cj, encoding="utf-8-sig") as fh2:
|
|
5587
|
+
c2 = json.load(fh2)
|
|
5588
|
+
for e2 in c2.get("trace_manifest") or []:
|
|
5589
|
+
traced_f.add(e2.get("unit"))
|
|
5590
|
+
for nid in e2.get("implements") or []:
|
|
5591
|
+
if nid in node_ids_f:
|
|
5592
|
+
covered_f.add(nid)
|
|
5593
|
+
for sec in ("source", "tests"):
|
|
5594
|
+
for u2 in c2.get(sec) or []:
|
|
5595
|
+
if u2.get("unit"):
|
|
5596
|
+
units_f.add(u2["unit"])
|
|
5597
|
+
except (OSError, ValueError, AttributeError):
|
|
5598
|
+
pass
|
|
5599
|
+
sobs, _uns = _extract_static_py(cd, [unit])
|
|
5600
|
+
fn_names, cls_names = [], []
|
|
5601
|
+
for o2 in sobs:
|
|
5602
|
+
m2 = re.search(r"defines function (\w+)", o2.get("statement", ""))
|
|
5603
|
+
if m2:
|
|
5604
|
+
fn_names.append(m2.group(1))
|
|
5605
|
+
m2 = re.search(r"defines class (\w+)", o2.get("statement", ""))
|
|
5606
|
+
if m2:
|
|
5607
|
+
cls_names.append(m2.group(1))
|
|
5608
|
+
unex = sorted(u for u in units_f if u not in traced_f)
|
|
5609
|
+
comp_rec["fidelity"] = {
|
|
5610
|
+
"trace_coverage": round(len(covered_f) / max(len(node_ids_f), 1), 3),
|
|
5611
|
+
"static_surface": {"functions": len(fn_names), "classes": len(cls_names),
|
|
5612
|
+
"names": sorted(fn_names + cls_names)},
|
|
5613
|
+
"oracle_passrate": (round(ores["passed"] / ores["total"], 3)
|
|
5614
|
+
if ores["total"] else None),
|
|
5615
|
+
"unexplained_share": round(len(unex) / max(len(units_f), 1), 3),
|
|
5616
|
+
"curation_closure": "UNMEASURED"}
|
|
5617
|
+
if curation:
|
|
5618
|
+
obs_ids_f = [o2["id"] for o2 in sobs]
|
|
5619
|
+
judged_f = sum(1 for oid in obs_ids_f if (name, d, oid) in curation)
|
|
5620
|
+
if judged_f:
|
|
5621
|
+
comp_rec["fidelity"]["curation_closure"] = round(
|
|
5622
|
+
judged_f / max(len(obs_ids_f), 1), 3)
|
|
5623
|
+
comp_rec["fidelity"]["curation"] = {"judged": judged_f,
|
|
5624
|
+
"total": len(obs_ids_f)}
|
|
5625
|
+
rec["compilations"].append(comp_rec)
|
|
5530
5626
|
impls = rec["compilations"]
|
|
5531
5627
|
impl_paths = [i["impl"] for i in impls if i["impl"] and os.path.isfile(i["impl"])]
|
|
5532
5628
|
if len(impl_paths) >= 2:
|
|
@@ -5616,6 +5712,21 @@ def _render_bench_md(table, anon, recs):
|
|
|
5616
5712
|
lines.append("- discrimination stub: %d/%d (%s)" % (
|
|
5617
5713
|
dsc["stub_passed"], dsc["total"],
|
|
5618
5714
|
"NON-DISCRIMINATING" if dsc["stub_green"] else "oracle rejects the stub"))
|
|
5715
|
+
for i in r["compilations"]:
|
|
5716
|
+
fd = i.get("fidelity")
|
|
5717
|
+
if fd:
|
|
5718
|
+
cur_v = fd["curation_closure"]
|
|
5719
|
+
if isinstance(cur_v, float):
|
|
5720
|
+
cur_v = "%.3f (judged %d/%d)" % (cur_v, fd["curation"]["judged"],
|
|
5721
|
+
fd["curation"]["total"])
|
|
5722
|
+
lines.append("- fidelity `%s` (%s): trace %.2f · surface %d fn / %d cls · "
|
|
5723
|
+
"oracle %s · unexplained %.2f · curation %s" % (
|
|
5724
|
+
i["dir"], anon.get(i["model"], i["model"] or "?"),
|
|
5725
|
+
fd["trace_coverage"], fd["static_surface"]["functions"],
|
|
5726
|
+
fd["static_surface"]["classes"],
|
|
5727
|
+
("%.3f" % fd["oracle_passrate"])
|
|
5728
|
+
if fd["oracle_passrate"] is not None else "n/a",
|
|
5729
|
+
fd["unexplained_share"], cur_v))
|
|
5619
5730
|
lines.append("")
|
|
5620
5731
|
return "\n".join(lines)
|
|
5621
5732
|
|
|
@@ -5630,7 +5741,22 @@ def cmd_bench(args):
|
|
|
5630
5741
|
print("[qa_ledger] bench: no entries under %s (an entry is a subdir with %s)"
|
|
5631
5742
|
% (args.dir, IR_FILE), file=sys.stderr)
|
|
5632
5743
|
sys.exit(2)
|
|
5633
|
-
|
|
5744
|
+
# the store is only ever consumed by the fidelity descriptor (ADR-023): a plain bench
|
|
5745
|
+
# run must not be blocked by a stray/corrupt store it was never going to read
|
|
5746
|
+
cur_map = {}
|
|
5747
|
+
if getattr(args, "fidelity", False):
|
|
5748
|
+
cur_recs, cur_errs = _load_bench_curation(args.dir)
|
|
5749
|
+
if cur_errs:
|
|
5750
|
+
for e in cur_errs:
|
|
5751
|
+
print("[qa_ledger] bench: %s" % e, file=sys.stderr)
|
|
5752
|
+
print("[qa_ledger] bench: a malformed verdict store must not silently degrade "
|
|
5753
|
+
"to UNMEASURED -- fix or remove %s." % _BENCH_CURATION_FILE,
|
|
5754
|
+
file=sys.stderr)
|
|
5755
|
+
sys.exit(2)
|
|
5756
|
+
cur_map = _bench_curation_map(cur_recs)
|
|
5757
|
+
recs = [_bench_entry(os.path.join(args.dir, e), e,
|
|
5758
|
+
fidelity=getattr(args, "fidelity", False), curation=cur_map)
|
|
5759
|
+
for e in entries]
|
|
5634
5760
|
models = sorted({i["model"] for r in recs for i in r["compilations"] if i.get("model")})
|
|
5635
5761
|
anon = {m: "M%d" % (k + 1) for k, m in enumerate(models)}
|
|
5636
5762
|
table = []
|
|
@@ -5659,6 +5785,102 @@ def cmd_bench(args):
|
|
|
5659
5785
|
sys.exit(0)
|
|
5660
5786
|
|
|
5661
5787
|
|
|
5788
|
+
def _bench_curable_obs(bench_dir, entry, cdir):
|
|
5789
|
+
"""The curable set for one compilation: the SAME observations the M1 static extractor
|
|
5790
|
+
produces for the descriptor's static_surface, re-extracted at call time -- a verdict
|
|
5791
|
+
must judge a real observation, and a fixture edited since --list invalidates the old
|
|
5792
|
+
content-addressed id instead of silently carrying the stale judgment (ADR-023).
|
|
5793
|
+
Returns (obs_list, error_string)."""
|
|
5794
|
+
cd = os.path.join(bench_dir, entry, cdir)
|
|
5795
|
+
cj = os.path.join(cd, "COMPILATION.json")
|
|
5796
|
+
if not os.path.isdir(os.path.join(bench_dir, entry)):
|
|
5797
|
+
return None, "entry %r is not a directory under %s" % (entry, bench_dir)
|
|
5798
|
+
if not os.path.isfile(cj):
|
|
5799
|
+
return None, "%s/%s has no COMPILATION.json -- not a compilation" % (entry, cdir)
|
|
5800
|
+
try:
|
|
5801
|
+
with open(cj, encoding="utf-8-sig") as fh:
|
|
5802
|
+
c = json.load(fh)
|
|
5803
|
+
src = c.get("source") or []
|
|
5804
|
+
unit = src[0].get("unit") if src else None
|
|
5805
|
+
except (OSError, ValueError, AttributeError, IndexError):
|
|
5806
|
+
unit = None
|
|
5807
|
+
if not unit or not os.path.isfile(os.path.join(cd, unit.replace("/", os.sep))):
|
|
5808
|
+
return None, ("%s/%s: no resolvable source unit -- nothing to extract a surface "
|
|
5809
|
+
"from" % (entry, cdir))
|
|
5810
|
+
sobs, _uns = _extract_static_py(cd, [unit])
|
|
5811
|
+
return sobs, None
|
|
5812
|
+
|
|
5813
|
+
|
|
5814
|
+
def cmd_bench_curate(args):
|
|
5815
|
+
"""ONE human verdict for ONE observation of ONE bench compilation, appended to the
|
|
5816
|
+
bench-root store (ADR-023). Inherits cmd_curate's refusals verbatim: no batch path
|
|
5817
|
+
exists and will not; an unknown observation is refused, never recorded."""
|
|
5818
|
+
if not os.path.isdir(args.bench):
|
|
5819
|
+
print("[qa_ledger] bench-curate: no directory %s" % args.bench, file=sys.stderr)
|
|
5820
|
+
sys.exit(2)
|
|
5821
|
+
obs_list, err = _bench_curable_obs(args.bench, args.entry, args.dir)
|
|
5822
|
+
if err:
|
|
5823
|
+
print("[qa_ledger] bench-curate: %s" % err, file=sys.stderr)
|
|
5824
|
+
sys.exit(2)
|
|
5825
|
+
records, errors = _load_bench_curation(args.bench)
|
|
5826
|
+
if errors:
|
|
5827
|
+
for e in errors:
|
|
5828
|
+
print("[qa_ledger] bench-curate: %s" % e, file=sys.stderr)
|
|
5829
|
+
sys.exit(2) # fail-closed, never degrade
|
|
5830
|
+
cur_map = _bench_curation_map(records)
|
|
5831
|
+
known = {o["id"] for o in obs_list}
|
|
5832
|
+
if args.list:
|
|
5833
|
+
print("BENCH-CURATE %s/%s: %d curable observation(s)"
|
|
5834
|
+
% (args.entry, args.dir, len(obs_list)))
|
|
5835
|
+
for o in obs_list:
|
|
5836
|
+
v = cur_map.get((args.entry, args.dir, o["id"]))
|
|
5837
|
+
print(" %s %-9s %s" % (o["id"], v or "unjudged", o["statement"]))
|
|
5838
|
+
stale = sorted(r["obs_id"] for r in records
|
|
5839
|
+
if r["entry"] == args.entry and r["dir"] == args.dir
|
|
5840
|
+
and r["obs_id"] not in known)
|
|
5841
|
+
for sid in stale:
|
|
5842
|
+
print(" %s STALE no longer in the extracted surface -- the fixture "
|
|
5843
|
+
"changed after this verdict" % sid)
|
|
5844
|
+
sys.exit(0)
|
|
5845
|
+
if not args.obs or not args.verdict:
|
|
5846
|
+
print("[qa_ledger] bench-curate: --obs and --verdict are required (or --list to "
|
|
5847
|
+
"see what awaits judgment).", file=sys.stderr)
|
|
5848
|
+
sys.exit(2)
|
|
5849
|
+
if args.obs != args.obs.strip() and not re.search(r"[,\s*]", args.obs.strip()):
|
|
5850
|
+
print("[qa_ledger] bench-curate: %r has leading/trailing whitespace -- pass the "
|
|
5851
|
+
"bare OBS id." % args.obs, file=sys.stderr)
|
|
5852
|
+
sys.exit(2)
|
|
5853
|
+
if re.search(r"[,\s*]", args.obs) or args.obs.lower() in ("all", "*"):
|
|
5854
|
+
print("[qa_ledger] bench-curate: %r -- one OBS, one human verdict. A batch-accept "
|
|
5855
|
+
"path does not exist and will not (ADR-023, INV-CURATION-01)." % args.obs,
|
|
5856
|
+
file=sys.stderr)
|
|
5857
|
+
sys.exit(2)
|
|
5858
|
+
if args.obs not in known:
|
|
5859
|
+
print("[qa_ledger] bench-curate: %s is not in the current extracted surface of "
|
|
5860
|
+
"%s/%s -- a verdict must judge a real observation (if the fixture changed, "
|
|
5861
|
+
"re-run --list)." % (args.obs, args.entry, args.dir), file=sys.stderr)
|
|
5862
|
+
sys.exit(2)
|
|
5863
|
+
prev = cur_map.get((args.entry, args.dir, args.obs))
|
|
5864
|
+
human = args.human or os.environ.get("USERNAME") or os.environ.get("USER") or "unknown"
|
|
5865
|
+
records.append({"obs_id": args.obs, "verdict": args.verdict, "human": human,
|
|
5866
|
+
"at": _now(), "note": args.note, "entry": args.entry, "dir": args.dir})
|
|
5867
|
+
store = {"_generated_by": "qa_ledger.py bench-curate (ADR-023) -- append-only human "
|
|
5868
|
+
"verdicts over bench compilations; the LAST record per "
|
|
5869
|
+
"(entry, dir, obs) wins, earlier ones stay as history",
|
|
5870
|
+
"records": records}
|
|
5871
|
+
with open(os.path.join(args.bench, _BENCH_CURATION_FILE), "w", encoding="utf-8",
|
|
5872
|
+
newline="\n") as fh:
|
|
5873
|
+
fh.write(json.dumps(store, indent=2, ensure_ascii=False) + "\n")
|
|
5874
|
+
if prev and prev != args.verdict:
|
|
5875
|
+
print("[qa_ledger] bench-curate: %s/%s %s = %s (supersedes %r -- the earlier "
|
|
5876
|
+
"record stays; append-only, never deleted)"
|
|
5877
|
+
% (args.entry, args.dir, args.obs, args.verdict, prev))
|
|
5878
|
+
else:
|
|
5879
|
+
print("[qa_ledger] bench-curate: %s/%s %s = %s (by %s)"
|
|
5880
|
+
% (args.entry, args.dir, args.obs, args.verdict, human))
|
|
5881
|
+
sys.exit(0)
|
|
5882
|
+
|
|
5883
|
+
|
|
5662
5884
|
# --------------------------------------------------------------------------- #
|
|
5663
5885
|
# lang-compare (Diamond controlled-language arm: the SAME canonical package in
|
|
5664
5886
|
# free prose vs EARS+STE, judged by the SAME withheld oracle, compiled by the
|
|
@@ -9982,9 +10204,29 @@ def build_parser():
|
|
|
9982
10204
|
pbn.add_argument("--dir", required=True,
|
|
9983
10205
|
help="the bench directory; each subdir with an IR.json is an entry")
|
|
9984
10206
|
pbn.add_argument("--out", default=None, help="write DIAMOND-BENCH.md here")
|
|
10207
|
+
pbn.add_argument("--fidelity", action="store_true",
|
|
10208
|
+
help="append the per-compiler fidelity descriptor (ADR-022): the M1 static extractor over each compiled source; advisory, never changes a verdict")
|
|
9985
10209
|
pbn.add_argument("--json", action="store_true")
|
|
9986
10210
|
pbn.set_defaults(func=cmd_bench)
|
|
9987
10211
|
|
|
10212
|
+
pbc = sub.add_parser(
|
|
10213
|
+
"bench-curate",
|
|
10214
|
+
help="record ONE human verdict (preserve|fix|undefined) for ONE observation of ONE "
|
|
10215
|
+
"bench compilation, appended to BENCH-CURATION.json; no batch path exists "
|
|
10216
|
+
"(ADR-023, INV-CURATION-01)")
|
|
10217
|
+
pbc.add_argument("--bench", required=True,
|
|
10218
|
+
help="the bench directory (the store lives at its root)")
|
|
10219
|
+
pbc.add_argument("--entry", required=True, help="the archetype entry")
|
|
10220
|
+
pbc.add_argument("--dir", required=True, help="the compilation dir (e.g. c-opus)")
|
|
10221
|
+
pbc.add_argument("--obs", default=None, help="a single OBS id from --list")
|
|
10222
|
+
pbc.add_argument("--verdict", default=None, choices=_BENCH_CURATION_VERDICTS)
|
|
10223
|
+
pbc.add_argument("--note", default=None)
|
|
10224
|
+
pbc.add_argument("--human", default=None, help="who judged (default: the OS user)")
|
|
10225
|
+
pbc.add_argument("--list", action="store_true",
|
|
10226
|
+
help="print the curable observations with their current verdicts "
|
|
10227
|
+
"(read-only), including stale verdicts whose obs no longer exists")
|
|
10228
|
+
pbc.set_defaults(func=cmd_bench_curate)
|
|
10229
|
+
|
|
9988
10230
|
plc = sub.add_parser(
|
|
9989
10231
|
"lang-compare",
|
|
9990
10232
|
help="controlled-language arm (ADR-019): compare a FREE-prose arm and an EARS+STE arm of "
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
|
|
3
3
|
"name": "uscha",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.80.0",
|
|
5
5
|
"displayName": "Uscha",
|
|
6
|
-
"description": "Spec-driven development for LLM coding agents: 9 skills (discovery, adr-refine, reverse-discovery, characterize, devloop, sysdoc, rubric, mirador, status) + a stdlib measurement engine (qa_ledger.py,
|
|
6
|
+
"description": "Spec-driven development for LLM coding agents: 9 skills (discovery, adr-refine, reverse-discovery, characterize, devloop, sysdoc, rubric, mirador, status) + a stdlib measurement engine (qa_ledger.py, 49 subcommands + universal installer + npm/npx router). Facts block, guesses advise; the human approves.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Andres Massello",
|
|
9
9
|
"url": "https://github.com/andresmassello"
|
package/uscha-kit/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# uscha-kit
|
|
2
2
|
|
|
3
|
-
**Kit version:** v1.
|
|
3
|
+
**Kit version:** v1.80.0 <!-- uscha:version --> · **[uscha.dev](https://uscha.dev)**
|
|
4
4
|
|
|
5
5
|
Spec-driven orchestrator + multi-repo QA for Claude Code, with a deterministic ledger.
|
|
6
6
|
**Nine skills** (`uscha-discovery`, `uscha-adr-refine`, `uscha-devloop`, `uscha-sysdoc`, `uscha-reverse-discovery`,
|
package/uscha-kit/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
uscha-kit 1.
|
|
1
|
+
uscha-kit 1.80.0
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"AC-DB-01": true, "AC-DB-02": true, "AC-DB-03": true, "AC-DB-05": true, "AC-DB-04": true, "AC-DB-06": true, "AC-BG-01": true, "AC-BG-02": true, "AC-BG-03": true, "AC-BG-04": true, "AC-BG-05": true}
|
|
1
|
+
{"AC-DB-01": true, "AC-DB-02": true, "AC-DB-03": true, "AC-DB-05": true, "AC-DB-04": true, "AC-DB-06": true, "AC-BG-01": true, "AC-BG-02": true, "AC-BG-03": true, "AC-BG-04": true, "AC-BG-05": true, "AC-FC-01": true, "AC-FC-02": true, "AC-FC-03": true}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"AC-BC-01": true, "AC-BC-02": true, "AC-BC-03": true}
|
|
@@ -5480,6 +5480,52 @@ def cmd_bootstrap_variance(args):
|
|
|
5480
5480
|
# --------------------------------------------------------------------------- #
|
|
5481
5481
|
# min oracle pass-rate for a non-green compilation to still count as PARTIAL (core identity).
|
|
5482
5482
|
_BENCH_PARTIAL_FLOOR = 0.8
|
|
5483
|
+
_BENCH_CURATION_FILE = "BENCH-CURATION.json"
|
|
5484
|
+
_BENCH_CURATION_VERDICTS = ("preserve", "fix", "undefined")
|
|
5485
|
+
|
|
5486
|
+
|
|
5487
|
+
def _load_bench_curation(bench_dir):
|
|
5488
|
+
"""Load the append-only bench curation store. Returns (records, errors): a missing
|
|
5489
|
+
file is a legitimate absence ([], None); a malformed one is (None, errors) and every
|
|
5490
|
+
caller fails closed on it -- a broken verdict store must never silently degrade to
|
|
5491
|
+
UNMEASURED, which would hide tampering behind the honest word (ADR-023)."""
|
|
5492
|
+
path = os.path.join(bench_dir, _BENCH_CURATION_FILE)
|
|
5493
|
+
if os.path.exists(path) and not os.path.isfile(path):
|
|
5494
|
+
# a directory (editor swap, broken merge) occupying the store's path must not
|
|
5495
|
+
# read as "no curation exists" -- that is the silent-degrade this loader forbids
|
|
5496
|
+
return None, ["%s exists but is not a regular file" % _BENCH_CURATION_FILE]
|
|
5497
|
+
if not os.path.isfile(path):
|
|
5498
|
+
return [], None
|
|
5499
|
+
try:
|
|
5500
|
+
with open(path, encoding="utf-8-sig") as fh:
|
|
5501
|
+
data = json.load(fh)
|
|
5502
|
+
except (OSError, ValueError) as exc:
|
|
5503
|
+
return None, ["%s unreadable or not JSON: %s" % (_BENCH_CURATION_FILE, exc)]
|
|
5504
|
+
recs = data.get("records") if isinstance(data, dict) else None
|
|
5505
|
+
if not isinstance(recs, list):
|
|
5506
|
+
return None, ["%s has no 'records' list" % _BENCH_CURATION_FILE]
|
|
5507
|
+
errs = []
|
|
5508
|
+
for i, r in enumerate(recs):
|
|
5509
|
+
if not isinstance(r, dict) or any(
|
|
5510
|
+
not (isinstance(r.get(k), str) and r.get(k))
|
|
5511
|
+
for k in ("obs_id", "verdict", "human", "at", "entry", "dir")):
|
|
5512
|
+
errs.append("record %d malformed (obs_id/verdict/human/at/entry/dir "
|
|
5513
|
+
"must be non-empty strings)" % i)
|
|
5514
|
+
elif r["verdict"] not in _BENCH_CURATION_VERDICTS:
|
|
5515
|
+
errs.append("record %d: verdict %r is not one of %s"
|
|
5516
|
+
% (i, r["verdict"], "|".join(_BENCH_CURATION_VERDICTS)))
|
|
5517
|
+
if errs:
|
|
5518
|
+
return None, errs
|
|
5519
|
+
return recs, None
|
|
5520
|
+
|
|
5521
|
+
|
|
5522
|
+
def _bench_curation_map(records):
|
|
5523
|
+
"""(entry, dir, obs_id) -> latest verdict. Append-only store: the LAST record for a
|
|
5524
|
+
key wins, earlier ones stay in the file as history (same discipline as `curate`)."""
|
|
5525
|
+
m = {}
|
|
5526
|
+
for r in records:
|
|
5527
|
+
m[(r["entry"], r["dir"], r["obs_id"])] = r["verdict"]
|
|
5528
|
+
return m
|
|
5483
5529
|
|
|
5484
5530
|
|
|
5485
5531
|
def _bench_oracle_all(impl_path, cases):
|
|
@@ -5489,7 +5535,7 @@ def _bench_oracle_all(impl_path, cases):
|
|
|
5489
5535
|
"failing": [r["name"] for r in results if not r["ok"]]}
|
|
5490
5536
|
|
|
5491
5537
|
|
|
5492
|
-
def _bench_entry(entry_dir, name):
|
|
5538
|
+
def _bench_entry(entry_dir, name, fidelity=False, curation=None):
|
|
5493
5539
|
"""Run compile-validate + the withheld oracle + variance over ONE bench entry and compute
|
|
5494
5540
|
its verdict. Reuses the M3/M4 organs unchanged; consults no model. A PASS is >=3 oracle-green
|
|
5495
5541
|
compilations that genuinely differ; PARTIAL is core identity with the divergence isolated;
|
|
@@ -5525,8 +5571,58 @@ def _bench_entry(entry_dir, name):
|
|
|
5525
5571
|
impl = os.path.join(cd, unit.replace("/", os.sep)) if unit else None
|
|
5526
5572
|
ores = (_bench_oracle_all(impl, cases) if impl and os.path.isfile(impl)
|
|
5527
5573
|
else {"passed": 0, "total": len(cases), "green": False, "failing": ["<no impl>"]})
|
|
5528
|
-
|
|
5529
|
-
|
|
5574
|
+
comp_rec = {"dir": d, "model": model, "impl": impl,
|
|
5575
|
+
"compile_valid": not errors, "oracle": ores}
|
|
5576
|
+
if fidelity and impl and os.path.isfile(impl) and unit:
|
|
5577
|
+
# the per-compiler fidelity descriptor (ADR-022): the M1 static extractor applied
|
|
5578
|
+
# to the compiled artifact -- reverse discovery per compiler. Advisory by
|
|
5579
|
+
# construction; curation_closure is UNMEASURED unless a human has judged at least
|
|
5580
|
+
# one of this compilation's observations via bench-curate (ADR-023) -- absence
|
|
5581
|
+
# named, never faked, and zero verdicts is an absence, not a 0.0.
|
|
5582
|
+
node_ids_f = {nd["id"] for nd in ir_graph.get("nodes") or []}
|
|
5583
|
+
covered_f = set()
|
|
5584
|
+
units_f, traced_f = set(), set()
|
|
5585
|
+
try:
|
|
5586
|
+
with open(cj, encoding="utf-8-sig") as fh2:
|
|
5587
|
+
c2 = json.load(fh2)
|
|
5588
|
+
for e2 in c2.get("trace_manifest") or []:
|
|
5589
|
+
traced_f.add(e2.get("unit"))
|
|
5590
|
+
for nid in e2.get("implements") or []:
|
|
5591
|
+
if nid in node_ids_f:
|
|
5592
|
+
covered_f.add(nid)
|
|
5593
|
+
for sec in ("source", "tests"):
|
|
5594
|
+
for u2 in c2.get(sec) or []:
|
|
5595
|
+
if u2.get("unit"):
|
|
5596
|
+
units_f.add(u2["unit"])
|
|
5597
|
+
except (OSError, ValueError, AttributeError):
|
|
5598
|
+
pass
|
|
5599
|
+
sobs, _uns = _extract_static_py(cd, [unit])
|
|
5600
|
+
fn_names, cls_names = [], []
|
|
5601
|
+
for o2 in sobs:
|
|
5602
|
+
m2 = re.search(r"defines function (\w+)", o2.get("statement", ""))
|
|
5603
|
+
if m2:
|
|
5604
|
+
fn_names.append(m2.group(1))
|
|
5605
|
+
m2 = re.search(r"defines class (\w+)", o2.get("statement", ""))
|
|
5606
|
+
if m2:
|
|
5607
|
+
cls_names.append(m2.group(1))
|
|
5608
|
+
unex = sorted(u for u in units_f if u not in traced_f)
|
|
5609
|
+
comp_rec["fidelity"] = {
|
|
5610
|
+
"trace_coverage": round(len(covered_f) / max(len(node_ids_f), 1), 3),
|
|
5611
|
+
"static_surface": {"functions": len(fn_names), "classes": len(cls_names),
|
|
5612
|
+
"names": sorted(fn_names + cls_names)},
|
|
5613
|
+
"oracle_passrate": (round(ores["passed"] / ores["total"], 3)
|
|
5614
|
+
if ores["total"] else None),
|
|
5615
|
+
"unexplained_share": round(len(unex) / max(len(units_f), 1), 3),
|
|
5616
|
+
"curation_closure": "UNMEASURED"}
|
|
5617
|
+
if curation:
|
|
5618
|
+
obs_ids_f = [o2["id"] for o2 in sobs]
|
|
5619
|
+
judged_f = sum(1 for oid in obs_ids_f if (name, d, oid) in curation)
|
|
5620
|
+
if judged_f:
|
|
5621
|
+
comp_rec["fidelity"]["curation_closure"] = round(
|
|
5622
|
+
judged_f / max(len(obs_ids_f), 1), 3)
|
|
5623
|
+
comp_rec["fidelity"]["curation"] = {"judged": judged_f,
|
|
5624
|
+
"total": len(obs_ids_f)}
|
|
5625
|
+
rec["compilations"].append(comp_rec)
|
|
5530
5626
|
impls = rec["compilations"]
|
|
5531
5627
|
impl_paths = [i["impl"] for i in impls if i["impl"] and os.path.isfile(i["impl"])]
|
|
5532
5628
|
if len(impl_paths) >= 2:
|
|
@@ -5616,6 +5712,21 @@ def _render_bench_md(table, anon, recs):
|
|
|
5616
5712
|
lines.append("- discrimination stub: %d/%d (%s)" % (
|
|
5617
5713
|
dsc["stub_passed"], dsc["total"],
|
|
5618
5714
|
"NON-DISCRIMINATING" if dsc["stub_green"] else "oracle rejects the stub"))
|
|
5715
|
+
for i in r["compilations"]:
|
|
5716
|
+
fd = i.get("fidelity")
|
|
5717
|
+
if fd:
|
|
5718
|
+
cur_v = fd["curation_closure"]
|
|
5719
|
+
if isinstance(cur_v, float):
|
|
5720
|
+
cur_v = "%.3f (judged %d/%d)" % (cur_v, fd["curation"]["judged"],
|
|
5721
|
+
fd["curation"]["total"])
|
|
5722
|
+
lines.append("- fidelity `%s` (%s): trace %.2f · surface %d fn / %d cls · "
|
|
5723
|
+
"oracle %s · unexplained %.2f · curation %s" % (
|
|
5724
|
+
i["dir"], anon.get(i["model"], i["model"] or "?"),
|
|
5725
|
+
fd["trace_coverage"], fd["static_surface"]["functions"],
|
|
5726
|
+
fd["static_surface"]["classes"],
|
|
5727
|
+
("%.3f" % fd["oracle_passrate"])
|
|
5728
|
+
if fd["oracle_passrate"] is not None else "n/a",
|
|
5729
|
+
fd["unexplained_share"], cur_v))
|
|
5619
5730
|
lines.append("")
|
|
5620
5731
|
return "\n".join(lines)
|
|
5621
5732
|
|
|
@@ -5630,7 +5741,22 @@ def cmd_bench(args):
|
|
|
5630
5741
|
print("[qa_ledger] bench: no entries under %s (an entry is a subdir with %s)"
|
|
5631
5742
|
% (args.dir, IR_FILE), file=sys.stderr)
|
|
5632
5743
|
sys.exit(2)
|
|
5633
|
-
|
|
5744
|
+
# the store is only ever consumed by the fidelity descriptor (ADR-023): a plain bench
|
|
5745
|
+
# run must not be blocked by a stray/corrupt store it was never going to read
|
|
5746
|
+
cur_map = {}
|
|
5747
|
+
if getattr(args, "fidelity", False):
|
|
5748
|
+
cur_recs, cur_errs = _load_bench_curation(args.dir)
|
|
5749
|
+
if cur_errs:
|
|
5750
|
+
for e in cur_errs:
|
|
5751
|
+
print("[qa_ledger] bench: %s" % e, file=sys.stderr)
|
|
5752
|
+
print("[qa_ledger] bench: a malformed verdict store must not silently degrade "
|
|
5753
|
+
"to UNMEASURED -- fix or remove %s." % _BENCH_CURATION_FILE,
|
|
5754
|
+
file=sys.stderr)
|
|
5755
|
+
sys.exit(2)
|
|
5756
|
+
cur_map = _bench_curation_map(cur_recs)
|
|
5757
|
+
recs = [_bench_entry(os.path.join(args.dir, e), e,
|
|
5758
|
+
fidelity=getattr(args, "fidelity", False), curation=cur_map)
|
|
5759
|
+
for e in entries]
|
|
5634
5760
|
models = sorted({i["model"] for r in recs for i in r["compilations"] if i.get("model")})
|
|
5635
5761
|
anon = {m: "M%d" % (k + 1) for k, m in enumerate(models)}
|
|
5636
5762
|
table = []
|
|
@@ -5659,6 +5785,102 @@ def cmd_bench(args):
|
|
|
5659
5785
|
sys.exit(0)
|
|
5660
5786
|
|
|
5661
5787
|
|
|
5788
|
+
def _bench_curable_obs(bench_dir, entry, cdir):
|
|
5789
|
+
"""The curable set for one compilation: the SAME observations the M1 static extractor
|
|
5790
|
+
produces for the descriptor's static_surface, re-extracted at call time -- a verdict
|
|
5791
|
+
must judge a real observation, and a fixture edited since --list invalidates the old
|
|
5792
|
+
content-addressed id instead of silently carrying the stale judgment (ADR-023).
|
|
5793
|
+
Returns (obs_list, error_string)."""
|
|
5794
|
+
cd = os.path.join(bench_dir, entry, cdir)
|
|
5795
|
+
cj = os.path.join(cd, "COMPILATION.json")
|
|
5796
|
+
if not os.path.isdir(os.path.join(bench_dir, entry)):
|
|
5797
|
+
return None, "entry %r is not a directory under %s" % (entry, bench_dir)
|
|
5798
|
+
if not os.path.isfile(cj):
|
|
5799
|
+
return None, "%s/%s has no COMPILATION.json -- not a compilation" % (entry, cdir)
|
|
5800
|
+
try:
|
|
5801
|
+
with open(cj, encoding="utf-8-sig") as fh:
|
|
5802
|
+
c = json.load(fh)
|
|
5803
|
+
src = c.get("source") or []
|
|
5804
|
+
unit = src[0].get("unit") if src else None
|
|
5805
|
+
except (OSError, ValueError, AttributeError, IndexError):
|
|
5806
|
+
unit = None
|
|
5807
|
+
if not unit or not os.path.isfile(os.path.join(cd, unit.replace("/", os.sep))):
|
|
5808
|
+
return None, ("%s/%s: no resolvable source unit -- nothing to extract a surface "
|
|
5809
|
+
"from" % (entry, cdir))
|
|
5810
|
+
sobs, _uns = _extract_static_py(cd, [unit])
|
|
5811
|
+
return sobs, None
|
|
5812
|
+
|
|
5813
|
+
|
|
5814
|
+
def cmd_bench_curate(args):
|
|
5815
|
+
"""ONE human verdict for ONE observation of ONE bench compilation, appended to the
|
|
5816
|
+
bench-root store (ADR-023). Inherits cmd_curate's refusals verbatim: no batch path
|
|
5817
|
+
exists and will not; an unknown observation is refused, never recorded."""
|
|
5818
|
+
if not os.path.isdir(args.bench):
|
|
5819
|
+
print("[qa_ledger] bench-curate: no directory %s" % args.bench, file=sys.stderr)
|
|
5820
|
+
sys.exit(2)
|
|
5821
|
+
obs_list, err = _bench_curable_obs(args.bench, args.entry, args.dir)
|
|
5822
|
+
if err:
|
|
5823
|
+
print("[qa_ledger] bench-curate: %s" % err, file=sys.stderr)
|
|
5824
|
+
sys.exit(2)
|
|
5825
|
+
records, errors = _load_bench_curation(args.bench)
|
|
5826
|
+
if errors:
|
|
5827
|
+
for e in errors:
|
|
5828
|
+
print("[qa_ledger] bench-curate: %s" % e, file=sys.stderr)
|
|
5829
|
+
sys.exit(2) # fail-closed, never degrade
|
|
5830
|
+
cur_map = _bench_curation_map(records)
|
|
5831
|
+
known = {o["id"] for o in obs_list}
|
|
5832
|
+
if args.list:
|
|
5833
|
+
print("BENCH-CURATE %s/%s: %d curable observation(s)"
|
|
5834
|
+
% (args.entry, args.dir, len(obs_list)))
|
|
5835
|
+
for o in obs_list:
|
|
5836
|
+
v = cur_map.get((args.entry, args.dir, o["id"]))
|
|
5837
|
+
print(" %s %-9s %s" % (o["id"], v or "unjudged", o["statement"]))
|
|
5838
|
+
stale = sorted(r["obs_id"] for r in records
|
|
5839
|
+
if r["entry"] == args.entry and r["dir"] == args.dir
|
|
5840
|
+
and r["obs_id"] not in known)
|
|
5841
|
+
for sid in stale:
|
|
5842
|
+
print(" %s STALE no longer in the extracted surface -- the fixture "
|
|
5843
|
+
"changed after this verdict" % sid)
|
|
5844
|
+
sys.exit(0)
|
|
5845
|
+
if not args.obs or not args.verdict:
|
|
5846
|
+
print("[qa_ledger] bench-curate: --obs and --verdict are required (or --list to "
|
|
5847
|
+
"see what awaits judgment).", file=sys.stderr)
|
|
5848
|
+
sys.exit(2)
|
|
5849
|
+
if args.obs != args.obs.strip() and not re.search(r"[,\s*]", args.obs.strip()):
|
|
5850
|
+
print("[qa_ledger] bench-curate: %r has leading/trailing whitespace -- pass the "
|
|
5851
|
+
"bare OBS id." % args.obs, file=sys.stderr)
|
|
5852
|
+
sys.exit(2)
|
|
5853
|
+
if re.search(r"[,\s*]", args.obs) or args.obs.lower() in ("all", "*"):
|
|
5854
|
+
print("[qa_ledger] bench-curate: %r -- one OBS, one human verdict. A batch-accept "
|
|
5855
|
+
"path does not exist and will not (ADR-023, INV-CURATION-01)." % args.obs,
|
|
5856
|
+
file=sys.stderr)
|
|
5857
|
+
sys.exit(2)
|
|
5858
|
+
if args.obs not in known:
|
|
5859
|
+
print("[qa_ledger] bench-curate: %s is not in the current extracted surface of "
|
|
5860
|
+
"%s/%s -- a verdict must judge a real observation (if the fixture changed, "
|
|
5861
|
+
"re-run --list)." % (args.obs, args.entry, args.dir), file=sys.stderr)
|
|
5862
|
+
sys.exit(2)
|
|
5863
|
+
prev = cur_map.get((args.entry, args.dir, args.obs))
|
|
5864
|
+
human = args.human or os.environ.get("USERNAME") or os.environ.get("USER") or "unknown"
|
|
5865
|
+
records.append({"obs_id": args.obs, "verdict": args.verdict, "human": human,
|
|
5866
|
+
"at": _now(), "note": args.note, "entry": args.entry, "dir": args.dir})
|
|
5867
|
+
store = {"_generated_by": "qa_ledger.py bench-curate (ADR-023) -- append-only human "
|
|
5868
|
+
"verdicts over bench compilations; the LAST record per "
|
|
5869
|
+
"(entry, dir, obs) wins, earlier ones stay as history",
|
|
5870
|
+
"records": records}
|
|
5871
|
+
with open(os.path.join(args.bench, _BENCH_CURATION_FILE), "w", encoding="utf-8",
|
|
5872
|
+
newline="\n") as fh:
|
|
5873
|
+
fh.write(json.dumps(store, indent=2, ensure_ascii=False) + "\n")
|
|
5874
|
+
if prev and prev != args.verdict:
|
|
5875
|
+
print("[qa_ledger] bench-curate: %s/%s %s = %s (supersedes %r -- the earlier "
|
|
5876
|
+
"record stays; append-only, never deleted)"
|
|
5877
|
+
% (args.entry, args.dir, args.obs, args.verdict, prev))
|
|
5878
|
+
else:
|
|
5879
|
+
print("[qa_ledger] bench-curate: %s/%s %s = %s (by %s)"
|
|
5880
|
+
% (args.entry, args.dir, args.obs, args.verdict, human))
|
|
5881
|
+
sys.exit(0)
|
|
5882
|
+
|
|
5883
|
+
|
|
5662
5884
|
# --------------------------------------------------------------------------- #
|
|
5663
5885
|
# lang-compare (Diamond controlled-language arm: the SAME canonical package in
|
|
5664
5886
|
# free prose vs EARS+STE, judged by the SAME withheld oracle, compiled by the
|
|
@@ -9982,9 +10204,29 @@ def build_parser():
|
|
|
9982
10204
|
pbn.add_argument("--dir", required=True,
|
|
9983
10205
|
help="the bench directory; each subdir with an IR.json is an entry")
|
|
9984
10206
|
pbn.add_argument("--out", default=None, help="write DIAMOND-BENCH.md here")
|
|
10207
|
+
pbn.add_argument("--fidelity", action="store_true",
|
|
10208
|
+
help="append the per-compiler fidelity descriptor (ADR-022): the M1 static extractor over each compiled source; advisory, never changes a verdict")
|
|
9985
10209
|
pbn.add_argument("--json", action="store_true")
|
|
9986
10210
|
pbn.set_defaults(func=cmd_bench)
|
|
9987
10211
|
|
|
10212
|
+
pbc = sub.add_parser(
|
|
10213
|
+
"bench-curate",
|
|
10214
|
+
help="record ONE human verdict (preserve|fix|undefined) for ONE observation of ONE "
|
|
10215
|
+
"bench compilation, appended to BENCH-CURATION.json; no batch path exists "
|
|
10216
|
+
"(ADR-023, INV-CURATION-01)")
|
|
10217
|
+
pbc.add_argument("--bench", required=True,
|
|
10218
|
+
help="the bench directory (the store lives at its root)")
|
|
10219
|
+
pbc.add_argument("--entry", required=True, help="the archetype entry")
|
|
10220
|
+
pbc.add_argument("--dir", required=True, help="the compilation dir (e.g. c-opus)")
|
|
10221
|
+
pbc.add_argument("--obs", default=None, help="a single OBS id from --list")
|
|
10222
|
+
pbc.add_argument("--verdict", default=None, choices=_BENCH_CURATION_VERDICTS)
|
|
10223
|
+
pbc.add_argument("--note", default=None)
|
|
10224
|
+
pbc.add_argument("--human", default=None, help="who judged (default: the OS user)")
|
|
10225
|
+
pbc.add_argument("--list", action="store_true",
|
|
10226
|
+
help="print the curable observations with their current verdicts "
|
|
10227
|
+
"(read-only), including stale verdicts whose obs no longer exists")
|
|
10228
|
+
pbc.set_defaults(func=cmd_bench_curate)
|
|
10229
|
+
|
|
9988
10230
|
plc = sub.add_parser(
|
|
9989
10231
|
"lang-compare",
|
|
9990
10232
|
help="controlled-language arm (ADR-019): compare a FREE-prose arm and an EARS+STE arm of "
|