@andresmassello/uscha 1.67.0 → 1.69.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +1 -1
- package/uscha-kit/.claude/skills/uscha-devloop/qa_ledger.py +947 -15
- package/uscha-kit/.claude/skills/uscha-reverse-discovery/SKILL.md +60 -44
- package/uscha-kit/.claude-plugin/plugin.json +2 -2
- package/uscha-kit/.codex-plugin/plugin.json +1 -1
- package/uscha-kit/README.md +1 -1
- package/uscha-kit/VERSION +1 -1
- package/uscha-kit/reports/junit/.delta-cases.json +1 -0
- package/uscha-kit/reports/junit/.facts-cases.json +1 -0
- package/uscha-kit/reports/junit/.fidelity-cases.json +1 -0
- package/uscha-kit/skills/uscha-devloop/qa_ledger.py +947 -15
- package/uscha-kit/skills/uscha-reverse-discovery/SKILL.md +60 -44
- package/uscha-kit/uscha.config.json +1 -1
|
@@ -52,6 +52,7 @@ Usage (see `--help` on each subcommand):
|
|
|
52
52
|
"""
|
|
53
53
|
|
|
54
54
|
import argparse
|
|
55
|
+
import ast
|
|
55
56
|
import glob
|
|
56
57
|
import hashlib
|
|
57
58
|
import json
|
|
@@ -2472,6 +2473,10 @@ def cmd_log_gate(args):
|
|
|
2472
2473
|
not-run -> a steps event ONLY, never an iterations record: absence is not
|
|
2473
2474
|
evidence — it neither reads as clean nor fakes a red (last state stands).
|
|
2474
2475
|
"""
|
|
2476
|
+
# INV-ADVISORY-01 note (ADR-014): --kind is a CLOSED vocabulary (argparse choices), so
|
|
2477
|
+
# an advisory-class dimension (e.g. "semantic") cannot be registered as a gate through
|
|
2478
|
+
# this door at all -- the refusal is structural. The smoke suite measures that the
|
|
2479
|
+
# vocabulary stays closed; widening it to admit an advisory kind is a red build.
|
|
2475
2480
|
ledger = _load(args.ledger)
|
|
2476
2481
|
node = _repo_node(ledger, args.repo)
|
|
2477
2482
|
tool = f"gate:{args.kind}"
|
|
@@ -2666,6 +2671,20 @@ def _derive_phase(ledger, name, node, k, qa_order):
|
|
|
2666
2671
|
if len(_cu["unjudged"]) > 3 else "")
|
|
2667
2672
|
+ " -- INV-CURATION-01: sin juicio no hay promocion")
|
|
2668
2673
|
conv = False
|
|
2674
|
+
# CANDIDATE-DELTA gate (ADR-013): same invariant, typed storage. Creating the delta IS
|
|
2675
|
+
# the opt-in; an uncurated OBS blocks exactly like an unjudged .md candidate did.
|
|
2676
|
+
_dl = _delta_state(ledger, name, _cu_cfg.get("path", "."))
|
|
2677
|
+
if _dl is not None:
|
|
2678
|
+
if _dl["malformed"]:
|
|
2679
|
+
reasons.append("CANDIDATE-DELTA invalido: " + "; ".join(_dl["malformed"][:3])
|
|
2680
|
+
+ " -- corregir antes de avanzar")
|
|
2681
|
+
conv = False
|
|
2682
|
+
elif _dl["uncurated"]:
|
|
2683
|
+
reasons.append("OBS sin veredicto humano: " + ", ".join(_dl["uncurated"][:3])
|
|
2684
|
+
+ (" (+%d)" % (len(_dl["uncurated"]) - 3)
|
|
2685
|
+
if len(_dl["uncurated"]) > 3 else "")
|
|
2686
|
+
+ " -- INV-CURATION-01: sin juicio no hay promocion")
|
|
2687
|
+
conv = False
|
|
2669
2688
|
_cr = _cr_cfg(ledger)
|
|
2670
2689
|
if _cr and _cr.get("mode") == "final":
|
|
2671
2690
|
_head = None
|
|
@@ -3758,6 +3777,29 @@ def _curation_state(repo_path):
|
|
|
3758
3777
|
return state
|
|
3759
3778
|
|
|
3760
3779
|
|
|
3780
|
+
_SPEC_MARKER_MAX_BYTES = 2 * 1024 * 1024 # a 2MB+ tracked file is not where a spec-id
|
|
3781
|
+
# marker lives; unbounded reads are the T112 lesson
|
|
3782
|
+
|
|
3783
|
+
|
|
3784
|
+
def _scan_spec_markers(repo_path, tracked):
|
|
3785
|
+
"""Every `uscha-spec: <id>` marker in the given tracked files, as (file, id) pairs.
|
|
3786
|
+
One scanner shared by roundtrip and fidelity (REUSE-FIRST)."""
|
|
3787
|
+
pat = re.compile(r"uscha-spec:\s*([\w.\-]+)")
|
|
3788
|
+
hits = []
|
|
3789
|
+
for f in tracked:
|
|
3790
|
+
full = os.path.join(repo_path, f.replace("/", os.sep))
|
|
3791
|
+
try:
|
|
3792
|
+
if os.path.getsize(full) > _SPEC_MARKER_MAX_BYTES:
|
|
3793
|
+
continue
|
|
3794
|
+
with open(full, encoding="utf-8", errors="replace") as fh:
|
|
3795
|
+
body = fh.read()
|
|
3796
|
+
except OSError:
|
|
3797
|
+
continue
|
|
3798
|
+
for m in pat.finditer(body):
|
|
3799
|
+
hits.append((f, m.group(1)))
|
|
3800
|
+
return hits
|
|
3801
|
+
|
|
3802
|
+
|
|
3761
3803
|
def cmd_roundtrip(args):
|
|
3762
3804
|
"""Advisory spec-id coverage (ADR-009 slice 2, v1): which PROMOTED candidates are
|
|
3763
3805
|
traceable in the code via an embedded `uscha-spec: <candidate>` marker. Coverage by id,
|
|
@@ -3779,21 +3821,8 @@ def cmd_roundtrip(args):
|
|
|
3779
3821
|
and not l.strip().startswith(CANDIDATE_DIR + "/")
|
|
3780
3822
|
and os.path.basename(l.strip()) != BEHAVIOR_LEDGER_FILE]
|
|
3781
3823
|
found = set()
|
|
3782
|
-
|
|
3783
|
-
|
|
3784
|
-
for f in tracked:
|
|
3785
|
-
full = os.path.join(repo_path, f)
|
|
3786
|
-
try:
|
|
3787
|
-
if os.path.getsize(full) > ROUNDTRIP_MAX_BYTES:
|
|
3788
|
-
continue # a 2MB+ tracked file is not where a spec-id marker lives; an
|
|
3789
|
-
# unbounded full-tree read is the T112 lesson, applied here
|
|
3790
|
-
with open(full, encoding="utf-8", errors="replace") as fh:
|
|
3791
|
-
body = fh.read()
|
|
3792
|
-
except OSError:
|
|
3793
|
-
continue
|
|
3794
|
-
for m in pat.finditer(body):
|
|
3795
|
-
mid = m.group(1)
|
|
3796
|
-
found.add(mid if mid.endswith(".md") else mid + ".md")
|
|
3824
|
+
for _f, mid in _scan_spec_markers(repo_path, tracked):
|
|
3825
|
+
found.add(mid if mid.endswith(".md") else mid + ".md")
|
|
3797
3826
|
covered = [c for c in promoted if c in found]
|
|
3798
3827
|
missing = [c for c in promoted if c not in found]
|
|
3799
3828
|
out = {"repo": args.repo, "promoted": len(promoted), "covered": len(covered),
|
|
@@ -3862,6 +3891,840 @@ def cmd_curation_check(args):
|
|
|
3862
3891
|
|
|
3863
3892
|
|
|
3864
3893
|
|
|
3894
|
+
# --------------------------------------------------------------------------- #
|
|
3895
|
+
# CANDIDATE-DELTA (Diamond M1: discovery emits typed observations, verdicts
|
|
3896
|
+
# become ledger objects, fidelity is a vector. ADR-013 / ADR-014.)
|
|
3897
|
+
# --------------------------------------------------------------------------- #
|
|
3898
|
+
|
|
3899
|
+
CANDIDATE_DELTA_FILE = "CANDIDATE-DELTA.json" # under discovery/, machine-canonical
|
|
3900
|
+
CANDIDATE_DELTA_TWIN = "CANDIDATE-DELTA.md" # rendered view, regenerated, never a source
|
|
3901
|
+
CANONICAL_FILE = "CANONICAL.json" # under discovery/: the promoted package
|
|
3902
|
+
ISSUES_DEFERRED_FILE = "ISSUES-DEFERRED.md"
|
|
3903
|
+
OBS_TYPES = ("behavior", "invariant", "contract", "config", "dependency", "decision_trace")
|
|
3904
|
+
EVIDENCE_CLASSES = ("measured", "static", "narrated")
|
|
3905
|
+
# ADR-014 / INV-ADVISORY-01: dimensions an LLM judges can only advise. The QUARANTINE is an
|
|
3906
|
+
# engine invariant: nothing here may ever be registered as a blocking gate.
|
|
3907
|
+
FIDELITY_DIMENSIONS = {
|
|
3908
|
+
"traceability": "measured", "behavior": "measured", "contracts": "measured",
|
|
3909
|
+
"curation_closure": "measured", "unexplained_code": "measured",
|
|
3910
|
+
"semantic": "advisory",
|
|
3911
|
+
}
|
|
3912
|
+
_DELTA_BANNER = ("GENERATED by qa_ledger.py discover (ADR-013). Rendered view of "
|
|
3913
|
+
+ CANDIDATE_DELTA_FILE + " -- hand edits are overwritten on regeneration.")
|
|
3914
|
+
|
|
3915
|
+
|
|
3916
|
+
def _obs_id(otype, statement, primary_prov):
|
|
3917
|
+
"""Content-addressed: OBS-sha256(type + LF + normalized statement + LF + primary
|
|
3918
|
+
provenance)[:12]. Normalization = lowercase + whitespace collapse. Re-running discovery
|
|
3919
|
+
over unchanged code MUST yield byte-identical ids (AC-DD-04) -- the id is the identity
|
|
3920
|
+
of the observation, not of the run."""
|
|
3921
|
+
norm = re.sub(r"\s+", " ", statement.strip().lower())
|
|
3922
|
+
blob = otype + "\n" + norm + "\n" + primary_prov
|
|
3923
|
+
return "OBS-" + hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12]
|
|
3924
|
+
|
|
3925
|
+
|
|
3926
|
+
def _tracked_files(repo_path):
|
|
3927
|
+
r = subprocess.run(["git", "ls-files"], cwd=repo_path, capture_output=True,
|
|
3928
|
+
text=True, encoding="utf-8", errors="replace")
|
|
3929
|
+
if r.returncode != 0:
|
|
3930
|
+
return None # no git: caller decides what that means
|
|
3931
|
+
return [l.strip() for l in r.stdout.splitlines() if l.strip()]
|
|
3932
|
+
|
|
3933
|
+
|
|
3934
|
+
_STATIC_PY_MAX_BYTES = 2 * 1024 * 1024 # the T112 lesson: never unbounded reads
|
|
3935
|
+
|
|
3936
|
+
|
|
3937
|
+
def _extract_static_py(repo_path, tracked):
|
|
3938
|
+
"""v0 static extractors, PYTHON ONLY (ADR-013): public signatures via ast, dependency
|
|
3939
|
+
manifests via requirements*.txt. Deterministic by construction -- if AST/manifest cannot
|
|
3940
|
+
establish it, it is not `static`. Every other stack is UNSUPPORTED: counted and named,
|
|
3941
|
+
never guessed at. Returns (observations, unsupported_count)."""
|
|
3942
|
+
obs, unsupported = [], 0
|
|
3943
|
+
code_exts = (".java", ".kt", ".js", ".ts", ".tsx", ".go", ".rs", ".cs",
|
|
3944
|
+
".cpp", ".c", ".swift", ".dart", ".rb", ".php")
|
|
3945
|
+
for rel in tracked:
|
|
3946
|
+
low = rel.lower()
|
|
3947
|
+
if low.endswith(code_exts):
|
|
3948
|
+
unsupported += 1
|
|
3949
|
+
continue
|
|
3950
|
+
if not low.endswith(".py"):
|
|
3951
|
+
continue
|
|
3952
|
+
full = os.path.join(repo_path, rel.replace("/", os.sep))
|
|
3953
|
+
try:
|
|
3954
|
+
if os.path.getsize(full) > _STATIC_PY_MAX_BYTES:
|
|
3955
|
+
continue
|
|
3956
|
+
with open(full, encoding="utf-8", errors="replace") as fh:
|
|
3957
|
+
tree = ast.parse(fh.read())
|
|
3958
|
+
except (OSError, SyntaxError):
|
|
3959
|
+
continue # unparseable code yields no static facts
|
|
3960
|
+
for nd in tree.body: # top-level only: the PUBLIC surface
|
|
3961
|
+
if isinstance(nd, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
3962
|
+
if nd.name.startswith("_"):
|
|
3963
|
+
continue
|
|
3964
|
+
sig = ", ".join(a.arg for a in nd.args.args)
|
|
3965
|
+
stmt = "%s defines function %s(%s)" % (rel, nd.name, sig)
|
|
3966
|
+
prov = "%s:%d" % (rel, nd.lineno)
|
|
3967
|
+
elif isinstance(nd, ast.ClassDef):
|
|
3968
|
+
if nd.name.startswith("_"):
|
|
3969
|
+
continue
|
|
3970
|
+
stmt = "%s defines class %s" % (rel, nd.name)
|
|
3971
|
+
prov = "%s:%d" % (rel, nd.lineno)
|
|
3972
|
+
else:
|
|
3973
|
+
continue
|
|
3974
|
+
obs.append({"id": _obs_id("contract", stmt, prov), "type": "contract",
|
|
3975
|
+
"statement": stmt, "evidence_class": "static",
|
|
3976
|
+
"provenance": {"files": [prov],
|
|
3977
|
+
"derivation": "AST scan (python ast, top-level defs)",
|
|
3978
|
+
"tool": "qa_ledger-static-py"}})
|
|
3979
|
+
for man in sorted(f for f in tracked
|
|
3980
|
+
if re.match(r"^requirements[^/]*\.txt$", f)):
|
|
3981
|
+
try:
|
|
3982
|
+
with open(os.path.join(repo_path, man), encoding="utf-8",
|
|
3983
|
+
errors="replace") as fh:
|
|
3984
|
+
lines = fh.read().splitlines()
|
|
3985
|
+
except OSError:
|
|
3986
|
+
continue
|
|
3987
|
+
for n, ln in enumerate(lines, 1):
|
|
3988
|
+
s = ln.strip()
|
|
3989
|
+
if not s or s.startswith("#"):
|
|
3990
|
+
continue
|
|
3991
|
+
stmt = "depends on %s (declared in %s)" % (s, man)
|
|
3992
|
+
prov = "%s:%d" % (man, n)
|
|
3993
|
+
obs.append({"id": _obs_id("dependency", stmt, prov), "type": "dependency",
|
|
3994
|
+
"statement": stmt, "evidence_class": "static",
|
|
3995
|
+
"provenance": {"files": [prov],
|
|
3996
|
+
"derivation": "dependency manifest",
|
|
3997
|
+
"tool": "qa_ledger-static-py"}})
|
|
3998
|
+
return obs, unsupported
|
|
3999
|
+
|
|
4000
|
+
|
|
4001
|
+
def _golden_backed_obs(ledger, repo, repo_path):
|
|
4002
|
+
"""Measured observations: one per approved golden fixture, backed by the LATEST ingested
|
|
4003
|
+
golden-diff gate record (AC-DD-03). Only source: real, ledger-ingested execution. No
|
|
4004
|
+
ingested run -> no measured OBS; a fixture on disk that nothing executed is not evidence."""
|
|
4005
|
+
node = ledger["repos"].get(repo) or {}
|
|
4006
|
+
latest = None
|
|
4007
|
+
for rec in node.get("iterations", []):
|
|
4008
|
+
if rec.get("tool") == "gate:golden-diff":
|
|
4009
|
+
latest = rec
|
|
4010
|
+
if latest is None or latest.get("gated_reported", 1) != 0:
|
|
4011
|
+
return [] # no clean ingested run: nothing measured
|
|
4012
|
+
obs = []
|
|
4013
|
+
for root, dirs, files in os.walk(repo_path):
|
|
4014
|
+
dirs[:] = [d for d in dirs if d not in (".git", "node_modules", ".uscha-worktrees")]
|
|
4015
|
+
for f in sorted(files):
|
|
4016
|
+
if ".approved." not in f:
|
|
4017
|
+
continue
|
|
4018
|
+
rel = os.path.relpath(os.path.join(root, f), repo_path).replace(os.sep, "/")
|
|
4019
|
+
stmt = "behavior frozen by golden fixture %s matches the approved baseline" % rel
|
|
4020
|
+
obs.append({"id": _obs_id("behavior", stmt, rel), "type": "behavior",
|
|
4021
|
+
"statement": stmt, "evidence_class": "measured",
|
|
4022
|
+
"provenance": {"files": [rel],
|
|
4023
|
+
"derivation": "golden-diff run ingested %s"
|
|
4024
|
+
% latest.get("at"),
|
|
4025
|
+
"tool": "golden-suite"}})
|
|
4026
|
+
return obs
|
|
4027
|
+
|
|
4028
|
+
|
|
4029
|
+
def _load_narrated(path, repo_path):
|
|
4030
|
+
"""Strict shape for the skill-supplied narrated observations: a JSON list of
|
|
4031
|
+
{type, statement, files}. The CLASS is the engine's to assign -- an input that declares
|
|
4032
|
+
evidence_class (or an id) is malformation, not a suggestion (ADR-013: the skill
|
|
4033
|
+
narrates, the engine classifies). Returns (observations, errors)."""
|
|
4034
|
+
errors = []
|
|
4035
|
+
try:
|
|
4036
|
+
with open(path, encoding="utf-8-sig") as fh:
|
|
4037
|
+
data = json.load(fh)
|
|
4038
|
+
except (OSError, ValueError) as exc:
|
|
4039
|
+
return [], ["unreadable narrated input: %s" % exc]
|
|
4040
|
+
if not isinstance(data, list):
|
|
4041
|
+
return [], ["narrated input must be a JSON list"]
|
|
4042
|
+
obs = []
|
|
4043
|
+
for i, item in enumerate(data):
|
|
4044
|
+
if not isinstance(item, dict):
|
|
4045
|
+
errors.append("item %d: not an object" % i)
|
|
4046
|
+
continue
|
|
4047
|
+
if "evidence_class" in item or "id" in item:
|
|
4048
|
+
errors.append("item %d: declares %s -- the class and id are the ENGINE's to "
|
|
4049
|
+
"assign; a narrated input cannot self-classify (ADR-013)"
|
|
4050
|
+
% (i, "/".join(k for k in ("evidence_class", "id") if k in item)))
|
|
4051
|
+
continue
|
|
4052
|
+
otype = item.get("type")
|
|
4053
|
+
stmt = item.get("statement")
|
|
4054
|
+
stmt = stmt.strip() if isinstance(stmt, str) else ""
|
|
4055
|
+
files = item.get("files") or []
|
|
4056
|
+
if otype not in OBS_TYPES:
|
|
4057
|
+
errors.append("item %d: type %r (expected %s)" % (i, otype, "|".join(OBS_TYPES)))
|
|
4058
|
+
continue
|
|
4059
|
+
if not stmt:
|
|
4060
|
+
errors.append("item %d: empty statement" % i)
|
|
4061
|
+
continue
|
|
4062
|
+
if not isinstance(files, list) or not all(isinstance(x, str) for x in files):
|
|
4063
|
+
# a non-string ref must be a NAMED refusal, never a TypeError traceback
|
|
4064
|
+
# (fresh-review MEDIUM, reproduced)
|
|
4065
|
+
errors.append("item %d: files must be a list of strings" % i)
|
|
4066
|
+
continue
|
|
4067
|
+
bad = None
|
|
4068
|
+
for ref in files:
|
|
4069
|
+
bad = _resolve_ref(repo_path, ref)
|
|
4070
|
+
if bad:
|
|
4071
|
+
errors.append("item %d: ref %r: %s" % (i, ref, bad))
|
|
4072
|
+
break
|
|
4073
|
+
if bad:
|
|
4074
|
+
continue
|
|
4075
|
+
prov = files[0] if files else "agent-inference"
|
|
4076
|
+
obs.append({"id": _obs_id(otype, stmt, prov), "type": otype,
|
|
4077
|
+
"statement": stmt, "evidence_class": "narrated",
|
|
4078
|
+
"provenance": {"files": files, "derivation": "agent inference",
|
|
4079
|
+
"tool": "skill"}})
|
|
4080
|
+
return obs, errors
|
|
4081
|
+
|
|
4082
|
+
|
|
4083
|
+
def _canonical_ids(repo_path, acceptance_file):
|
|
4084
|
+
"""The ids the canonical package answers to today: traceable AC-nn ids from the
|
|
4085
|
+
acceptance file. Match is by ID REFERENCE (ADR-013) -- fuzzy semantic matching is
|
|
4086
|
+
exactly what stays out of scope."""
|
|
4087
|
+
path = acceptance_file if os.path.isabs(acceptance_file) \
|
|
4088
|
+
else os.path.join(repo_path, acceptance_file)
|
|
4089
|
+
if not os.path.isfile(path):
|
|
4090
|
+
return {}
|
|
4091
|
+
ids = {}
|
|
4092
|
+
try:
|
|
4093
|
+
items, _legacy = _parse_acceptance_items(path)
|
|
4094
|
+
except Exception:
|
|
4095
|
+
return {}
|
|
4096
|
+
for it in items or []:
|
|
4097
|
+
if it.get("id"): # normalized "AC-<n>" (numeric ids only)
|
|
4098
|
+
ids[int(it["id"].split("-")[1])] = it["id"]
|
|
4099
|
+
return ids
|
|
4100
|
+
|
|
4101
|
+
|
|
4102
|
+
def _match_canonical(statement, canon_ids):
|
|
4103
|
+
m = re.search(r"(?i)\bAC[-_]?0*(\d+)\b", statement)
|
|
4104
|
+
if m and int(m.group(1)) in canon_ids:
|
|
4105
|
+
return canon_ids[int(m.group(1))]
|
|
4106
|
+
return None
|
|
4107
|
+
|
|
4108
|
+
|
|
4109
|
+
def _render_delta_md(delta, verdicts):
|
|
4110
|
+
lines = ["<!-- %s -->" % _DELTA_BANNER, "",
|
|
4111
|
+
"# CANDIDATE-DELTA (rendered view)", "",
|
|
4112
|
+
"| id | type | class | verdict | statement | provenance |",
|
|
4113
|
+
"|----|------|-------|---------|-----------|------------|"]
|
|
4114
|
+
for o in delta["observations"]:
|
|
4115
|
+
v = verdicts.get(o["id"], "(uncurated)")
|
|
4116
|
+
files = ", ".join(o["provenance"].get("files") or []) or "-"
|
|
4117
|
+
stmt = o["statement"].replace("|", "\\|")
|
|
4118
|
+
lines.append("| %s | %s | %s | %s | %s | %s |"
|
|
4119
|
+
% (o["id"], o["type"], o["evidence_class"], v, stmt, files))
|
|
4120
|
+
lines.append("")
|
|
4121
|
+
return "\n".join(lines)
|
|
4122
|
+
|
|
4123
|
+
|
|
4124
|
+
def _delta_path(repo_path):
|
|
4125
|
+
return os.path.join(repo_path, CANDIDATE_DIR, CANDIDATE_DELTA_FILE)
|
|
4126
|
+
|
|
4127
|
+
|
|
4128
|
+
def _load_delta(repo_path):
|
|
4129
|
+
"""Strict loader (ADR-013): malformation is exit-2 class, never a silent degrade. The
|
|
4130
|
+
id of every OBS is RECOMPUTED -- the delta is mechanically derived and never hand-edited,
|
|
4131
|
+
so an id that no longer matches its content is tampering, said as such. Returns
|
|
4132
|
+
(delta, errors); delta None when the file does not exist (feature unused)."""
|
|
4133
|
+
path = _delta_path(repo_path)
|
|
4134
|
+
if not os.path.isfile(path):
|
|
4135
|
+
return None, []
|
|
4136
|
+
errors = []
|
|
4137
|
+
try:
|
|
4138
|
+
with open(path, encoding="utf-8-sig") as fh:
|
|
4139
|
+
delta = json.load(fh)
|
|
4140
|
+
except (OSError, ValueError) as exc:
|
|
4141
|
+
return {}, ["unreadable: %s" % exc]
|
|
4142
|
+
obs = delta.get("observations")
|
|
4143
|
+
if not isinstance(obs, list):
|
|
4144
|
+
return {}, ["no observations list"]
|
|
4145
|
+
seen = set()
|
|
4146
|
+
for i, o in enumerate(obs):
|
|
4147
|
+
if not isinstance(o, dict):
|
|
4148
|
+
errors.append("observation %d: not an object" % i)
|
|
4149
|
+
continue
|
|
4150
|
+
oid = o.get("id")
|
|
4151
|
+
if o.get("type") not in OBS_TYPES:
|
|
4152
|
+
errors.append("%s: type %r" % (oid or i, o.get("type")))
|
|
4153
|
+
if o.get("evidence_class") not in EVIDENCE_CLASSES:
|
|
4154
|
+
errors.append("%s: evidence_class %r" % (oid or i, o.get("evidence_class")))
|
|
4155
|
+
stmt = o.get("statement")
|
|
4156
|
+
prov = o.get("provenance")
|
|
4157
|
+
files = prov.get("files") if isinstance(prov, dict) else None
|
|
4158
|
+
# SHAPE before use (fresh-review HIGH, reproduced): a provenance that is a list, a
|
|
4159
|
+
# non-string statement or a non-string ref must be a NAMED error, never a traceback
|
|
4160
|
+
# -- the crash would take down phase/dashboard, the read-only readouts.
|
|
4161
|
+
if (not isinstance(stmt, str) or not stmt.strip()
|
|
4162
|
+
or not isinstance(prov, dict) or not isinstance(files, list)
|
|
4163
|
+
or not all(isinstance(x, str) for x in files)):
|
|
4164
|
+
errors.append("%s: statement/provenance shape invalid -- the delta is "
|
|
4165
|
+
"derived, never hand-edited" % (oid or i))
|
|
4166
|
+
elif o.get("type") in OBS_TYPES:
|
|
4167
|
+
primary = files[0] if files else "agent-inference"
|
|
4168
|
+
want = _obs_id(o["type"], stmt, primary)
|
|
4169
|
+
if oid != want:
|
|
4170
|
+
errors.append("%s: id does not match its content (recomputed %s) -- the "
|
|
4171
|
+
"delta is derived, never hand-edited" % (oid, want))
|
|
4172
|
+
if oid in seen:
|
|
4173
|
+
errors.append("%s: duplicate id" % oid)
|
|
4174
|
+
seen.add(oid)
|
|
4175
|
+
# the ids cover type+statement+primary provenance; the SEAL covers everything else
|
|
4176
|
+
# (evidence_class, canonical_match, the full ref list) -- without it, one JSON edit
|
|
4177
|
+
# launders narrated inference into measured evidence (fresh-review MEDIUM, reproduced)
|
|
4178
|
+
if not errors:
|
|
4179
|
+
seal = delta.get("_integrity")
|
|
4180
|
+
want = _integrity_hash({"observations": obs})
|
|
4181
|
+
if seal != want:
|
|
4182
|
+
errors.append("integrity seal %s does not match the observations -- the delta "
|
|
4183
|
+
"is derived, never hand-edited (regenerate via `discover`)"
|
|
4184
|
+
% ("missing" if seal is None else repr(seal)))
|
|
4185
|
+
return delta, errors
|
|
4186
|
+
|
|
4187
|
+
|
|
4188
|
+
def _curation_verdicts(ledger, repo):
|
|
4189
|
+
"""Latest verdict per OBS from the append-only ledger records (re-curation supersedes,
|
|
4190
|
+
never deletes -- every superseded record stays retrievable, AC-CU-05)."""
|
|
4191
|
+
verdicts = {}
|
|
4192
|
+
for rec in ledger.get("curation") or []:
|
|
4193
|
+
if rec.get("repo") == repo:
|
|
4194
|
+
verdicts[rec["obs_id"]] = rec["verdict"]
|
|
4195
|
+
return verdicts
|
|
4196
|
+
|
|
4197
|
+
|
|
4198
|
+
def _delta_state(ledger, repo, repo_path):
|
|
4199
|
+
"""Everything the gates/readouts need about the delta, or None when unused."""
|
|
4200
|
+
delta, errors = _load_delta(repo_path)
|
|
4201
|
+
if delta is None:
|
|
4202
|
+
return None
|
|
4203
|
+
verdicts = _curation_verdicts(ledger, repo)
|
|
4204
|
+
obs = delta.get("observations") or [] if not errors else []
|
|
4205
|
+
uncurated = [o["id"] for o in obs if o.get("id") not in verdicts]
|
|
4206
|
+
undefined = [o["id"] for o in obs
|
|
4207
|
+
if verdicts.get(o.get("id")) == "undefined"]
|
|
4208
|
+
return {"total": len(obs), "curated": len(obs) - len(uncurated),
|
|
4209
|
+
"uncurated": uncurated, "undefined_open": undefined,
|
|
4210
|
+
"malformed": errors}
|
|
4211
|
+
|
|
4212
|
+
|
|
4213
|
+
def cmd_discover(args):
|
|
4214
|
+
"""Emit discovery/CANDIDATE-DELTA.json (ADR-013): typed, content-addressed observations
|
|
4215
|
+
from three strictly separated sources -- measured (ledger-ingested golden runs), static
|
|
4216
|
+
(deterministic extractors, Python-only v0), narrated (skill-supplied inference; the
|
|
4217
|
+
engine classifies and stores, it NEVER calls an LLM). Plus a rendered .md twin."""
|
|
4218
|
+
ledger = _load(args.ledger)
|
|
4219
|
+
_repo_node(ledger, args.repo)
|
|
4220
|
+
repo_path = _scope_path(ledger, args.repo)
|
|
4221
|
+
tracked = _tracked_files(repo_path)
|
|
4222
|
+
if tracked is None:
|
|
4223
|
+
print("[qa_ledger] discover: %s is not a git tree -- discovery derives provenance "
|
|
4224
|
+
"from tracked files and cannot proceed without it." % repo_path,
|
|
4225
|
+
file=sys.stderr)
|
|
4226
|
+
sys.exit(2)
|
|
4227
|
+
static_obs, unsupported = _extract_static_py(repo_path, tracked)
|
|
4228
|
+
measured_obs = _golden_backed_obs(ledger, args.repo, repo_path)
|
|
4229
|
+
narrated_obs, nerrs = ([], [])
|
|
4230
|
+
if args.narrated:
|
|
4231
|
+
narrated_obs, nerrs = _load_narrated(args.narrated, repo_path)
|
|
4232
|
+
if nerrs:
|
|
4233
|
+
for e in nerrs:
|
|
4234
|
+
print("[qa_ledger] discover: %s" % e, file=sys.stderr)
|
|
4235
|
+
sys.exit(2) # malformed input: refuse, never degrade
|
|
4236
|
+
by_id = {}
|
|
4237
|
+
for o in measured_obs + static_obs + narrated_obs:
|
|
4238
|
+
by_id.setdefault(o["id"], o) # identical content = the same observation
|
|
4239
|
+
acc = (args.acceptance
|
|
4240
|
+
or ledger.get("config", {}).get("defaults", {}).get("acceptance_file")
|
|
4241
|
+
or "ACCEPTANCE.md")
|
|
4242
|
+
canon = _canonical_ids(repo_path, acc)
|
|
4243
|
+
for o in by_id.values():
|
|
4244
|
+
o["canonical_match"] = _match_canonical(o["statement"], canon)
|
|
4245
|
+
observations = sorted(by_id.values(), key=lambda o: o["id"])
|
|
4246
|
+
delta = {"_generated_by": "qa_ledger.py discover (ADR-013) -- machine-canonical; "
|
|
4247
|
+
"never hand-edit (ids are content-addressed, the seal "
|
|
4248
|
+
"covers the rest)",
|
|
4249
|
+
"_integrity": _integrity_hash({"observations": observations}),
|
|
4250
|
+
"repo": args.repo,
|
|
4251
|
+
"observations": observations,
|
|
4252
|
+
"static_unsupported": {"files": unsupported,
|
|
4253
|
+
"note": "static extractors are Python-only in v0 "
|
|
4254
|
+
"(ADR-013); other stacks report here, "
|
|
4255
|
+
"never guess"}}
|
|
4256
|
+
disc = os.path.join(repo_path, CANDIDATE_DIR)
|
|
4257
|
+
os.makedirs(disc, exist_ok=True)
|
|
4258
|
+
with open(_delta_path(repo_path), "w", encoding="utf-8", newline="\n") as fh:
|
|
4259
|
+
fh.write(json.dumps(delta, indent=2, ensure_ascii=False) + "\n")
|
|
4260
|
+
twin_path = os.path.join(disc, CANDIDATE_DELTA_TWIN)
|
|
4261
|
+
twin = _render_delta_md(delta, _curation_verdicts(ledger, args.repo))
|
|
4262
|
+
prev = None
|
|
4263
|
+
if os.path.isfile(twin_path):
|
|
4264
|
+
try:
|
|
4265
|
+
with open(twin_path, encoding="utf-8-sig") as fh:
|
|
4266
|
+
prev = fh.read()
|
|
4267
|
+
except OSError:
|
|
4268
|
+
prev = None
|
|
4269
|
+
with open(twin_path, "w", encoding="utf-8", newline="\n") as fh:
|
|
4270
|
+
fh.write(twin)
|
|
4271
|
+
if prev is not None and prev != twin:
|
|
4272
|
+
# stderr, not stdout: on this exact path --json must still emit parseable output
|
|
4273
|
+
# (fresh-review MEDIUM, reproduced)
|
|
4274
|
+
print("[qa_ledger] discover: %s differed from the regenerated render -- overwritten "
|
|
4275
|
+
"(the .md twin is a rendered view, never a source; edit verdicts via `curate`)"
|
|
4276
|
+
% CANDIDATE_DELTA_TWIN, file=sys.stderr)
|
|
4277
|
+
counts = {c: sum(1 for o in observations if o["evidence_class"] == c)
|
|
4278
|
+
for c in EVIDENCE_CLASSES}
|
|
4279
|
+
out = {"repo": args.repo, "observations": len(observations), "by_class": counts,
|
|
4280
|
+
"static_unsupported_files": unsupported,
|
|
4281
|
+
"delta": os.path.join(CANDIDATE_DIR, CANDIDATE_DELTA_FILE)}
|
|
4282
|
+
if args.json:
|
|
4283
|
+
print(json.dumps(out, indent=2, ensure_ascii=False))
|
|
4284
|
+
else:
|
|
4285
|
+
print("DISCOVER %s: %d observation(s) (%d measured / %d static / %d narrated) -> %s"
|
|
4286
|
+
% (args.repo, len(observations), counts["measured"], counts["static"],
|
|
4287
|
+
counts["narrated"], out["delta"]))
|
|
4288
|
+
if unsupported:
|
|
4289
|
+
print(" -- %d non-Python source file(s): static extraction UNSUPPORTED in v0 "
|
|
4290
|
+
"(ADR-013) -- reported, not guessed" % unsupported)
|
|
4291
|
+
sys.exit(0)
|
|
4292
|
+
|
|
4293
|
+
|
|
4294
|
+
def cmd_curate(args):
|
|
4295
|
+
"""ONE human verdict for ONE observation, recorded as an append-only ledger object
|
|
4296
|
+
(ADR-013). No batch path exists -- and this refusal is the assertion of its absence:
|
|
4297
|
+
curation is the human's judgment applied per-item, never a bulk operation."""
|
|
4298
|
+
if re.search(r"[,\s*]", args.obs) or args.obs.lower() in ("all", "*"):
|
|
4299
|
+
print("[qa_ledger] curate: %r -- one OBS, one human verdict. A batch-accept path "
|
|
4300
|
+
"does not exist and will not (ADR-013, INV-CURATION-01)." % args.obs,
|
|
4301
|
+
file=sys.stderr)
|
|
4302
|
+
sys.exit(2)
|
|
4303
|
+
ledger = _load(args.ledger)
|
|
4304
|
+
_repo_node(ledger, args.repo)
|
|
4305
|
+
repo_path = _scope_path(ledger, args.repo)
|
|
4306
|
+
delta, errors = _load_delta(repo_path)
|
|
4307
|
+
if delta is None:
|
|
4308
|
+
print("[qa_ledger] curate: no %s -- run `discover` first."
|
|
4309
|
+
% os.path.join(CANDIDATE_DIR, CANDIDATE_DELTA_FILE), file=sys.stderr)
|
|
4310
|
+
sys.exit(2)
|
|
4311
|
+
if errors:
|
|
4312
|
+
for e in errors:
|
|
4313
|
+
print("[qa_ledger] curate: delta malformed: %s" % e, file=sys.stderr)
|
|
4314
|
+
sys.exit(2)
|
|
4315
|
+
known = {o["id"] for o in delta.get("observations") or []}
|
|
4316
|
+
if args.obs not in known:
|
|
4317
|
+
print("[qa_ledger] curate: %s is not in the current delta -- a verdict must judge "
|
|
4318
|
+
"a real observation." % args.obs, file=sys.stderr)
|
|
4319
|
+
sys.exit(2)
|
|
4320
|
+
prev = _curation_verdicts(ledger, args.repo).get(args.obs)
|
|
4321
|
+
human = args.human or os.environ.get("USERNAME") or os.environ.get("USER") or "unknown"
|
|
4322
|
+
rec = {"obs_id": args.obs, "verdict": args.verdict, "human": human,
|
|
4323
|
+
"at": _now(), "note": args.note, "repo": args.repo}
|
|
4324
|
+
ledger.setdefault("curation", []).append(rec)
|
|
4325
|
+
_save(args.ledger, ledger)
|
|
4326
|
+
if prev and prev != args.verdict:
|
|
4327
|
+
print("[qa_ledger] curate: %s = %s (supersedes %r -- the earlier record stays; "
|
|
4328
|
+
"append-only, never deleted)" % (args.obs, args.verdict, prev))
|
|
4329
|
+
else:
|
|
4330
|
+
print("[qa_ledger] curate: %s = %s (by %s)" % (args.obs, args.verdict, human))
|
|
4331
|
+
sys.exit(0)
|
|
4332
|
+
|
|
4333
|
+
|
|
4334
|
+
def cmd_promote(args):
|
|
4335
|
+
"""Move ONLY preserve-verdict observations into the canonical package, with
|
|
4336
|
+
`derived_from` lineage (ADR-013). fix -> a work item in ISSUES-DEFERRED.md, never
|
|
4337
|
+
canonical. undefined -> stays open in the readouts. ANY uncurated OBS -> hard refusal
|
|
4338
|
+
naming the ids; nothing moves (INV-CURATION-01, fail-closed)."""
|
|
4339
|
+
ledger = _load(args.ledger)
|
|
4340
|
+
_repo_node(ledger, args.repo)
|
|
4341
|
+
repo_path = _scope_path(ledger, args.repo)
|
|
4342
|
+
delta, errors = _load_delta(repo_path)
|
|
4343
|
+
if delta is None:
|
|
4344
|
+
print("[qa_ledger] promote: no %s -- run `discover` first."
|
|
4345
|
+
% os.path.join(CANDIDATE_DIR, CANDIDATE_DELTA_FILE), file=sys.stderr)
|
|
4346
|
+
sys.exit(2)
|
|
4347
|
+
if errors:
|
|
4348
|
+
for e in errors:
|
|
4349
|
+
print("[qa_ledger] promote: delta malformed: %s" % e, file=sys.stderr)
|
|
4350
|
+
sys.exit(2)
|
|
4351
|
+
obs = delta.get("observations") or []
|
|
4352
|
+
verdicts = _curation_verdicts(ledger, args.repo)
|
|
4353
|
+
uncurated = [o["id"] for o in obs if o["id"] not in verdicts]
|
|
4354
|
+
if uncurated:
|
|
4355
|
+
print("[qa_ledger] promote: REFUSED -- %d observation(s) without a human verdict: %s"
|
|
4356
|
+
% (len(uncurated), ", ".join(uncurated[:5])
|
|
4357
|
+
+ (" (+%d)" % (len(uncurated) - 5) if len(uncurated) > 5 else "")),
|
|
4358
|
+
file=sys.stderr)
|
|
4359
|
+
print(" INV-CURATION-01: nothing promotes unjudged. Curate each with "
|
|
4360
|
+
"`curate --obs <id> --verdict preserve|fix|undefined`.", file=sys.stderr)
|
|
4361
|
+
sys.exit(1)
|
|
4362
|
+
canon_path = os.path.join(repo_path, CANDIDATE_DIR, CANONICAL_FILE)
|
|
4363
|
+
canonical = {"_generated_by": "qa_ledger.py promote (ADR-013) -- items carry "
|
|
4364
|
+
"derived_from lineage to their OBS", "items": []}
|
|
4365
|
+
if os.path.isfile(canon_path):
|
|
4366
|
+
try:
|
|
4367
|
+
with open(canon_path, encoding="utf-8-sig") as fh:
|
|
4368
|
+
canonical = json.load(fh)
|
|
4369
|
+
except (OSError, ValueError) as exc:
|
|
4370
|
+
print("[qa_ledger] promote: %s unreadable: %s" % (CANONICAL_FILE, exc),
|
|
4371
|
+
file=sys.stderr)
|
|
4372
|
+
sys.exit(2)
|
|
4373
|
+
have = {it.get("derived_from") for it in canonical.get("items") or []}
|
|
4374
|
+
promoted, fixes, undefined_open = [], [], []
|
|
4375
|
+
for o in obs:
|
|
4376
|
+
v = verdicts[o["id"]]
|
|
4377
|
+
if v == "preserve":
|
|
4378
|
+
if o["id"] not in have:
|
|
4379
|
+
canonical.setdefault("items", []).append(
|
|
4380
|
+
{"statement": o["statement"], "type": o["type"],
|
|
4381
|
+
"evidence_class": o["evidence_class"],
|
|
4382
|
+
"provenance": o["provenance"], "derived_from": o["id"]})
|
|
4383
|
+
promoted.append(o["id"])
|
|
4384
|
+
elif v == "fix":
|
|
4385
|
+
fixes.append(o)
|
|
4386
|
+
else:
|
|
4387
|
+
undefined_open.append(o["id"])
|
|
4388
|
+
canonical["items"] = sorted(canonical.get("items") or [],
|
|
4389
|
+
key=lambda it: it.get("derived_from") or "")
|
|
4390
|
+
with open(canon_path, "w", encoding="utf-8", newline="\n") as fh:
|
|
4391
|
+
fh.write(json.dumps(canonical, indent=2, ensure_ascii=False) + "\n")
|
|
4392
|
+
new_fix = []
|
|
4393
|
+
if fixes:
|
|
4394
|
+
dpath = os.path.join(repo_path, ISSUES_DEFERRED_FILE)
|
|
4395
|
+
existing = ""
|
|
4396
|
+
if os.path.isfile(dpath):
|
|
4397
|
+
with open(dpath, encoding="utf-8-sig", errors="replace") as fh:
|
|
4398
|
+
existing = fh.read()
|
|
4399
|
+
add = [o for o in fixes if o["id"] not in existing]
|
|
4400
|
+
if add:
|
|
4401
|
+
with open(dpath, "a", encoding="utf-8", newline="\n") as fh:
|
|
4402
|
+
if existing and not existing.endswith("\n"):
|
|
4403
|
+
fh.write("\n")
|
|
4404
|
+
for o in add:
|
|
4405
|
+
fh.write("- [ ] %s (curated `fix`): %s -- observed behavior the human "
|
|
4406
|
+
"ruled a defect; NEVER canonical (ADR-013)\n"
|
|
4407
|
+
% (o["id"], o["statement"]))
|
|
4408
|
+
new_fix = [o["id"] for o in add]
|
|
4409
|
+
ledger["candidate_delta"] = {"repo": args.repo, "total": len(obs),
|
|
4410
|
+
"curated": len(obs),
|
|
4411
|
+
"undefined_open": undefined_open,
|
|
4412
|
+
"canonical_items": len(canonical["items"]),
|
|
4413
|
+
"at": _now()}
|
|
4414
|
+
_save(args.ledger, ledger)
|
|
4415
|
+
out = {"repo": args.repo, "promoted": promoted, "fix_deferred": new_fix,
|
|
4416
|
+
"undefined_open": undefined_open,
|
|
4417
|
+
"canonical": os.path.join(CANDIDATE_DIR, CANONICAL_FILE)}
|
|
4418
|
+
if args.json:
|
|
4419
|
+
print(json.dumps(out, indent=2, ensure_ascii=False))
|
|
4420
|
+
else:
|
|
4421
|
+
print("PROMOTE %s: %d promoted, %d fix -> %s, %d undefined OPEN"
|
|
4422
|
+
% (args.repo, len(promoted), len(new_fix), ISSUES_DEFERRED_FILE,
|
|
4423
|
+
len(undefined_open)))
|
|
4424
|
+
for oid in undefined_open:
|
|
4425
|
+
print(" .. %s: undefined -- stays open and visible until re-curated" % oid)
|
|
4426
|
+
sys.exit(0)
|
|
4427
|
+
|
|
4428
|
+
|
|
4429
|
+
def _fid_dim(value, provenance, **extra):
|
|
4430
|
+
d = {"value": value, "provenance": provenance}
|
|
4431
|
+
d.update(extra)
|
|
4432
|
+
return d
|
|
4433
|
+
|
|
4434
|
+
|
|
4435
|
+
def cmd_fidelity(args):
|
|
4436
|
+
"""The fidelity VECTOR (ADR-014): five independently measured dimensions, each with its
|
|
4437
|
+
own provenance, plus the advisory quarantine. Deterministic: same inputs, same numbers,
|
|
4438
|
+
no LLM anywhere in the measured path. Advisory dimensions can NEVER gate -- attempting
|
|
4439
|
+
to configure one as blocking is an engine refusal, not a configuration
|
|
4440
|
+
(INV-ADVISORY-01)."""
|
|
4441
|
+
ledger = _load(args.ledger)
|
|
4442
|
+
node = _repo_node(ledger, args.repo)
|
|
4443
|
+
repo_path = _scope_path(ledger, args.repo)
|
|
4444
|
+
cfg = {}
|
|
4445
|
+
if os.path.isfile(args.config):
|
|
4446
|
+
try:
|
|
4447
|
+
with open(args.config, encoding="utf-8-sig") as fh:
|
|
4448
|
+
cfg = json.load(fh)
|
|
4449
|
+
except (OSError, ValueError) as exc:
|
|
4450
|
+
# a config that cannot be parsed cannot declare gates -- swallowing the error
|
|
4451
|
+
# would DISABLE the INV-ADVISORY-01 refusal on a syntax slip (fresh-review
|
|
4452
|
+
# HIGH, reproduced). Malformation is exit 2, never a silent degrade.
|
|
4453
|
+
print("[qa_ledger] fidelity: %s unreadable: %s -- refusing to guess what it "
|
|
4454
|
+
"declares." % (args.config, exc), file=sys.stderr)
|
|
4455
|
+
sys.exit(2)
|
|
4456
|
+
declared = ((cfg.get("defaults") or {}).get("fidelity") or {}).get("gate") or []
|
|
4457
|
+
for dim in (ledger.get("config", {}).get("defaults", {}).get("fidelity")
|
|
4458
|
+
or {}).get("gate") or []:
|
|
4459
|
+
if dim not in declared: # both surfaces checked -- no silent path
|
|
4460
|
+
declared.append(dim)
|
|
4461
|
+
for dim in declared:
|
|
4462
|
+
cls = FIDELITY_DIMENSIONS.get(dim)
|
|
4463
|
+
if cls is None:
|
|
4464
|
+
print("[qa_ledger] fidelity: %r is not a dimension (%s)"
|
|
4465
|
+
% (dim, ", ".join(sorted(FIDELITY_DIMENSIONS))), file=sys.stderr)
|
|
4466
|
+
sys.exit(2)
|
|
4467
|
+
if cls == "advisory":
|
|
4468
|
+
print("[qa_ledger] fidelity: REFUSED -- %r is an ADVISORY-class dimension and "
|
|
4469
|
+
"can never be registered as blocking. This is an engine invariant "
|
|
4470
|
+
"(INV-ADVISORY-01, ADR-014), not a configuration." % dim, file=sys.stderr)
|
|
4471
|
+
sys.exit(2)
|
|
4472
|
+
print("[qa_ledger] fidelity: gating on measured dimension %r is not implemented "
|
|
4473
|
+
"in v0 (ADR-014) -- declared but unenforceable is a silent lie; remove it "
|
|
4474
|
+
"or wait for the milestone that wires it." % dim, file=sys.stderr)
|
|
4475
|
+
sys.exit(2)
|
|
4476
|
+
delta, derrs = _load_delta(repo_path)
|
|
4477
|
+
if derrs:
|
|
4478
|
+
# same posture as curate/promote: a delta that EXISTS but cannot be validated is
|
|
4479
|
+
# exit 2, and "no delta" must never be the label for it (fresh-review MEDIUM)
|
|
4480
|
+
for e in derrs:
|
|
4481
|
+
print("[qa_ledger] fidelity: delta malformed: %s" % e, file=sys.stderr)
|
|
4482
|
+
sys.exit(2)
|
|
4483
|
+
obs = (delta.get("observations") or []) if delta else []
|
|
4484
|
+
verdicts = _curation_verdicts(ledger, args.repo)
|
|
4485
|
+
dims = {}
|
|
4486
|
+
# traceability: canonical items reachable in code via the uscha-spec id machinery
|
|
4487
|
+
canon_items = []
|
|
4488
|
+
canon_path = os.path.join(repo_path, CANDIDATE_DIR, CANONICAL_FILE)
|
|
4489
|
+
if os.path.isfile(canon_path):
|
|
4490
|
+
try:
|
|
4491
|
+
with open(canon_path, encoding="utf-8-sig") as fh:
|
|
4492
|
+
canon_items = json.load(fh).get("items") or []
|
|
4493
|
+
except (OSError, ValueError):
|
|
4494
|
+
canon_items = []
|
|
4495
|
+
tracked = _tracked_files(repo_path) or []
|
|
4496
|
+
marked, marker_files = set(), set()
|
|
4497
|
+
for f, mid in _scan_spec_markers(repo_path, tracked):
|
|
4498
|
+
marked.add(mid)
|
|
4499
|
+
marker_files.add(f)
|
|
4500
|
+
if canon_items:
|
|
4501
|
+
traced = sum(1 for it in canon_items if (it.get("derived_from") or "") in marked)
|
|
4502
|
+
dims["traceability"] = _fid_dim(round(traced / len(canon_items), 4),
|
|
4503
|
+
"uscha-spec marker scan over %d git-tracked "
|
|
4504
|
+
"files vs %d canonical item(s)"
|
|
4505
|
+
% (len(tracked), len(canon_items)))
|
|
4506
|
+
else:
|
|
4507
|
+
dims["traceability"] = _fid_dim(None, "UNMEASURED: no canonical items promoted yet")
|
|
4508
|
+
# behavior: the latest ingested golden-diff gate verdict
|
|
4509
|
+
latest_g = None
|
|
4510
|
+
for rec in node.get("iterations", []):
|
|
4511
|
+
if rec.get("tool") == "gate:golden-diff":
|
|
4512
|
+
latest_g = rec
|
|
4513
|
+
if latest_g is not None:
|
|
4514
|
+
ok = latest_g.get("gated_reported", 1) == 0
|
|
4515
|
+
prov = "gate:golden-diff record ingested %s (iteration %s)" % (
|
|
4516
|
+
latest_g.get("at"), latest_g.get("iteration"))
|
|
4517
|
+
cr = _cr_latest(ledger, args.repo)
|
|
4518
|
+
if cr and cr.get("status") == "GREEN":
|
|
4519
|
+
prov += "; clean-room GREEN at %s" % (cr.get("ref") or "?")[:12]
|
|
4520
|
+
dims["behavior"] = _fid_dim(1.0 if ok else 0.0, prov)
|
|
4521
|
+
else:
|
|
4522
|
+
dims["behavior"] = _fid_dim(None, "UNMEASURED: no golden-diff run ingested "
|
|
4523
|
+
"(log-gate --kind golden-diff)")
|
|
4524
|
+
# contracts: canonical static items still derivable from the code RIGHT NOW
|
|
4525
|
+
static_canon = [it for it in canon_items if it.get("evidence_class") == "static"]
|
|
4526
|
+
if static_canon:
|
|
4527
|
+
cur_static, _u = _extract_static_py(repo_path, tracked)
|
|
4528
|
+
cur_ids = {o["id"] for o in cur_static}
|
|
4529
|
+
okc = sum(1 for it in static_canon if it.get("derived_from") in cur_ids)
|
|
4530
|
+
dims["contracts"] = _fid_dim(round(okc / len(static_canon), 4),
|
|
4531
|
+
"re-ran static extractors; %d/%d canonical static "
|
|
4532
|
+
"item(s) still derive from the code"
|
|
4533
|
+
% (okc, len(static_canon)))
|
|
4534
|
+
else:
|
|
4535
|
+
dims["contracts"] = _fid_dim(None, "UNMEASURED: no static-class canonical items")
|
|
4536
|
+
# curation_closure: curated OBS / total OBS in the active delta
|
|
4537
|
+
if obs:
|
|
4538
|
+
cur = sum(1 for o in obs if o["id"] in verdicts)
|
|
4539
|
+
dims["curation_closure"] = _fid_dim(round(cur / len(obs), 4),
|
|
4540
|
+
"%d/%d OBS in %s carry a ledger verdict"
|
|
4541
|
+
% (cur, len(obs), CANDIDATE_DELTA_FILE))
|
|
4542
|
+
else:
|
|
4543
|
+
dims["curation_closure"] = _fid_dim(
|
|
4544
|
+
None, "UNMEASURED: no delta" if delta is None
|
|
4545
|
+
else "no observations in the delta")
|
|
4546
|
+
# unexplained_code: v0 deliberately crude -- unit = source FILE (ADR-014: crude and
|
|
4547
|
+
# honest beats fine-grained and narrated; over-fires on monoliths BY DESIGN)
|
|
4548
|
+
rtype = (_repo_cfg(ledger, args.repo).get("type")
|
|
4549
|
+
if args.repo != "integration" else None)
|
|
4550
|
+
src_exts = (".py", ".java", ".kt", ".js", ".ts", ".tsx", ".go", ".rs", ".cs",
|
|
4551
|
+
".cpp", ".c", ".swift", ".dart", ".rb", ".php")
|
|
4552
|
+
prod = [f for f in tracked
|
|
4553
|
+
if f.lower().endswith(src_exts)
|
|
4554
|
+
and not f.startswith(CANDIDATE_DIR + "/")
|
|
4555
|
+
and not _is_test_path(f, rtype)]
|
|
4556
|
+
lineage = set(marker_files) # a file CARRYING a spec marker is explained
|
|
4557
|
+
for o in obs:
|
|
4558
|
+
if verdicts.get(o["id"]) == "preserve":
|
|
4559
|
+
for ref in o["provenance"].get("files") or []:
|
|
4560
|
+
lineage.add(ref.split(":")[0].split("#")[0])
|
|
4561
|
+
for it in canon_items:
|
|
4562
|
+
for ref in (it.get("provenance") or {}).get("files") or []:
|
|
4563
|
+
lineage.add(ref.split(":")[0].split("#")[0])
|
|
4564
|
+
if prod:
|
|
4565
|
+
unex = [f for f in prod if f not in lineage]
|
|
4566
|
+
dims["unexplained_code"] = _fid_dim(
|
|
4567
|
+
round(len(unex) / len(prod), 4),
|
|
4568
|
+
"%d/%d tracked prod source file(s) with no path to a canonical item or "
|
|
4569
|
+
"preserved OBS (v0 granularity: FILE)" % (len(unex), len(prod)),
|
|
4570
|
+
files=unex[:10] + (["(+%d)" % (len(unex) - 10)] if len(unex) > 10 else []))
|
|
4571
|
+
else:
|
|
4572
|
+
dims["unexplained_code"] = _fid_dim(None, "UNMEASURED: no tracked prod source files")
|
|
4573
|
+
dims["semantic"] = _fid_dim(None, "not wired: an LLM-judged comparison enters as "
|
|
4574
|
+
"advisory only and can NEVER gate (INV-ADVISORY-01)")
|
|
4575
|
+
out = {"repo": args.repo,
|
|
4576
|
+
"dimensions": {k: dict(dims[k], **{"class": FIDELITY_DIMENSIONS[k]})
|
|
4577
|
+
for k in ("traceability", "behavior", "contracts",
|
|
4578
|
+
"curation_closure", "unexplained_code", "semantic")}}
|
|
4579
|
+
ledger["fidelity"] = dict(out, at=_now())
|
|
4580
|
+
_save(args.ledger, ledger)
|
|
4581
|
+
if args.json:
|
|
4582
|
+
print(json.dumps(out, indent=2, ensure_ascii=False))
|
|
4583
|
+
else:
|
|
4584
|
+
print("FIDELITY %s (vector -- no blend; each number stands on its own evidence):"
|
|
4585
|
+
% args.repo)
|
|
4586
|
+
for k, d in out["dimensions"].items():
|
|
4587
|
+
val = "UNMEASURED" if d["value"] is None else "%.2f" % d["value"]
|
|
4588
|
+
print(" %-17s %-10s [%s] %s" % (k, val, d["class"], d["provenance"]))
|
|
4589
|
+
sys.exit(0)
|
|
4590
|
+
|
|
4591
|
+
|
|
4592
|
+
# --------------------------------------------------------------------------- #
|
|
4593
|
+
# facts (T0 / SYSTEM-FACTS: public claims become compiled artifacts of repo
|
|
4594
|
+
# facts -- Diamond applied to Diamond. ADR-012.)
|
|
4595
|
+
# --------------------------------------------------------------------------- #
|
|
4596
|
+
|
|
4597
|
+
FACTS_FILE = "SYSTEM-FACTS.json"
|
|
4598
|
+
|
|
4599
|
+
|
|
4600
|
+
def _derive_facts():
|
|
4601
|
+
"""Facts derived from the ARTIFACTS themselves, never from prose and never from greps
|
|
4602
|
+
over documentation: the subcommand list comes from introspecting the REAL parser, the
|
|
4603
|
+
skill list from the REAL kit tree, the version from the kit VERSION file. No timestamp
|
|
4604
|
+
on purpose: regeneration over an unchanged repo must be byte-identical (AC-SF-01)."""
|
|
4605
|
+
here = os.path.abspath(__file__)
|
|
4606
|
+
# kit root by MARKER, not by fixed depth: the canonical engine sits 4 levels deep
|
|
4607
|
+
# (.claude/skills/uscha-devloop/) and the Codex twin 3 (skills/uscha-devloop/). A fixed
|
|
4608
|
+
# dirname walk made the twin silently derive the OUTER repo root -- version None,
|
|
4609
|
+
# 0 skills, no error (fresh-review HIGH, reproduced by running both copies).
|
|
4610
|
+
kit, cur = None, os.path.dirname(here)
|
|
4611
|
+
for _ in range(6):
|
|
4612
|
+
if os.path.isfile(os.path.join(cur, "VERSION")):
|
|
4613
|
+
kit = cur
|
|
4614
|
+
break
|
|
4615
|
+
nxt = os.path.dirname(cur)
|
|
4616
|
+
if nxt == cur:
|
|
4617
|
+
break
|
|
4618
|
+
cur = nxt
|
|
4619
|
+
if kit is None:
|
|
4620
|
+
print("[qa_ledger] facts: no VERSION file found walking up from the engine -- "
|
|
4621
|
+
"facts that cannot locate their own kit are not facts.", file=sys.stderr)
|
|
4622
|
+
sys.exit(2)
|
|
4623
|
+
with open(os.path.join(kit, "VERSION"), encoding="utf-8") as fh:
|
|
4624
|
+
version = fh.read().split()[-1].strip()
|
|
4625
|
+
subs = []
|
|
4626
|
+
for action in build_parser()._subparsers._group_actions:
|
|
4627
|
+
subs = sorted(action.choices.keys())
|
|
4628
|
+
skills = []
|
|
4629
|
+
for sdir in (os.path.join(kit, ".claude", "skills"), os.path.join(kit, "skills")):
|
|
4630
|
+
# canonical tree first; the Codex install ships only skills/ -- same inventory
|
|
4631
|
+
if os.path.isdir(sdir):
|
|
4632
|
+
skills = sorted(d for d in os.listdir(sdir)
|
|
4633
|
+
if os.path.isfile(os.path.join(sdir, d, "SKILL.md")))
|
|
4634
|
+
break
|
|
4635
|
+
return {
|
|
4636
|
+
"version": version,
|
|
4637
|
+
"subcommands": {"count": len(subs), "list": subs},
|
|
4638
|
+
"skills": {"count": len(skills), "list": skills},
|
|
4639
|
+
"_derivation": {
|
|
4640
|
+
"version": "uscha-kit/VERSION",
|
|
4641
|
+
"subcommands": "argparse introspection of build_parser()",
|
|
4642
|
+
"skills": "SKILL.md inventory under uscha-kit/.claude/skills/",
|
|
4643
|
+
"omitted": "stack matrix and REAL/VISION registry: no mechanical "
|
|
4644
|
+
"source exists yet -- omitted, not guessed (ADR-012)",
|
|
4645
|
+
},
|
|
4646
|
+
}
|
|
4647
|
+
|
|
4648
|
+
|
|
4649
|
+
_CLAIM_PATTERNS = (
|
|
4650
|
+
# (fact key path, regex over one line, needs-context substring or None)
|
|
4651
|
+
("version", r"v(\d+\.\d+\.\d+)", "kit"),
|
|
4652
|
+
("version", r"uscha-kit\s+v?(\d+\.\d+\.\d+)", None),
|
|
4653
|
+
("subcommands.count", r"(\d+)\s+sub-?comm?ands", None),
|
|
4654
|
+
("subcommands.count", r"(\d+)\s+subcomandos", None),
|
|
4655
|
+
("skills.count", r"(\d+)\s+skills", None),
|
|
4656
|
+
)
|
|
4657
|
+
|
|
4658
|
+
|
|
4659
|
+
def _fact_value(facts, dotted):
|
|
4660
|
+
cur = facts
|
|
4661
|
+
for part in dotted.split("."):
|
|
4662
|
+
cur = cur[part]
|
|
4663
|
+
return str(cur)
|
|
4664
|
+
|
|
4665
|
+
|
|
4666
|
+
def cmd_facts(args):
|
|
4667
|
+
"""Generate SYSTEM-FACTS.json, or --check published claims against the derived facts.
|
|
4668
|
+
|
|
4669
|
+
The founding fixture (recorded in ADR-012): the site claimed kit 1.65.0 with 32 engine
|
|
4670
|
+
subcommands while the repo was at 1.67.0 with 35 -- factual drift, live, in the project
|
|
4671
|
+
about factual drift. A claim that CI does not compare against a derived fact will
|
|
4672
|
+
drift; this makes the comparison mechanical and the drift a named red."""
|
|
4673
|
+
facts = _derive_facts()
|
|
4674
|
+
if args.check:
|
|
4675
|
+
problems = []
|
|
4676
|
+
# 1) the committed facts file must match a fresh derivation (stale facts are drift)
|
|
4677
|
+
if os.path.isfile(args.out):
|
|
4678
|
+
with open(args.out, encoding="utf-8") as fh:
|
|
4679
|
+
committed = fh.read()
|
|
4680
|
+
fresh = json.dumps(facts, indent=2, ensure_ascii=False, sort_keys=True) + "\n"
|
|
4681
|
+
if committed.replace("\r\n", "\n") != fresh:
|
|
4682
|
+
problems.append((args.out, 0, "committed facts file",
|
|
4683
|
+
"stale vs regenerated", "run: qa_ledger.py facts"))
|
|
4684
|
+
else:
|
|
4685
|
+
problems.append((args.out, 0, "facts file", "absent",
|
|
4686
|
+
"run: qa_ledger.py facts"))
|
|
4687
|
+
# 2) every recognizable claim in the given files must equal the derived fact
|
|
4688
|
+
for path in args.check:
|
|
4689
|
+
try:
|
|
4690
|
+
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
4691
|
+
lines = fh.read().splitlines()
|
|
4692
|
+
except OSError as exc:
|
|
4693
|
+
problems.append((path, 0, "file", "unreadable: %s" % exc, ""))
|
|
4694
|
+
continue
|
|
4695
|
+
for n, line in enumerate(lines, 1):
|
|
4696
|
+
# an HTML comment is not a published claim -- the first live run flagged a
|
|
4697
|
+
# section marker (a comment reading "2 Skills") as a drifted count
|
|
4698
|
+
line = re.sub(r"<!--.*?-->", "", line)
|
|
4699
|
+
low = line.lower()
|
|
4700
|
+
for key, pat, ctx in _CLAIM_PATTERNS:
|
|
4701
|
+
if ctx and ctx not in low:
|
|
4702
|
+
continue
|
|
4703
|
+
for m in re.finditer(pat, line, re.I):
|
|
4704
|
+
claimed = m.group(1)
|
|
4705
|
+
actual = _fact_value(facts, key)
|
|
4706
|
+
if claimed != actual:
|
|
4707
|
+
problems.append((path, n, key, claimed, actual))
|
|
4708
|
+
if problems:
|
|
4709
|
+
print("FACTUAL DRIFT: %d claim(s) disagree with the derived facts"
|
|
4710
|
+
% len(problems))
|
|
4711
|
+
for path, n, key, claimed, actual in problems:
|
|
4712
|
+
loc = "%s:%d" % (path, n) if n else path
|
|
4713
|
+
print(" !! %s: %s claims %r, the artifact says %r"
|
|
4714
|
+
% (loc, key, claimed, actual))
|
|
4715
|
+
sys.exit(1)
|
|
4716
|
+
print("FACTS: %d file(s) checked, every claim matches the derived facts"
|
|
4717
|
+
% len(args.check))
|
|
4718
|
+
sys.exit(0)
|
|
4719
|
+
body = json.dumps(facts, indent=2, ensure_ascii=False, sort_keys=True) + "\n"
|
|
4720
|
+
with open(args.out, "w", encoding="utf-8", newline="\n") as fh:
|
|
4721
|
+
fh.write(body)
|
|
4722
|
+
print("FACTS -> %s: version %s · %d subcommands · %d skills"
|
|
4723
|
+
% (args.out, facts["version"], facts["subcommands"]["count"],
|
|
4724
|
+
facts["skills"]["count"]))
|
|
4725
|
+
|
|
4726
|
+
|
|
4727
|
+
|
|
3865
4728
|
def cmd_escalate(args):
|
|
3866
4729
|
ledger = _load(args.ledger)
|
|
3867
4730
|
_repo_node(ledger, args.repo)
|
|
@@ -4913,6 +5776,20 @@ def cmd_dashboard(args):
|
|
|
4913
5776
|
for r in {e.get("repo") for e in ledger[CLEAN_ROOM_KEY]}}
|
|
4914
5777
|
if ledger.get("roundtrip"):
|
|
4915
5778
|
out["roundtrip"] = ledger["roundtrip"]
|
|
5779
|
+
# AC-CU-04: undefined verdicts stay OPEN and visible in the readouts -- derived live
|
|
5780
|
+
# from delta + curation records per repo (conditional key, the roundtrip pattern)
|
|
5781
|
+
_dl_all = {}
|
|
5782
|
+
for _rn in ledger["repos"]:
|
|
5783
|
+
try:
|
|
5784
|
+
_dls = _delta_state(ledger, _rn, _scope_path(ledger, _rn))
|
|
5785
|
+
except SystemExit:
|
|
5786
|
+
_dls = None
|
|
5787
|
+
if _dls is not None:
|
|
5788
|
+
_dl_all[_rn] = _dls
|
|
5789
|
+
if _dl_all:
|
|
5790
|
+
out["candidate_delta"] = _dl_all
|
|
5791
|
+
if ledger.get("fidelity"):
|
|
5792
|
+
out["fidelity"] = ledger["fidelity"]
|
|
4916
5793
|
if getattr(args, "json", False):
|
|
4917
5794
|
print(json.dumps(out, indent=2, ensure_ascii=False))
|
|
4918
5795
|
return
|
|
@@ -7696,6 +8573,13 @@ def build_parser():
|
|
|
7696
8573
|
pcu.add_argument("--json", action="store_true")
|
|
7697
8574
|
pcu.set_defaults(func=cmd_curation_check)
|
|
7698
8575
|
|
|
8576
|
+
pfa = sub.add_parser("facts",
|
|
8577
|
+
help="SYSTEM-FACTS: derive repo facts from the artifacts, or --check published claims against them (ADR-012)")
|
|
8578
|
+
pfa.add_argument("--out", default="SYSTEM-FACTS.json")
|
|
8579
|
+
pfa.add_argument("--check", nargs="*", default=None,
|
|
8580
|
+
help="files whose claims must match the derived facts; exit 1 on drift")
|
|
8581
|
+
pfa.set_defaults(func=cmd_facts)
|
|
8582
|
+
|
|
7699
8583
|
prt = sub.add_parser("roundtrip",
|
|
7700
8584
|
help="advisory: which promoted candidates are traceable in code via uscha-spec ids (ADR-009 slice 2)")
|
|
7701
8585
|
prt.add_argument("--ledger", default="QA-LEDGER.json")
|
|
@@ -7703,6 +8587,54 @@ def build_parser():
|
|
|
7703
8587
|
prt.add_argument("--json", action="store_true")
|
|
7704
8588
|
prt.set_defaults(func=cmd_roundtrip)
|
|
7705
8589
|
|
|
8590
|
+
pdd = sub.add_parser(
|
|
8591
|
+
"discover",
|
|
8592
|
+
help="emit discovery/CANDIDATE-DELTA.json: typed observations with content-addressed "
|
|
8593
|
+
"OBS ids -- measured/static/narrated, strictly classified (ADR-013)")
|
|
8594
|
+
pdd.add_argument("--ledger", default="QA-LEDGER.json")
|
|
8595
|
+
pdd.add_argument("--repo", required=True)
|
|
8596
|
+
pdd.add_argument("--narrated", default=None,
|
|
8597
|
+
help="JSON list of skill-supplied observations {type, statement, files}; "
|
|
8598
|
+
"the engine classifies them narrated -- it never calls an LLM")
|
|
8599
|
+
pdd.add_argument("--acceptance", default=None,
|
|
8600
|
+
help="acceptance file for canonical_match (default: config "
|
|
8601
|
+
"defaults.acceptance_file, else ACCEPTANCE.md)")
|
|
8602
|
+
pdd.add_argument("--json", action="store_true")
|
|
8603
|
+
pdd.set_defaults(func=cmd_discover)
|
|
8604
|
+
|
|
8605
|
+
pcv = sub.add_parser(
|
|
8606
|
+
"curate",
|
|
8607
|
+
help="record ONE human verdict (preserve|fix|undefined) for ONE observation as an "
|
|
8608
|
+
"append-only ledger object; no batch path exists (ADR-013)")
|
|
8609
|
+
pcv.add_argument("--ledger", default="QA-LEDGER.json")
|
|
8610
|
+
pcv.add_argument("--repo", required=True)
|
|
8611
|
+
pcv.add_argument("--obs", required=True, help="a single OBS id from the delta")
|
|
8612
|
+
pcv.add_argument("--verdict", required=True, choices=("preserve", "fix", "undefined"))
|
|
8613
|
+
pcv.add_argument("--note", default=None)
|
|
8614
|
+
pcv.add_argument("--human", default=None,
|
|
8615
|
+
help="who judged (default: the OS user)")
|
|
8616
|
+
pcv.set_defaults(func=cmd_curate)
|
|
8617
|
+
|
|
8618
|
+
ppr = sub.add_parser(
|
|
8619
|
+
"promote",
|
|
8620
|
+
help="move preserve-verdict observations into discovery/CANONICAL.json with "
|
|
8621
|
+
"derived_from lineage; refuses over ANY uncurated OBS (INV-CURATION-01)")
|
|
8622
|
+
ppr.add_argument("--ledger", default="QA-LEDGER.json")
|
|
8623
|
+
ppr.add_argument("--repo", required=True)
|
|
8624
|
+
ppr.add_argument("--json", action="store_true")
|
|
8625
|
+
ppr.set_defaults(func=cmd_promote)
|
|
8626
|
+
|
|
8627
|
+
pfv = sub.add_parser(
|
|
8628
|
+
"fidelity",
|
|
8629
|
+
help="the fidelity vector: 5 measured dimensions + advisory quarantine; an advisory "
|
|
8630
|
+
"dimension can NEVER gate (ADR-014, INV-ADVISORY-01)")
|
|
8631
|
+
pfv.add_argument("--ledger", default="QA-LEDGER.json")
|
|
8632
|
+
pfv.add_argument("--repo", required=True)
|
|
8633
|
+
pfv.add_argument("--config", default="uscha.config.json",
|
|
8634
|
+
help="checked for defaults.fidelity.gate -- advisory there is a refusal")
|
|
8635
|
+
pfv.add_argument("--json", action="store_true")
|
|
8636
|
+
pfv.set_defaults(func=cmd_fidelity)
|
|
8637
|
+
|
|
7706
8638
|
pcr = sub.add_parser("cleanroom",
|
|
7707
8639
|
help="run a command against a CLEAN checkout of one commit in a throwaway worktree (ADR-008)")
|
|
7708
8640
|
pcr.add_argument("--ledger", default="QA-LEDGER.json")
|