bmad-module-ultracode-goal 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +5 -5
- package/docs/404.md +1 -1
- package/docs/_internal/STABILITY.md +13 -9
- package/docs/architecture.md +17 -12
- package/docs/comparison.md +19 -11
- package/docs/gate-model.md +24 -0
- package/docs/getting-started.md +1 -1
- package/docs/how-it-works.md +3 -3
- package/docs/index.md +0 -1
- package/docs/operating-tips.md +4 -2
- package/docs/troubleshooting.md +9 -5
- package/docs/why-ultracode-goal.md +1 -1
- package/package.json +5 -4
- package/skills/ucg-formalize/SKILL.md +136 -39
- package/skills/ucg-resolve/SKILL.md +86 -58
- package/skills/ucg-status/SKILL.md +64 -30
- package/skills/ultracode-goal/SKILL.md +9 -11
- package/skills/ultracode-goal/assets/module-help.csv +1 -1
- package/skills/ultracode-goal/assets/module.yaml +8 -3
- package/skills/ultracode-goal/assets/ucg-awareness/bmad-testarch-test-design.toml +8 -0
- package/skills/ultracode-goal/customize.toml +12 -7
- package/skills/ultracode-goal/references/define-done.md +3 -3
- package/skills/ultracode-goal/references/execute.md +40 -53
- package/skills/ultracode-goal/references/finalize.md +12 -12
- package/skills/ultracode-goal/references/gate.md +42 -11
- package/skills/ultracode-goal/references/health-check.md +7 -4
- package/skills/ultracode-goal/references/ingest-and-scope.md +4 -4
- package/skills/ultracode-goal/references/preflight.md +10 -9
- package/skills/ultracode-goal/scripts/gate_eval.py +684 -39
- package/skills/ultracode-goal/scripts/gate_trail.py +131 -32
- package/skills/ultracode-goal/scripts/health_check_fp.py +98 -3
- package/skills/ultracode-goal/scripts/hooks/guard_pretooluse.py +56 -14
- package/skills/ultracode-goal/scripts/mem_observation.py +48 -10
- package/skills/ultracode-goal/scripts/mutate_check.py +353 -0
- package/skills/ultracode-goal/scripts/preflight_check.py +397 -3
- package/skills/ultracode-goal/scripts/status_render.py +20 -2
- package/skills/ultracode-goal/scripts/story_sizing.py +193 -0
- package/tools/build-docs.js +24 -1
- package/tools/validate-docs-links.js +169 -8
- package/website/astro.config.mjs +0 -1
- package/docs/parallel-mode.md +0 -68
- package/skills/ultracode-goal/assets/execute-epic.workflow.js +0 -249
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Drive a table of mutations against a green baseline, and prove each one red.
|
|
3
|
+
|
|
4
|
+
The module's anti-vacuity discipline says a guard is non-vacuous only under
|
|
5
|
+
mutation. Every run used to hand-roll the harness for that into scratch space
|
|
6
|
+
(and delete it at story end, so the next story hand-rolled it again). This is
|
|
7
|
+
the tracked form: a table in, a ledger out, with the properties the hand-rolled
|
|
8
|
+
versions kept losing built in as refusals rather than reminders.
|
|
9
|
+
|
|
10
|
+
Table (JSON array; every key a string, all five required, ``note`` optional):
|
|
11
|
+
|
|
12
|
+
[{"file": "src/guard.py",
|
|
13
|
+
"anchor": "if depth > MAX_DEPTH:",
|
|
14
|
+
"replacement": "if False:",
|
|
15
|
+
"command": "npx turbo run test --force --filter=guard",
|
|
16
|
+
"expect_red": "test_depth_guard_fires",
|
|
17
|
+
"note": "AC-3 twin"}]
|
|
18
|
+
|
|
19
|
+
Per row, in order: assert the anchor occurs EXACTLY once (0 occurrences is
|
|
20
|
+
reported distinctly as STALE - the formatter-reflow signature - and more than
|
|
21
|
+
one as NOT UNIQUE, because a duplicated anchor mutates a different site than it
|
|
22
|
+
reports); back the file up; apply; run the command; classify; restore from the
|
|
23
|
+
backup and verify the restore BYTE-IDENTICAL. Before any row: run every
|
|
24
|
+
distinct command once and refuse the whole table unless all are green, because
|
|
25
|
+
a mutation over a red baseline proves nothing.
|
|
26
|
+
|
|
27
|
+
Classification, per row:
|
|
28
|
+
|
|
29
|
+
- GREEN the command stayed green: the mutation SURVIVED. This is
|
|
30
|
+
the output that matters - a guard that cannot fail - so
|
|
31
|
+
the ledger leads with these.
|
|
32
|
+
- KILLED red, the expected case named in the output, and exactly
|
|
33
|
+
one failure counted.
|
|
34
|
+
- KILLED+others red and named, but more than one case failed (or the
|
|
35
|
+
count could not be parsed): the kill stands, the extra
|
|
36
|
+
reds are surfaced rather than silently absorbed.
|
|
37
|
+
- MISATTRIBUTED red, but the expected case never appears beside a
|
|
38
|
+
failure token (or the output's own count says 0 failed):
|
|
39
|
+
something failed, and it was not the observer this row
|
|
40
|
+
claims to test. The name must sit on a line that speaks
|
|
41
|
+
of failure, because verbose reporters echo PASSING names.
|
|
42
|
+
- STALE / NOT UNIQUE / UNDECODABLE the anchor (or the file) did not hold;
|
|
43
|
+
nothing was mutated for the row.
|
|
44
|
+
|
|
45
|
+
Exit code: 0 when every row KILLED (with or without +others); 1 when any row
|
|
46
|
+
is GREEN, MISATTRIBUTED, STALE, NOT UNIQUE or UNDECODABLE (each is a finding
|
|
47
|
+
about the suite or the table); 2 on a refusal - invalid table, red baseline, or a
|
|
48
|
+
restore that did not verify byte-identical (the one failure mode that leaves
|
|
49
|
+
the tree dirty, so it aborts the run rather than annotating it).
|
|
50
|
+
|
|
51
|
+
The task-runner cache rule binds the table's ``command`` verbatim: an
|
|
52
|
+
unchanged task hash replays the pre-mutation green, so every mutation
|
|
53
|
+
"survives" with zero failing cases. Defeat the cache in the command itself
|
|
54
|
+
(``--force`` and friends); this runner cannot do it for you.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
from __future__ import annotations
|
|
58
|
+
|
|
59
|
+
import argparse
|
|
60
|
+
import json
|
|
61
|
+
import re
|
|
62
|
+
import signal
|
|
63
|
+
import subprocess
|
|
64
|
+
import sys
|
|
65
|
+
from pathlib import Path
|
|
66
|
+
|
|
67
|
+
REQUIRED_KEYS = ("file", "anchor", "replacement", "command", "expect_red")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _refuse(message: str) -> "SystemExit":
|
|
71
|
+
"""Exit 2, the refusal lane. `raise SystemExit(str)` would exit 1 and be
|
|
72
|
+
indistinguishable from a findings run; the docstring promises 2."""
|
|
73
|
+
print(f"REFUSED: {message}", file=sys.stderr)
|
|
74
|
+
return SystemExit(2)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _terminated(signum, _frame):
|
|
78
|
+
raise SystemExit(128 + signum)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _install_signal_unwind() -> None:
|
|
82
|
+
"""Convert termination signals into SystemExit so `finally` restores run.
|
|
83
|
+
|
|
84
|
+
SIGTERM's default disposition kills the interpreter WITHOUT unwinding, so
|
|
85
|
+
the finally block that restores a mutated file never executes - and TERM
|
|
86
|
+
is the exact kill a harness timeout delivers mid-command. SIGKILL cannot
|
|
87
|
+
be caught: after any abnormal end, `git status` before trusting the tree.
|
|
88
|
+
"""
|
|
89
|
+
for name in ("SIGTERM", "SIGHUP", "SIGBREAK"):
|
|
90
|
+
sig = getattr(signal, name, None)
|
|
91
|
+
if sig is None:
|
|
92
|
+
continue
|
|
93
|
+
try:
|
|
94
|
+
signal.signal(sig, _terminated)
|
|
95
|
+
except (ValueError, OSError):
|
|
96
|
+
continue
|
|
97
|
+
|
|
98
|
+
# Failure-count shapes across the common runners (pytest, vitest/jest, mocha,
|
|
99
|
+
# cargo, go). First match wins; no match means the count is unparseable and the
|
|
100
|
+
# kill is reported without the exactly-one claim.
|
|
101
|
+
_FAIL_COUNT_RES = (
|
|
102
|
+
re.compile(r"(\d+)\s+failed", re.IGNORECASE),
|
|
103
|
+
re.compile(r"failures[:=]\s*(\d+)", re.IGNORECASE),
|
|
104
|
+
re.compile(r"(\d+)\s+failing", re.IGNORECASE),
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _load_table(path: Path) -> list[dict]:
|
|
109
|
+
try:
|
|
110
|
+
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
111
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
112
|
+
raise _refuse(f"table {path} unreadable or not JSON: {exc}")
|
|
113
|
+
if not isinstance(raw, list) or not raw:
|
|
114
|
+
raise _refuse(f"table {path} must be a non-empty JSON array of rows")
|
|
115
|
+
for index, row in enumerate(raw):
|
|
116
|
+
if not isinstance(row, dict):
|
|
117
|
+
raise _refuse(f"row {index} is not an object")
|
|
118
|
+
for key in REQUIRED_KEYS:
|
|
119
|
+
if not isinstance(row.get(key), str) or not row[key]:
|
|
120
|
+
raise _refuse(f"row {index} is missing required string key {key!r}")
|
|
121
|
+
if row["anchor"] == row["replacement"]:
|
|
122
|
+
raise _refuse(
|
|
123
|
+
f"row {index} anchor and replacement are identical - "
|
|
124
|
+
"that mutation mutates nothing"
|
|
125
|
+
)
|
|
126
|
+
return raw
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _run(command: str, cwd: Path) -> tuple[int, str]:
|
|
130
|
+
proc = subprocess.run(
|
|
131
|
+
command,
|
|
132
|
+
shell=True,
|
|
133
|
+
cwd=cwd,
|
|
134
|
+
capture_output=True,
|
|
135
|
+
# utf-8 + replace, never the strict locale codec: a failing assertion
|
|
136
|
+
# that dumps raw bytes must classify as a red, not crash the run - and
|
|
137
|
+
# on Windows the locale default (cp1252) also mojibakes the very names
|
|
138
|
+
# attribution matches on.
|
|
139
|
+
encoding="utf-8",
|
|
140
|
+
errors="replace",
|
|
141
|
+
check=False,
|
|
142
|
+
)
|
|
143
|
+
return proc.returncode, (proc.stdout or "") + (proc.stderr or "")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _fail_count(output: str) -> int | None:
|
|
147
|
+
for pattern in _FAIL_COUNT_RES:
|
|
148
|
+
match = pattern.search(output)
|
|
149
|
+
if match:
|
|
150
|
+
return int(match.group(1))
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
_FAIL_TOKEN_RE = re.compile(r"fail|error|not ok|assert|✗|✘", re.IGNORECASE)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _named_in_failure_context(output: str, expect_red: str) -> bool:
|
|
158
|
+
"""True when the expected case is named on a line that speaks of failure.
|
|
159
|
+
|
|
160
|
+
A whole-output substring match is defeated by any verbose reporter that
|
|
161
|
+
echoes PASSING names (`pytest -v` prints every test), which made a wrongly
|
|
162
|
+
attributed kill read as a clean one. So the name only counts on a line
|
|
163
|
+
that also carries a failure token. Heuristic, and documented as one: a
|
|
164
|
+
runner whose failure lines carry none of the tokens reads MISATTRIBUTED,
|
|
165
|
+
which fails toward the honest side - a human looks.
|
|
166
|
+
"""
|
|
167
|
+
return any(
|
|
168
|
+
expect_red in line and _FAIL_TOKEN_RE.search(line)
|
|
169
|
+
for line in output.splitlines()
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _classify_red(output: str, expect_red: str) -> tuple[str, str, bool]:
|
|
174
|
+
"""(status, detail, is_finding) for a red run."""
|
|
175
|
+
if not _named_in_failure_context(output, expect_red):
|
|
176
|
+
return (
|
|
177
|
+
"MISATTRIBUTED",
|
|
178
|
+
f"red, but {expect_red!r} never appears beside a failure token - "
|
|
179
|
+
"whatever failed, it was not the observer this row claims",
|
|
180
|
+
True,
|
|
181
|
+
)
|
|
182
|
+
count = _fail_count(output)
|
|
183
|
+
if count == 1:
|
|
184
|
+
return "KILLED", f"exactly {expect_red!r} went red", False
|
|
185
|
+
if count == 0:
|
|
186
|
+
# A red exit whose own summary counts zero failures is a collection or
|
|
187
|
+
# harness error wearing a red exit code, not the observer firing.
|
|
188
|
+
return (
|
|
189
|
+
"MISATTRIBUTED",
|
|
190
|
+
"red, but the output's own count says 0 failed - a collection or "
|
|
191
|
+
"harness error, not the observer",
|
|
192
|
+
True,
|
|
193
|
+
)
|
|
194
|
+
if count is None:
|
|
195
|
+
return "KILLED+others", f"{expect_red!r} went red; failure count unparseable", False
|
|
196
|
+
return "KILLED+others", f"{expect_red!r} went red among {count} failures", False
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def run_table(table: list[dict], project_root: Path) -> tuple[list[dict], int]:
|
|
200
|
+
"""Execute the table; return (ledger rows, exit code)."""
|
|
201
|
+
# Baseline first, each distinct command once, in first-appearance order.
|
|
202
|
+
seen: list[str] = []
|
|
203
|
+
for row in table:
|
|
204
|
+
if row["command"] not in seen:
|
|
205
|
+
seen.append(row["command"])
|
|
206
|
+
for command in seen:
|
|
207
|
+
code, output = _run(command, project_root)
|
|
208
|
+
if code != 0:
|
|
209
|
+
raise _refuse(
|
|
210
|
+
"baseline is not green for command "
|
|
211
|
+
f"{command!r} (exit {code}) - a mutation over a red baseline "
|
|
212
|
+
f"proves nothing. Last lines:\n{output[-2000:]}"
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
rows: list[dict] = []
|
|
216
|
+
findings = False
|
|
217
|
+
for index, row in enumerate(table):
|
|
218
|
+
target = project_root / row["file"]
|
|
219
|
+
entry = {
|
|
220
|
+
"index": index,
|
|
221
|
+
"file": row["file"],
|
|
222
|
+
"expect_red": row["expect_red"],
|
|
223
|
+
"note": row.get("note", ""),
|
|
224
|
+
"restore": "n/a",
|
|
225
|
+
}
|
|
226
|
+
try:
|
|
227
|
+
original = target.read_bytes()
|
|
228
|
+
except OSError as exc:
|
|
229
|
+
entry.update(status="STALE", detail=f"file unreadable: {exc}")
|
|
230
|
+
rows.append(entry)
|
|
231
|
+
findings = True
|
|
232
|
+
continue
|
|
233
|
+
try:
|
|
234
|
+
text = original.decode("utf-8")
|
|
235
|
+
except UnicodeDecodeError as exc:
|
|
236
|
+
entry.update(status="UNDECODABLE", detail=f"file is not UTF-8: {exc}")
|
|
237
|
+
rows.append(entry)
|
|
238
|
+
findings = True
|
|
239
|
+
continue
|
|
240
|
+
occurrences = text.count(row["anchor"])
|
|
241
|
+
if occurrences != 1:
|
|
242
|
+
entry.update(
|
|
243
|
+
status="STALE" if occurrences == 0 else "NOT UNIQUE",
|
|
244
|
+
detail=(
|
|
245
|
+
f"anchor occurs {occurrences} times - "
|
|
246
|
+
+ (
|
|
247
|
+
"0 is the formatter-reflow signature; re-derive the anchor"
|
|
248
|
+
if occurrences == 0
|
|
249
|
+
else "a duplicated anchor mutates a different site than it reports"
|
|
250
|
+
)
|
|
251
|
+
),
|
|
252
|
+
)
|
|
253
|
+
rows.append(entry)
|
|
254
|
+
findings = True
|
|
255
|
+
continue
|
|
256
|
+
|
|
257
|
+
target.write_bytes(text.replace(row["anchor"], row["replacement"], 1).encode("utf-8"))
|
|
258
|
+
try:
|
|
259
|
+
code, output = _run(row["command"], project_root)
|
|
260
|
+
finally:
|
|
261
|
+
# Restore is unconditional and verified, never assumed: an earlier
|
|
262
|
+
# hand-rolled run in the field lost a file to its own cleanup. A
|
|
263
|
+
# restore WRITE that fails is the tree-dirty case and must land in
|
|
264
|
+
# the exit-2 refusal lane, not propagate as a traceback into the
|
|
265
|
+
# findings exit code.
|
|
266
|
+
try:
|
|
267
|
+
target.write_bytes(original)
|
|
268
|
+
except OSError as exc:
|
|
269
|
+
raise _refuse(
|
|
270
|
+
f"restore of {row['file']} FAILED ({exc}) - the tree still "
|
|
271
|
+
"carries the mutation; fix the tree before trusting "
|
|
272
|
+
"anything else this run printed"
|
|
273
|
+
) from exc
|
|
274
|
+
if target.read_bytes() != original:
|
|
275
|
+
raise _refuse(
|
|
276
|
+
f"restore of {row['file']} did not verify byte-identical - "
|
|
277
|
+
"the tree is dirty; fix that before trusting anything else this run printed"
|
|
278
|
+
)
|
|
279
|
+
entry["restore"] = "verified"
|
|
280
|
+
|
|
281
|
+
if code == 0:
|
|
282
|
+
entry.update(
|
|
283
|
+
status="GREEN",
|
|
284
|
+
detail="the mutation SURVIVED: nothing in the suite can tell - this is a finding",
|
|
285
|
+
)
|
|
286
|
+
findings = True
|
|
287
|
+
else:
|
|
288
|
+
status, detail, is_finding = _classify_red(output, row["expect_red"])
|
|
289
|
+
entry.update(status=status, detail=detail)
|
|
290
|
+
findings = findings or is_finding
|
|
291
|
+
rows.append(entry)
|
|
292
|
+
|
|
293
|
+
return rows, 1 if findings else 0
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
_LEDGER_ORDER = {"GREEN": 0, "MISATTRIBUTED": 1, "STALE": 2, "NOT UNIQUE": 2, "UNDECODABLE": 2}
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def render_ledger(rows: list[dict]) -> str:
|
|
300
|
+
"""The generated markdown ledger, GREEN survivors first."""
|
|
301
|
+
ordered = sorted(rows, key=lambda r: (_LEDGER_ORDER.get(r["status"], 9), r["index"]))
|
|
302
|
+
greens = sum(1 for r in rows if r["status"] == "GREEN")
|
|
303
|
+
killed = sum(1 for r in rows if r["status"].startswith("KILLED"))
|
|
304
|
+
lines = [
|
|
305
|
+
"# Mutation ledger",
|
|
306
|
+
"",
|
|
307
|
+
f"{len(rows)} rows: {greens} GREEN (survived), {killed} killed, "
|
|
308
|
+
f"{len(rows) - greens - killed} not applied.",
|
|
309
|
+
"",
|
|
310
|
+
"| # | status | file | expected red | detail | restore | note |",
|
|
311
|
+
"|---|--------|------|--------------|--------|---------|------|",
|
|
312
|
+
]
|
|
313
|
+
for r in ordered:
|
|
314
|
+
lines.append(
|
|
315
|
+
f"| {r['index']} | {r['status']} | `{r['file']}` | `{r['expect_red']}` "
|
|
316
|
+
f"| {r['detail']} | {r['restore']} | {r['note']} |"
|
|
317
|
+
)
|
|
318
|
+
lines.append("")
|
|
319
|
+
return "\n".join(lines)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def main(argv: list[str] | None = None) -> int:
|
|
323
|
+
parser = argparse.ArgumentParser(
|
|
324
|
+
description="Run a mutation table against a green baseline; ledger out, greens first."
|
|
325
|
+
)
|
|
326
|
+
parser.add_argument("--table", required=True, help="PATH to the JSON mutation table.")
|
|
327
|
+
parser.add_argument(
|
|
328
|
+
"--project-root",
|
|
329
|
+
default=".",
|
|
330
|
+
help="Directory row file paths resolve against and commands run in (default: cwd).",
|
|
331
|
+
)
|
|
332
|
+
parser.add_argument(
|
|
333
|
+
"--output",
|
|
334
|
+
help="Also write the markdown ledger to this PATH (it always prints to stdout).",
|
|
335
|
+
)
|
|
336
|
+
args = parser.parse_args(argv)
|
|
337
|
+
|
|
338
|
+
_install_signal_unwind()
|
|
339
|
+
table = _load_table(Path(args.table))
|
|
340
|
+
project_root = Path(args.project_root).resolve()
|
|
341
|
+
if not project_root.is_dir():
|
|
342
|
+
raise _refuse(f"project root {project_root} is not a directory")
|
|
343
|
+
|
|
344
|
+
rows, exit_code = run_table(table, project_root)
|
|
345
|
+
ledger = render_ledger(rows)
|
|
346
|
+
print(ledger)
|
|
347
|
+
if args.output:
|
|
348
|
+
Path(args.output).write_text(ledger, encoding="utf-8")
|
|
349
|
+
return exit_code
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
if __name__ == "__main__":
|
|
353
|
+
sys.exit(main())
|