bmad-module-ultracode-goal 2.0.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +5 -5
  3. package/docs/404.md +1 -1
  4. package/docs/_internal/STABILITY.md +13 -9
  5. package/docs/architecture.md +17 -12
  6. package/docs/comparison.md +19 -11
  7. package/docs/gate-model.md +24 -0
  8. package/docs/getting-started.md +1 -1
  9. package/docs/how-it-works.md +3 -3
  10. package/docs/index.md +0 -1
  11. package/docs/operating-tips.md +4 -2
  12. package/docs/troubleshooting.md +9 -5
  13. package/docs/why-ultracode-goal.md +1 -1
  14. package/package.json +5 -4
  15. package/skills/ucg-formalize/SKILL.md +136 -39
  16. package/skills/ucg-resolve/SKILL.md +86 -58
  17. package/skills/ucg-status/SKILL.md +64 -30
  18. package/skills/ultracode-goal/SKILL.md +9 -11
  19. package/skills/ultracode-goal/assets/module-help.csv +1 -1
  20. package/skills/ultracode-goal/assets/module.yaml +8 -3
  21. package/skills/ultracode-goal/assets/ucg-awareness/bmad-testarch-test-design.toml +8 -0
  22. package/skills/ultracode-goal/customize.toml +12 -7
  23. package/skills/ultracode-goal/references/define-done.md +3 -3
  24. package/skills/ultracode-goal/references/execute.md +40 -53
  25. package/skills/ultracode-goal/references/finalize.md +12 -12
  26. package/skills/ultracode-goal/references/gate.md +42 -11
  27. package/skills/ultracode-goal/references/health-check.md +7 -4
  28. package/skills/ultracode-goal/references/ingest-and-scope.md +4 -4
  29. package/skills/ultracode-goal/references/preflight.md +10 -9
  30. package/skills/ultracode-goal/scripts/gate_eval.py +684 -39
  31. package/skills/ultracode-goal/scripts/gate_trail.py +131 -32
  32. package/skills/ultracode-goal/scripts/health_check_fp.py +98 -3
  33. package/skills/ultracode-goal/scripts/hooks/guard_pretooluse.py +56 -14
  34. package/skills/ultracode-goal/scripts/mem_observation.py +48 -10
  35. package/skills/ultracode-goal/scripts/mutate_check.py +353 -0
  36. package/skills/ultracode-goal/scripts/preflight_check.py +397 -3
  37. package/skills/ultracode-goal/scripts/status_render.py +20 -2
  38. package/skills/ultracode-goal/scripts/story_sizing.py +193 -0
  39. package/tools/build-docs.js +24 -1
  40. package/tools/validate-docs-links.js +169 -8
  41. package/website/astro.config.mjs +0 -1
  42. package/docs/parallel-mode.md +0 -68
  43. package/skills/ultracode-goal/assets/execute-epic.workflow.js +0 -249
@@ -0,0 +1,353 @@
1
+ #!/usr/bin/env python3
2
+ """Drive a table of mutations against a green baseline, and prove each one red.
3
+
4
+ The module's anti-vacuity discipline says a guard is non-vacuous only under
5
+ mutation. Every run used to hand-roll the harness for that into scratch space
6
+ (and delete it at story end, so the next story hand-rolled it again). This is
7
+ the tracked form: a table in, a ledger out, with the properties the hand-rolled
8
+ versions kept losing built in as refusals rather than reminders.
9
+
10
+ Table (JSON array; every key a string, all five required, ``note`` optional):
11
+
12
+ [{"file": "src/guard.py",
13
+ "anchor": "if depth > MAX_DEPTH:",
14
+ "replacement": "if False:",
15
+ "command": "npx turbo run test --force --filter=guard",
16
+ "expect_red": "test_depth_guard_fires",
17
+ "note": "AC-3 twin"}]
18
+
19
+ Per row, in order: assert the anchor occurs EXACTLY once (0 occurrences is
20
+ reported distinctly as STALE - the formatter-reflow signature - and more than
21
+ one as NOT UNIQUE, because a duplicated anchor mutates a different site than it
22
+ reports); back the file up; apply; run the command; classify; restore from the
23
+ backup and verify the restore BYTE-IDENTICAL. Before any row: run every
24
+ distinct command once and refuse the whole table unless all are green, because
25
+ a mutation over a red baseline proves nothing.
26
+
27
+ Classification, per row:
28
+
29
+ - GREEN the command stayed green: the mutation SURVIVED. This is
30
+ the output that matters - a guard that cannot fail - so
31
+ the ledger leads with these.
32
+ - KILLED red, the expected case named in the output, and exactly
33
+ one failure counted.
34
+ - KILLED+others red and named, but more than one case failed (or the
35
+ count could not be parsed): the kill stands, the extra
36
+ reds are surfaced rather than silently absorbed.
37
+ - MISATTRIBUTED red, but the expected case never appears beside a
38
+ failure token (or the output's own count says 0 failed):
39
+ something failed, and it was not the observer this row
40
+ claims to test. The name must sit on a line that speaks
41
+ of failure, because verbose reporters echo PASSING names.
42
+ - STALE / NOT UNIQUE / UNDECODABLE the anchor (or the file) did not hold;
43
+ nothing was mutated for the row.
44
+
45
+ Exit code: 0 when every row KILLED (with or without +others); 1 when any row
46
+ is GREEN, MISATTRIBUTED, STALE, NOT UNIQUE or UNDECODABLE (each is a finding
47
+ about the suite or the table); 2 on a refusal - invalid table, red baseline, or a
48
+ restore that did not verify byte-identical (the one failure mode that leaves
49
+ the tree dirty, so it aborts the run rather than annotating it).
50
+
51
+ The task-runner cache rule binds the table's ``command`` verbatim: an
52
+ unchanged task hash replays the pre-mutation green, so every mutation
53
+ "survives" with zero failing cases. Defeat the cache in the command itself
54
+ (``--force`` and friends); this runner cannot do it for you.
55
+ """
56
+
57
+ from __future__ import annotations
58
+
59
+ import argparse
60
+ import json
61
+ import re
62
+ import signal
63
+ import subprocess
64
+ import sys
65
+ from pathlib import Path
66
+
67
+ REQUIRED_KEYS = ("file", "anchor", "replacement", "command", "expect_red")
68
+
69
+
70
+ def _refuse(message: str) -> "SystemExit":
71
+ """Exit 2, the refusal lane. `raise SystemExit(str)` would exit 1 and be
72
+ indistinguishable from a findings run; the docstring promises 2."""
73
+ print(f"REFUSED: {message}", file=sys.stderr)
74
+ return SystemExit(2)
75
+
76
+
77
+ def _terminated(signum, _frame):
78
+ raise SystemExit(128 + signum)
79
+
80
+
81
+ def _install_signal_unwind() -> None:
82
+ """Convert termination signals into SystemExit so `finally` restores run.
83
+
84
+ SIGTERM's default disposition kills the interpreter WITHOUT unwinding, so
85
+ the finally block that restores a mutated file never executes - and TERM
86
+ is the exact kill a harness timeout delivers mid-command. SIGKILL cannot
87
+ be caught: after any abnormal end, `git status` before trusting the tree.
88
+ """
89
+ for name in ("SIGTERM", "SIGHUP", "SIGBREAK"):
90
+ sig = getattr(signal, name, None)
91
+ if sig is None:
92
+ continue
93
+ try:
94
+ signal.signal(sig, _terminated)
95
+ except (ValueError, OSError):
96
+ continue
97
+
98
+ # Failure-count shapes across the common runners (pytest, vitest/jest, mocha,
99
+ # cargo, go). First match wins; no match means the count is unparseable and the
100
+ # kill is reported without the exactly-one claim.
101
+ _FAIL_COUNT_RES = (
102
+ re.compile(r"(\d+)\s+failed", re.IGNORECASE),
103
+ re.compile(r"failures[:=]\s*(\d+)", re.IGNORECASE),
104
+ re.compile(r"(\d+)\s+failing", re.IGNORECASE),
105
+ )
106
+
107
+
108
+ def _load_table(path: Path) -> list[dict]:
109
+ try:
110
+ raw = json.loads(path.read_text(encoding="utf-8"))
111
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
112
+ raise _refuse(f"table {path} unreadable or not JSON: {exc}")
113
+ if not isinstance(raw, list) or not raw:
114
+ raise _refuse(f"table {path} must be a non-empty JSON array of rows")
115
+ for index, row in enumerate(raw):
116
+ if not isinstance(row, dict):
117
+ raise _refuse(f"row {index} is not an object")
118
+ for key in REQUIRED_KEYS:
119
+ if not isinstance(row.get(key), str) or not row[key]:
120
+ raise _refuse(f"row {index} is missing required string key {key!r}")
121
+ if row["anchor"] == row["replacement"]:
122
+ raise _refuse(
123
+ f"row {index} anchor and replacement are identical - "
124
+ "that mutation mutates nothing"
125
+ )
126
+ return raw
127
+
128
+
129
+ def _run(command: str, cwd: Path) -> tuple[int, str]:
130
+ proc = subprocess.run(
131
+ command,
132
+ shell=True,
133
+ cwd=cwd,
134
+ capture_output=True,
135
+ # utf-8 + replace, never the strict locale codec: a failing assertion
136
+ # that dumps raw bytes must classify as a red, not crash the run - and
137
+ # on Windows the locale default (cp1252) also mojibakes the very names
138
+ # attribution matches on.
139
+ encoding="utf-8",
140
+ errors="replace",
141
+ check=False,
142
+ )
143
+ return proc.returncode, (proc.stdout or "") + (proc.stderr or "")
144
+
145
+
146
+ def _fail_count(output: str) -> int | None:
147
+ for pattern in _FAIL_COUNT_RES:
148
+ match = pattern.search(output)
149
+ if match:
150
+ return int(match.group(1))
151
+ return None
152
+
153
+
154
+ _FAIL_TOKEN_RE = re.compile(r"fail|error|not ok|assert|✗|✘", re.IGNORECASE)
155
+
156
+
157
+ def _named_in_failure_context(output: str, expect_red: str) -> bool:
158
+ """True when the expected case is named on a line that speaks of failure.
159
+
160
+ A whole-output substring match is defeated by any verbose reporter that
161
+ echoes PASSING names (`pytest -v` prints every test), which made a wrongly
162
+ attributed kill read as a clean one. So the name only counts on a line
163
+ that also carries a failure token. Heuristic, and documented as one: a
164
+ runner whose failure lines carry none of the tokens reads MISATTRIBUTED,
165
+ which fails toward the honest side - a human looks.
166
+ """
167
+ return any(
168
+ expect_red in line and _FAIL_TOKEN_RE.search(line)
169
+ for line in output.splitlines()
170
+ )
171
+
172
+
173
+ def _classify_red(output: str, expect_red: str) -> tuple[str, str, bool]:
174
+ """(status, detail, is_finding) for a red run."""
175
+ if not _named_in_failure_context(output, expect_red):
176
+ return (
177
+ "MISATTRIBUTED",
178
+ f"red, but {expect_red!r} never appears beside a failure token - "
179
+ "whatever failed, it was not the observer this row claims",
180
+ True,
181
+ )
182
+ count = _fail_count(output)
183
+ if count == 1:
184
+ return "KILLED", f"exactly {expect_red!r} went red", False
185
+ if count == 0:
186
+ # A red exit whose own summary counts zero failures is a collection or
187
+ # harness error wearing a red exit code, not the observer firing.
188
+ return (
189
+ "MISATTRIBUTED",
190
+ "red, but the output's own count says 0 failed - a collection or "
191
+ "harness error, not the observer",
192
+ True,
193
+ )
194
+ if count is None:
195
+ return "KILLED+others", f"{expect_red!r} went red; failure count unparseable", False
196
+ return "KILLED+others", f"{expect_red!r} went red among {count} failures", False
197
+
198
+
199
+ def run_table(table: list[dict], project_root: Path) -> tuple[list[dict], int]:
200
+ """Execute the table; return (ledger rows, exit code)."""
201
+ # Baseline first, each distinct command once, in first-appearance order.
202
+ seen: list[str] = []
203
+ for row in table:
204
+ if row["command"] not in seen:
205
+ seen.append(row["command"])
206
+ for command in seen:
207
+ code, output = _run(command, project_root)
208
+ if code != 0:
209
+ raise _refuse(
210
+ "baseline is not green for command "
211
+ f"{command!r} (exit {code}) - a mutation over a red baseline "
212
+ f"proves nothing. Last lines:\n{output[-2000:]}"
213
+ )
214
+
215
+ rows: list[dict] = []
216
+ findings = False
217
+ for index, row in enumerate(table):
218
+ target = project_root / row["file"]
219
+ entry = {
220
+ "index": index,
221
+ "file": row["file"],
222
+ "expect_red": row["expect_red"],
223
+ "note": row.get("note", ""),
224
+ "restore": "n/a",
225
+ }
226
+ try:
227
+ original = target.read_bytes()
228
+ except OSError as exc:
229
+ entry.update(status="STALE", detail=f"file unreadable: {exc}")
230
+ rows.append(entry)
231
+ findings = True
232
+ continue
233
+ try:
234
+ text = original.decode("utf-8")
235
+ except UnicodeDecodeError as exc:
236
+ entry.update(status="UNDECODABLE", detail=f"file is not UTF-8: {exc}")
237
+ rows.append(entry)
238
+ findings = True
239
+ continue
240
+ occurrences = text.count(row["anchor"])
241
+ if occurrences != 1:
242
+ entry.update(
243
+ status="STALE" if occurrences == 0 else "NOT UNIQUE",
244
+ detail=(
245
+ f"anchor occurs {occurrences} times - "
246
+ + (
247
+ "0 is the formatter-reflow signature; re-derive the anchor"
248
+ if occurrences == 0
249
+ else "a duplicated anchor mutates a different site than it reports"
250
+ )
251
+ ),
252
+ )
253
+ rows.append(entry)
254
+ findings = True
255
+ continue
256
+
257
+ target.write_bytes(text.replace(row["anchor"], row["replacement"], 1).encode("utf-8"))
258
+ try:
259
+ code, output = _run(row["command"], project_root)
260
+ finally:
261
+ # Restore is unconditional and verified, never assumed: an earlier
262
+ # hand-rolled run in the field lost a file to its own cleanup. A
263
+ # restore WRITE that fails is the tree-dirty case and must land in
264
+ # the exit-2 refusal lane, not propagate as a traceback into the
265
+ # findings exit code.
266
+ try:
267
+ target.write_bytes(original)
268
+ except OSError as exc:
269
+ raise _refuse(
270
+ f"restore of {row['file']} FAILED ({exc}) - the tree still "
271
+ "carries the mutation; fix the tree before trusting "
272
+ "anything else this run printed"
273
+ ) from exc
274
+ if target.read_bytes() != original:
275
+ raise _refuse(
276
+ f"restore of {row['file']} did not verify byte-identical - "
277
+ "the tree is dirty; fix that before trusting anything else this run printed"
278
+ )
279
+ entry["restore"] = "verified"
280
+
281
+ if code == 0:
282
+ entry.update(
283
+ status="GREEN",
284
+ detail="the mutation SURVIVED: nothing in the suite can tell - this is a finding",
285
+ )
286
+ findings = True
287
+ else:
288
+ status, detail, is_finding = _classify_red(output, row["expect_red"])
289
+ entry.update(status=status, detail=detail)
290
+ findings = findings or is_finding
291
+ rows.append(entry)
292
+
293
+ return rows, 1 if findings else 0
294
+
295
+
296
+ _LEDGER_ORDER = {"GREEN": 0, "MISATTRIBUTED": 1, "STALE": 2, "NOT UNIQUE": 2, "UNDECODABLE": 2}
297
+
298
+
299
+ def render_ledger(rows: list[dict]) -> str:
300
+ """The generated markdown ledger, GREEN survivors first."""
301
+ ordered = sorted(rows, key=lambda r: (_LEDGER_ORDER.get(r["status"], 9), r["index"]))
302
+ greens = sum(1 for r in rows if r["status"] == "GREEN")
303
+ killed = sum(1 for r in rows if r["status"].startswith("KILLED"))
304
+ lines = [
305
+ "# Mutation ledger",
306
+ "",
307
+ f"{len(rows)} rows: {greens} GREEN (survived), {killed} killed, "
308
+ f"{len(rows) - greens - killed} not applied.",
309
+ "",
310
+ "| # | status | file | expected red | detail | restore | note |",
311
+ "|---|--------|------|--------------|--------|---------|------|",
312
+ ]
313
+ for r in ordered:
314
+ lines.append(
315
+ f"| {r['index']} | {r['status']} | `{r['file']}` | `{r['expect_red']}` "
316
+ f"| {r['detail']} | {r['restore']} | {r['note']} |"
317
+ )
318
+ lines.append("")
319
+ return "\n".join(lines)
320
+
321
+
322
+ def main(argv: list[str] | None = None) -> int:
323
+ parser = argparse.ArgumentParser(
324
+ description="Run a mutation table against a green baseline; ledger out, greens first."
325
+ )
326
+ parser.add_argument("--table", required=True, help="PATH to the JSON mutation table.")
327
+ parser.add_argument(
328
+ "--project-root",
329
+ default=".",
330
+ help="Directory row file paths resolve against and commands run in (default: cwd).",
331
+ )
332
+ parser.add_argument(
333
+ "--output",
334
+ help="Also write the markdown ledger to this PATH (it always prints to stdout).",
335
+ )
336
+ args = parser.parse_args(argv)
337
+
338
+ _install_signal_unwind()
339
+ table = _load_table(Path(args.table))
340
+ project_root = Path(args.project_root).resolve()
341
+ if not project_root.is_dir():
342
+ raise _refuse(f"project root {project_root} is not a directory")
343
+
344
+ rows, exit_code = run_table(table, project_root)
345
+ ledger = render_ledger(rows)
346
+ print(ledger)
347
+ if args.output:
348
+ Path(args.output).write_text(ledger, encoding="utf-8")
349
+ return exit_code
350
+
351
+
352
+ if __name__ == "__main__":
353
+ sys.exit(main())