bmad-module-ultracode-goal 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +5 -5
- package/docs/404.md +1 -1
- package/docs/_internal/STABILITY.md +13 -9
- package/docs/architecture.md +17 -12
- package/docs/comparison.md +19 -11
- package/docs/gate-model.md +24 -0
- package/docs/getting-started.md +1 -1
- package/docs/how-it-works.md +3 -3
- package/docs/index.md +0 -1
- package/docs/operating-tips.md +4 -2
- package/docs/troubleshooting.md +9 -5
- package/docs/why-ultracode-goal.md +1 -1
- package/package.json +5 -4
- package/skills/ucg-formalize/SKILL.md +136 -39
- package/skills/ucg-resolve/SKILL.md +86 -58
- package/skills/ucg-status/SKILL.md +64 -30
- package/skills/ultracode-goal/SKILL.md +9 -11
- package/skills/ultracode-goal/assets/module-help.csv +1 -1
- package/skills/ultracode-goal/assets/module.yaml +8 -3
- package/skills/ultracode-goal/assets/ucg-awareness/bmad-testarch-test-design.toml +8 -0
- package/skills/ultracode-goal/customize.toml +12 -7
- package/skills/ultracode-goal/references/define-done.md +3 -3
- package/skills/ultracode-goal/references/execute.md +40 -53
- package/skills/ultracode-goal/references/finalize.md +12 -12
- package/skills/ultracode-goal/references/gate.md +42 -11
- package/skills/ultracode-goal/references/health-check.md +7 -4
- package/skills/ultracode-goal/references/ingest-and-scope.md +4 -4
- package/skills/ultracode-goal/references/preflight.md +10 -9
- package/skills/ultracode-goal/scripts/gate_eval.py +684 -39
- package/skills/ultracode-goal/scripts/gate_trail.py +131 -32
- package/skills/ultracode-goal/scripts/health_check_fp.py +98 -3
- package/skills/ultracode-goal/scripts/hooks/guard_pretooluse.py +56 -14
- package/skills/ultracode-goal/scripts/mem_observation.py +48 -10
- package/skills/ultracode-goal/scripts/mutate_check.py +353 -0
- package/skills/ultracode-goal/scripts/preflight_check.py +397 -3
- package/skills/ultracode-goal/scripts/status_render.py +20 -2
- package/skills/ultracode-goal/scripts/story_sizing.py +193 -0
- package/tools/build-docs.js +24 -1
- package/tools/validate-docs-links.js +169 -8
- package/website/astro.config.mjs +0 -1
- package/docs/parallel-mode.md +0 -68
- package/skills/ultracode-goal/assets/execute-epic.workflow.js +0 -249
|
@@ -45,13 +45,25 @@ Artifact resolution:
|
|
|
45
45
|
and 11-6 is never confused with 1-11-6.
|
|
46
46
|
|
|
47
47
|
With no per-story artifact found, what happens next depends on how the dir
|
|
48
|
-
is NAMED — FAIL-CLOSED, deliberately. If any
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
is NAMED — FAIL-CLOSED, deliberately. If any artifact there is named for SOME
|
|
49
|
+
story or epic (a known artifact prefix followed by a component starting with
|
|
50
|
+
a digit), the dir is per-story-named and the requested story is genuinely
|
|
51
51
|
absent: resolution fails closed to NOT_EVALUATED rather than handing back an
|
|
52
52
|
unrelated story's gate. If every candidate is generically named (trace.md,
|
|
53
53
|
gate-decision.json), the dir holds one story's artifacts and the unscoped
|
|
54
|
-
resolution still applies, so a single-story dir is unchanged.
|
|
54
|
+
resolution still applies, so a single-story dir is unchanged. One exception,
|
|
55
|
+
because the story is not absent: a story that wrote its own
|
|
56
|
+
e2e-trace-summary-<id>.json is PRESENT, and resolution names its own slim
|
|
57
|
+
path so the summary fallback below reads that file rather than the shared one.
|
|
58
|
+
|
|
59
|
+
THE FRONTMATTER HINT IS BOUNDED. A `gateDecisionFile` must name a file
|
|
60
|
+
directly inside <trace-output> — not a subdirectory, not a `..` path, not an
|
|
61
|
+
absolute path elsewhere on disk — and, with --story in scope, must either
|
|
62
|
+
carry that story's id or carry no id at all in a directory holding no OTHER
|
|
63
|
+
story's artifacts. A hint that fails either test is skipped rather than
|
|
64
|
+
followed; resolution continues to the story's own conventionally-named file
|
|
65
|
+
and then to the fail-closed rule above. Following one verbatim let a story
|
|
66
|
+
advance on a neighbour's gate, and on files outside the directory entirely.
|
|
55
67
|
|
|
56
68
|
python3 gate_eval.py --trace-output DIR --profile production --story 11-6 \
|
|
57
69
|
--nfr DIR/nfr-assessment.md --test-review DIR/test-review.md
|
|
@@ -77,8 +89,38 @@ GATE_VERDICT = {
|
|
|
77
89
|
_FRONTMATTER_GATE_KEYS = ("gateDecisionFile", "gateDecisionPath", "gate_decision_path")
|
|
78
90
|
|
|
79
91
|
|
|
80
|
-
def _read_json(path: Path) -> dict:
|
|
81
|
-
|
|
92
|
+
def _read_json(path: Path) -> dict | None:
|
|
93
|
+
"""The JSON object in a file, or None when it cannot be read as one.
|
|
94
|
+
|
|
95
|
+
Unreadable, non-UTF-8, not JSON, and JSON that is not an object all land on
|
|
96
|
+
None together, because they mean the same thing to the callers below: this
|
|
97
|
+
file yields no gate fields, so it gets the fail-closed NOT_EVALUATED an
|
|
98
|
+
absent artifact already gets. A bare ``json.loads`` here RAISED instead: a
|
|
99
|
+
zero-byte or half-written gate file (the residue of a killed TEA write), a
|
|
100
|
+
trailing comma in a hand-authored ``gate-decision-<id>.json``, or a JSON
|
|
101
|
+
array crashed the gate with a traceback and no verdict JSON at all — and
|
|
102
|
+
under ``--rollup`` one such file aborted the whole epic aggregate after
|
|
103
|
+
every story was already done.
|
|
104
|
+
"""
|
|
105
|
+
try:
|
|
106
|
+
loaded = json.loads(path.read_text(encoding="utf-8"))
|
|
107
|
+
except (OSError, ValueError):
|
|
108
|
+
# ValueError covers JSONDecodeError and UnicodeDecodeError alike.
|
|
109
|
+
return None
|
|
110
|
+
return loaded if isinstance(loaded, dict) else None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _read_text(path: Path) -> str | None:
|
|
114
|
+
"""The file's text, or None when it cannot be read.
|
|
115
|
+
|
|
116
|
+
The production-signal scanners below declare a FAIL-CLOSED CONTRACT for
|
|
117
|
+
evidence they cannot read; a bare ``read_text`` raised out of that function
|
|
118
|
+
on an unreadable or non-UTF-8 artifact instead of failing closed.
|
|
119
|
+
"""
|
|
120
|
+
try:
|
|
121
|
+
return path.read_text(encoding="utf-8")
|
|
122
|
+
except (OSError, ValueError):
|
|
123
|
+
return None
|
|
82
124
|
|
|
83
125
|
|
|
84
126
|
def _frontmatter(text: str) -> dict[str, str]:
|
|
@@ -141,11 +183,32 @@ def _stem_matches_story(stem: str, story: str) -> bool:
|
|
|
141
183
|
# The artifact-name prefixes this module writes. A stem that is EXACTLY one of
|
|
142
184
|
# these carries no id; a stem that is one of these followed by something starting
|
|
143
185
|
# with a digit is named for a story or an epic.
|
|
186
|
+
#
|
|
187
|
+
# `traceability-matrix` is in this table and the `("trace",)` entry does NOT
|
|
188
|
+
# cover it: the components are `traceability` / `matrix`. `bmad-testarch-trace`'s
|
|
189
|
+
# `default_output_file` is the BARE `traceability-matrix.md` (verified in its
|
|
190
|
+
# workflow.yaml), so both spellings matter and they matter in opposite
|
|
191
|
+
# directions:
|
|
192
|
+
#
|
|
193
|
+
# - bare `traceability-matrix.md` must read as GENERIC, which is what makes
|
|
194
|
+
# TEA's own untouched output the sanctioned isolated single-story directory.
|
|
195
|
+
# Listing the prefix is what puts it in `gate_trail.GENERIC_ARTIFACT_STEMS`,
|
|
196
|
+
# which is derived from this table - without it the trail rendered `n/a` for
|
|
197
|
+
# a file the gate read as PASS.
|
|
198
|
+
# - `traceability-matrix-<id>.md`, the per-story spelling a run produces when
|
|
199
|
+
# it names TEA's report per story in a shared directory, must read as NAMED.
|
|
200
|
+
# It did not: `_has_trailing_id` returned False, so a directory of them read
|
|
201
|
+
# as generic, the fail-closed branch switched off for the whole directory,
|
|
202
|
+
# and never-driven stories resolved a neighbour's PASS.
|
|
203
|
+
#
|
|
204
|
+
# Both follow from one rule, which is why the entry is a prefix and not a special
|
|
205
|
+
# case: these entries carry an id only when a component FOLLOWS the prefix.
|
|
144
206
|
_ARTIFACT_PREFIXES = (
|
|
145
207
|
("gate", "decision"),
|
|
146
208
|
("e2e", "trace", "summary"),
|
|
147
209
|
("nfr", "assessment"),
|
|
148
210
|
("test", "review"),
|
|
211
|
+
("traceability", "matrix"),
|
|
149
212
|
("trace",),
|
|
150
213
|
("atdd", "checklist"),
|
|
151
214
|
)
|
|
@@ -167,9 +230,14 @@ def _has_trailing_id(stem: str) -> bool:
|
|
|
167
230
|
returned ``PASS``. That story wrote nothing at all.
|
|
168
231
|
|
|
169
232
|
The rule is now structural: match a known artifact prefix, then require the
|
|
170
|
-
NEXT component to start with a digit.
|
|
171
|
-
|
|
172
|
-
|
|
233
|
+
NEXT component to start with a digit. A stem that is EXACTLY a prefix carries
|
|
234
|
+
no id and stays GENERIC, which keeps the documented unscoped fallback for a
|
|
235
|
+
directory that names no story - so bare ``traceability-matrix.md`` is generic
|
|
236
|
+
while ``traceability-matrix-<id>.md`` is named for a story. That pair is
|
|
237
|
+
listed explicitly in the table because the ``("trace",)`` entry does NOT
|
|
238
|
+
cover it: the components are ``traceability`` / ``matrix``, so every
|
|
239
|
+
per-story file from a TEA build using its own default report filename read
|
|
240
|
+
as generic, and the whole directory with it.
|
|
173
241
|
"""
|
|
174
242
|
parts = [p for p in re.split(r"[-._]", stem) if p]
|
|
175
243
|
if not parts:
|
|
@@ -186,9 +254,43 @@ def _is_per_story_named(trace_output: Path) -> bool:
|
|
|
186
254
|
Distinguishes a shared multi-story dir (where a story with no artifacts is
|
|
187
255
|
genuinely absent) from the single-story dir whose generically-named files
|
|
188
256
|
the unscoped fallback is documented to resolve.
|
|
257
|
+
|
|
258
|
+
The glob is EVERY md and json here, not `*.md` plus `gate-decision*.json`.
|
|
259
|
+
The narrower pair was blind to a directory whose per-story artifacts are
|
|
260
|
+
spelled `e2e-trace-summary-<id>.json`: nothing matched, the directory read as
|
|
261
|
+
generic, the fail-closed branch switched off, and a never-driven story
|
|
262
|
+
resolved the shared roll-up `e2e-trace-summary.json` as its own PASS.
|
|
263
|
+
`gate_trail._only_generic_artifacts`, which decides the same question from
|
|
264
|
+
the other side, has always globbed both; the two now see one directory.
|
|
189
265
|
"""
|
|
190
|
-
|
|
191
|
-
|
|
266
|
+
return _per_story_naming_kind(trace_output) is not None
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _per_story_naming_kind(trace_output: Path) -> str | None:
|
|
270
|
+
"""The artifact KIND whose per-story name makes this directory per-story-named.
|
|
271
|
+
|
|
272
|
+
The boolean above answers "is it"; this answers "because of what", so a
|
|
273
|
+
fail-closed reason can say which artifact class flipped the directory. That
|
|
274
|
+
is not derivable from the docs: the rule counts every prefix in
|
|
275
|
+
`_ARTIFACT_PREFIXES`, so a lone `nfr-assessment-<id>.md` or
|
|
276
|
+
`test-review-<id>.md` — files `references/gate.md` ORDERS the non-web author
|
|
277
|
+
to write into this same directory — is enough on its own, and an operator
|
|
278
|
+
who reads only the two gate-bearing names goes hunting for a trace/gate stem
|
|
279
|
+
mismatch that does not exist.
|
|
280
|
+
|
|
281
|
+
The KIND only, never the filename: the story this reason is written for is
|
|
282
|
+
ABSENT, and quoting a neighbour's id in its verdict is the shape of the
|
|
283
|
+
fail-open this branch exists to prevent. Sorted, so the answer is stable.
|
|
284
|
+
"""
|
|
285
|
+
candidates = sorted(list(trace_output.glob("*.md")) + list(trace_output.glob("*.json")))
|
|
286
|
+
for path in candidates:
|
|
287
|
+
if not _has_trailing_id(path.stem):
|
|
288
|
+
continue
|
|
289
|
+
parts = [p for p in re.split(r"[-._]", path.stem) if p]
|
|
290
|
+
for prefix in _ARTIFACT_PREFIXES:
|
|
291
|
+
if len(parts) > len(prefix) and tuple(parts[: len(prefix)]) == prefix:
|
|
292
|
+
return "-".join(prefix)
|
|
293
|
+
return None
|
|
192
294
|
|
|
193
295
|
|
|
194
296
|
def _per_story_slim(trace_output: Path, story: str | None) -> Path | None:
|
|
@@ -200,6 +302,31 @@ def _per_story_slim(trace_output: Path, story: str | None) -> Path | None:
|
|
|
200
302
|
return None
|
|
201
303
|
|
|
202
304
|
|
|
305
|
+
def _owns_a_per_story_summary(trace_output: Path, story: str | None) -> bool:
|
|
306
|
+
"""True iff this story wrote its own ``e2e-trace-summary-<id>.json`` here.
|
|
307
|
+
|
|
308
|
+
The fail-closed branches below mean "this story is absent from a directory
|
|
309
|
+
that names its artifacts per story". That is FALSE when the story wrote its
|
|
310
|
+
own per-story summary, and reading it as absent was a liveness regression
|
|
311
|
+
rather than a safety win: the summary is the ALWAYS-written artifact, so a
|
|
312
|
+
story whose run was not gate-eligible has that file and nothing else, and
|
|
313
|
+
refusing it reports NOT_EVALUATED for a story whose own evidence is sitting
|
|
314
|
+
in the directory.
|
|
315
|
+
|
|
316
|
+
Deliberately narrower than "owns any file here". `references/gate.md` has the
|
|
317
|
+
non-web author write `nfr-assessment-<id>.md` and `test-review-<id>.md` into
|
|
318
|
+
this same directory, and neither carries a gate decision - a story with only
|
|
319
|
+
those must still fail closed, which is a behaviour with its own test. The
|
|
320
|
+
slim file is not checked either, because every caller has already returned on
|
|
321
|
+
it (`scoped_slim`) before reaching here.
|
|
322
|
+
"""
|
|
323
|
+
if not story or not story.strip():
|
|
324
|
+
return False
|
|
325
|
+
return any(
|
|
326
|
+
(trace_output / f"e2e-trace-summary-{v}.json").is_file() for v in _story_variants(story)
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
|
|
203
330
|
# The `workflowType` values a trace report must declare. A report that declares
|
|
204
331
|
# neither is skipped by the hint loop, so it can carry no gate decision.
|
|
205
332
|
_TRACE_WORKFLOW_TYPES = ("testarch-trace", "trace")
|
|
@@ -219,7 +346,84 @@ def _declares_trace_report(report: Path) -> bool:
|
|
|
219
346
|
return fm.get("workflowType") in _TRACE_WORKFLOW_TYPES
|
|
220
347
|
|
|
221
348
|
|
|
222
|
-
def
|
|
349
|
+
def _holds_another_storys_artifact(trace_output: Path, story: str) -> bool:
|
|
350
|
+
"""True iff some artifact here is named for a story OTHER than this one.
|
|
351
|
+
|
|
352
|
+
The question a GENERIC filename raises is not "does this directory name
|
|
353
|
+
anything per story" but "could this unnamed file be someone else's". Those
|
|
354
|
+
differ on exactly the shape `references/gate.md` tells an operator to build:
|
|
355
|
+
isolate one story's artifacts into a fresh directory. A TEA build that names
|
|
356
|
+
the trace report per story but the gate decision generically produces
|
|
357
|
+
`trace-<id>.md` + `gate-decision.json` there, which IS per-story-named and
|
|
358
|
+
holds no other story - so the generic gate file provably belongs to the one
|
|
359
|
+
story being gated, and refusing it escalates a story with a real PASS.
|
|
360
|
+
"""
|
|
361
|
+
candidates = list(trace_output.glob("*.md")) + list(trace_output.glob("*.json"))
|
|
362
|
+
return any(
|
|
363
|
+
_has_trailing_id(p.stem) and not _stem_matches_story(p.stem, story) for p in candidates
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _hinted_gate_file(trace_output: Path, hint: str, story: str | None) -> Path | None:
|
|
368
|
+
"""The file a ``gateDecisionFile`` hint names, or None when it must not be followed.
|
|
369
|
+
|
|
370
|
+
A hint is a string inside a trace report, and the report is written by the
|
|
371
|
+
same run whose completion it is evidence for. It was previously followed
|
|
372
|
+
verbatim, with no name check and no containment check at all, so three
|
|
373
|
+
distinct shapes each handed a story a PASS it did not own: a hint naming a
|
|
374
|
+
NEIGHBOUR's ``gate-decision-<other>.json``, a relative hint escaping the
|
|
375
|
+
artifact directory (``../other-epic/gate-decision.json``), and an ABSOLUTE
|
|
376
|
+
hint naming any path on disk. All three were measured returning
|
|
377
|
+
``PASS -> advance`` for a story that owned no gate decision.
|
|
378
|
+
|
|
379
|
+
Two conditions now, and both are the artifact-resolution property rather than
|
|
380
|
+
new policy:
|
|
381
|
+
|
|
382
|
+
- CONTAINMENT. The file must sit directly in ``trace_output``. ``--trace-output``
|
|
383
|
+
is the boundary the caller drew around this run's evidence; a gate read from
|
|
384
|
+
outside it is not this run's gate. Absolute hints stay legal, but only when
|
|
385
|
+
they resolve back inside that directory.
|
|
386
|
+
- OWNERSHIP. With a story in scope, a hinted file that carries an id must
|
|
387
|
+
carry THIS story's id. A hinted file carrying NO id is accepted only when
|
|
388
|
+
the directory holds no OTHER story's artifacts, which is the isolated
|
|
389
|
+
single-story directory `references/gate.md` tells an operator to build -
|
|
390
|
+
there the unnamed file provably belongs to the one story being gated. The
|
|
391
|
+
test is "is anyone else here", not "is anything here named": keying on the
|
|
392
|
+
latter escalated a story whose TEA build named its trace report per story
|
|
393
|
+
and its gate decision generically, which is a shape that workflow produces.
|
|
394
|
+
|
|
395
|
+
Returning None means "do not follow this hint". The caller still prefers the
|
|
396
|
+
story's OWN ``gate-decision-<id>.json`` when one exists - that artifact is
|
|
397
|
+
named for this story, so it is not a file nobody pointed at - but it resolves
|
|
398
|
+
NOTHING else afterwards. A refused hint is not a missing hint: the run named
|
|
399
|
+
a file and this reader declined it, so falling on to the next candidate can
|
|
400
|
+
RAISE the verdict, and it did (a refused FAIL became an advance off the
|
|
401
|
+
story's own summary). See the ``refused_hint`` latch in the caller.
|
|
402
|
+
"""
|
|
403
|
+
hinted = Path(hint)
|
|
404
|
+
candidate = hinted if hinted.is_absolute() else trace_output / hinted
|
|
405
|
+
try:
|
|
406
|
+
resolved = candidate.resolve()
|
|
407
|
+
inside = resolved.relative_to(trace_output.resolve())
|
|
408
|
+
except (ValueError, OSError):
|
|
409
|
+
return None
|
|
410
|
+
# Exactly one component: a plain filename sitting directly in the directory.
|
|
411
|
+
# Testing `inside.parent != Path(".")` instead let a hint of `.` or `""`
|
|
412
|
+
# through, because both resolve to the DIRECTORY, whose relative form is `.`
|
|
413
|
+
# and whose parent is also `.` - so resolution returned the artifact
|
|
414
|
+
# directory itself as the gate file.
|
|
415
|
+
if len(inside.parts) != 1:
|
|
416
|
+
return None
|
|
417
|
+
if story and story.strip():
|
|
418
|
+
if _has_trailing_id(resolved.stem):
|
|
419
|
+
return candidate if _stem_matches_story(resolved.stem, story) else None
|
|
420
|
+
return None if _holds_another_storys_artifact(trace_output, story) else candidate
|
|
421
|
+
return candidate
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _resolve_gate_file(
|
|
425
|
+
trace_output: Path, story: str | None = None, notes: list[str] | None = None
|
|
426
|
+
) -> Path | None:
|
|
223
427
|
"""Locate the slim gate-decision file, honoring a trace-report hint.
|
|
224
428
|
|
|
225
429
|
With ``story`` set, scope resolution to that story's artifacts so a single
|
|
@@ -231,6 +435,7 @@ def _resolve_gate_file(trace_output: Path, story: str | None = None) -> Path | N
|
|
|
231
435
|
"""
|
|
232
436
|
reports = sorted(trace_output.glob("*.md"))
|
|
233
437
|
scoped_slim = _per_story_slim(trace_output, story)
|
|
438
|
+
refused_hint = False
|
|
234
439
|
if story and story.strip():
|
|
235
440
|
# A markdown file merely NAMED for this story is not a trace report, and
|
|
236
441
|
# letting one count here suppresses the fail-closed branch below.
|
|
@@ -264,25 +469,89 @@ def _resolve_gate_file(trace_output: Path, story: str | None = None) -> Path | N
|
|
|
264
469
|
# gate — observed handing an unrelated epic's PASS back as this
|
|
265
470
|
# story's verdict. The story's own file wins, before any hint is read.
|
|
266
471
|
return scoped_slim
|
|
267
|
-
elif _is_per_story_named(trace_output)
|
|
472
|
+
elif _is_per_story_named(trace_output) and not _owns_a_per_story_summary(
|
|
473
|
+
trace_output, story
|
|
474
|
+
):
|
|
268
475
|
return None
|
|
269
476
|
for report in reports:
|
|
270
477
|
try:
|
|
271
478
|
fm = _frontmatter(report.read_text(encoding="utf-8"))
|
|
272
|
-
except OSError:
|
|
479
|
+
except (OSError, UnicodeDecodeError):
|
|
480
|
+
# UnicodeDecodeError is NOT an OSError, so a single non-UTF-8 markdown
|
|
481
|
+
# file anywhere in the directory crashed resolution here. Catching it
|
|
482
|
+
# is right; treating it as "carries no hint" is NOT, and `reports` is
|
|
483
|
+
# the UNFILTERED `*.md` glob on this path (the `_declares_trace_report`
|
|
484
|
+
# filter applies only to the story-scoped branch above), so this reader
|
|
485
|
+
# cannot know whether the file it failed to decode was a trace report
|
|
486
|
+
# carrying a `gateDecisionFile` hint.
|
|
487
|
+
#
|
|
488
|
+
# An unreadable report is a hint that could not be READ, never a report
|
|
489
|
+
# with no hint - the same distinction the `refused_hint` branch below
|
|
490
|
+
# exists to hold. Skipping silently let resolution fall through to the
|
|
491
|
+
# unscoped `gate-decision.json`: measured, a story whose hinted file
|
|
492
|
+
# said FAIL advanced on a neighbour's PASS, and the log read like an
|
|
493
|
+
# ordinary successful resolution because the fallback named a real file.
|
|
494
|
+
if notes is not None:
|
|
495
|
+
notes.append(
|
|
496
|
+
f"{report.name} could not be decoded; any gate hint it carries "
|
|
497
|
+
f"was not read, so no unpointed-at file is resolved in its place"
|
|
498
|
+
)
|
|
499
|
+
refused_hint = True
|
|
273
500
|
continue
|
|
274
501
|
if fm.get("workflowType") not in _TRACE_WORKFLOW_TYPES:
|
|
275
502
|
continue
|
|
276
503
|
for key in _FRONTMATTER_GATE_KEYS:
|
|
277
504
|
hint = fm.get(key)
|
|
278
505
|
if hint:
|
|
279
|
-
hinted =
|
|
280
|
-
|
|
506
|
+
hinted = _hinted_gate_file(trace_output, hint, story)
|
|
507
|
+
if hinted is not None:
|
|
508
|
+
return hinted
|
|
509
|
+
# A hint that names another story's file, or a path outside the
|
|
510
|
+
# artifact directory, is not followed and is not fatal either:
|
|
511
|
+
# fall through to this story's own conventionally-named file and,
|
|
512
|
+
# failing that, to the fail-closed branch below.
|
|
513
|
+
#
|
|
514
|
+
# It is RECORDED, though. Refusing a hint and then resolving some
|
|
515
|
+
# other file changes which artifact decided, and a reader who
|
|
516
|
+
# cannot see that happened has no way to tell a gate read from
|
|
517
|
+
# the intended file from one read from the fallback - which can
|
|
518
|
+
# carry a different verdict.
|
|
519
|
+
if notes is not None:
|
|
520
|
+
notes.append(
|
|
521
|
+
f"{report.name} hints at {hint!r}, which is not a file this story owns "
|
|
522
|
+
f"inside {trace_output}; the hint was not followed"
|
|
523
|
+
)
|
|
524
|
+
refused_hint = True
|
|
525
|
+
break
|
|
281
526
|
# No frontmatter hint. With a story in scope, prefer the conventionally-named
|
|
282
527
|
# per-story slim file before the shared default so the shared dir resolves
|
|
283
528
|
# the right story even when no trace report points at it.
|
|
284
529
|
if scoped_slim is not None:
|
|
285
530
|
return scoped_slim
|
|
531
|
+
if refused_hint:
|
|
532
|
+
# A REFUSED HINT IS NOT A MISSING HINT. The run pointed at a specific
|
|
533
|
+
# file and this reader declined to read it, so every remaining candidate
|
|
534
|
+
# below is a file nobody pointed at. Resolving one anyway does not merely
|
|
535
|
+
# lose information, it can RAISE the verdict: measured, a subdirectory
|
|
536
|
+
# hint carrying FAIL was refused and the story then advanced on its own
|
|
537
|
+
# summary's PASS, and an unscoped run whose hinted CONCERNS was refused
|
|
538
|
+
# advanced on the generic summary. Both read as ordinary successful
|
|
539
|
+
# resolutions in the log, because the fallback names a real file.
|
|
540
|
+
#
|
|
541
|
+
# The story's OWN slim file is the one exception, and it is already
|
|
542
|
+
# returned above: that artifact is named for this story, so it is not a
|
|
543
|
+
# file nobody pointed at. Everything after this line is.
|
|
544
|
+
return None
|
|
545
|
+
if story and _is_per_story_named(trace_output) and _owns_a_per_story_summary(trace_output, story):
|
|
546
|
+
# THIS STORY IS PRESENT, but its only gate-bearing artifact is its own
|
|
547
|
+
# summary. Falling through to the shared `gate-decision.json` below would
|
|
548
|
+
# hand it the RUN-WIDE verdict: measured, a story whose own summary said
|
|
549
|
+
# CONCERNS came back PASS -> advance off the generic file. Name this
|
|
550
|
+
# story's own conventional slim path instead. It does not exist (a slim
|
|
551
|
+
# file that did exist was returned as `scoped_slim` long before here), so
|
|
552
|
+
# `load_gate` falls through to `_resolve_summary_file`, which resolves the
|
|
553
|
+
# per-story summary this branch exists for.
|
|
554
|
+
return trace_output / f"gate-decision-{story}.json"
|
|
286
555
|
if story and _is_per_story_named(trace_output):
|
|
287
556
|
# The SAME fail-closed rule as the branch above, applied to the last
|
|
288
557
|
# resort. That `return None` is guarded by `elif`, so it is reachable
|
|
@@ -292,6 +561,11 @@ def _resolve_gate_file(trace_output: Path, story: str | None = None) -> Path | N
|
|
|
292
561
|
# unscoped `gate-decision.json` is a NEIGHBOUR's, or the epic roll-up's,
|
|
293
562
|
# verdict. Having written a trace report must not buy a story a pass out
|
|
294
563
|
# of the fail-closed lane that a story with no artifacts at all gets.
|
|
564
|
+
#
|
|
565
|
+
# A story that wrote its OWN per-story summary is not in that lane: it
|
|
566
|
+
# is present, its evidence is here, and `load_gate` reads that summary
|
|
567
|
+
# one step below. Falling closed on it reported NOT_EVALUATED for a story
|
|
568
|
+
# whose only artifact is the always-written one.
|
|
295
569
|
return None
|
|
296
570
|
return trace_output / "gate-decision.json"
|
|
297
571
|
|
|
@@ -322,14 +596,25 @@ def _resolve_summary_file(trace_output: Path, story: str | None) -> Path:
|
|
|
322
596
|
|
|
323
597
|
def load_gate(trace_output: Path, reasons: list[str], story: str | None = None) -> dict:
|
|
324
598
|
"""Return normalized gate fields, preferring the slim file, else the summary."""
|
|
325
|
-
gate_file = _resolve_gate_file(trace_output, story)
|
|
599
|
+
gate_file = _resolve_gate_file(trace_output, story, reasons)
|
|
326
600
|
if gate_file is None:
|
|
327
601
|
# Fail-closed: the dir names its artifacts per story and none is this
|
|
328
602
|
# story's, so there is nothing to read. Do NOT fall back to the unscoped
|
|
329
603
|
# read — it would report a neighbouring story's gate as this one's.
|
|
604
|
+
# Name WHY the fallback was refused. "The directory is named per story"
|
|
605
|
+
# alone is not diagnosable: the rule fires on any known artifact prefix
|
|
606
|
+
# followed by an id, so a per-story NFR or test-review is enough, and a
|
|
607
|
+
# reader who assumes only the gate-bearing names hunts for a stem
|
|
608
|
+
# mismatch that is not there.
|
|
609
|
+
kind = _per_story_naming_kind(trace_output)
|
|
610
|
+
why = (
|
|
611
|
+
f"the directory is named per story (a {kind}-* file there carries a story id)"
|
|
612
|
+
if kind
|
|
613
|
+
else "the gate file the trace report hinted at was refused above"
|
|
614
|
+
)
|
|
330
615
|
reasons.append(
|
|
331
616
|
f"story {story} has no trace report or gate decision in {trace_output}; "
|
|
332
|
-
"
|
|
617
|
+
f"{why}, so no unscoped fallback was taken"
|
|
333
618
|
)
|
|
334
619
|
return {
|
|
335
620
|
"gate_status": "NOT_EVALUATED",
|
|
@@ -340,6 +625,19 @@ def load_gate(trace_output: Path, reasons: list[str], story: str | None = None)
|
|
|
340
625
|
|
|
341
626
|
if gate_file.is_file():
|
|
342
627
|
slim = _read_json(gate_file)
|
|
628
|
+
if slim is None:
|
|
629
|
+
# Present but unreadable is not "no artifact": the file exists, so no
|
|
630
|
+
# fallback below is this story's gate either. Fail closed on it.
|
|
631
|
+
reasons.append(
|
|
632
|
+
f"{gate_file.name} in {trace_output} is present but is not readable as a "
|
|
633
|
+
"JSON object (empty, malformed, or not an object); treated as no gate decision"
|
|
634
|
+
)
|
|
635
|
+
return {
|
|
636
|
+
"gate_status": "NOT_EVALUATED",
|
|
637
|
+
"p0_status": None,
|
|
638
|
+
"p1_status": None,
|
|
639
|
+
"overall_status": None,
|
|
640
|
+
}
|
|
343
641
|
reasons.append(f"gate read from {gate_file.name}")
|
|
344
642
|
return {
|
|
345
643
|
"gate_status": slim.get("gate_status", "NOT_EVALUATED"),
|
|
@@ -361,8 +659,9 @@ def load_gate(trace_output: Path, reasons: list[str], story: str | None = None)
|
|
|
361
659
|
# different filename. A per-story-named summary is still honored above.
|
|
362
660
|
reasons.append(
|
|
363
661
|
f"story {story} has no per-story gate decision or summary in {trace_output}; "
|
|
364
|
-
"the directory is named per story
|
|
365
|
-
"
|
|
662
|
+
"the directory is named per story (a "
|
|
663
|
+
f"{_per_story_naming_kind(trace_output)}-* file there carries a story id), "
|
|
664
|
+
"so the shared e2e-trace-summary.json was not read as this story's gate"
|
|
366
665
|
)
|
|
367
666
|
return {
|
|
368
667
|
"gate_status": "NOT_EVALUATED",
|
|
@@ -371,10 +670,22 @@ def load_gate(trace_output: Path, reasons: list[str], story: str | None = None)
|
|
|
371
670
|
"overall_status": None,
|
|
372
671
|
}
|
|
373
672
|
if summary_file.is_file():
|
|
673
|
+
summary = _read_json(summary_file)
|
|
674
|
+
if summary is None:
|
|
675
|
+
reasons.append(
|
|
676
|
+
f"{gate_file.name} absent and {summary_file.name} is not readable as a "
|
|
677
|
+
"JSON object (empty, malformed, or not an object); treated as no gate decision"
|
|
678
|
+
)
|
|
679
|
+
return {
|
|
680
|
+
"gate_status": "NOT_EVALUATED",
|
|
681
|
+
"p0_status": None,
|
|
682
|
+
"p1_status": None,
|
|
683
|
+
"overall_status": None,
|
|
684
|
+
}
|
|
374
685
|
reasons.append(
|
|
375
686
|
f"{gate_file.name} absent; gate read from {summary_file.name} (not a failure)"
|
|
376
687
|
)
|
|
377
|
-
return _gate_fields_from_summary(
|
|
688
|
+
return _gate_fields_from_summary(summary)
|
|
378
689
|
|
|
379
690
|
reasons.append(
|
|
380
691
|
f"neither {gate_file.name} nor e2e-trace-summary.json present in {trace_output}"
|
|
@@ -449,8 +760,14 @@ def apply_production_and(
|
|
|
449
760
|
failed = False
|
|
450
761
|
|
|
451
762
|
if nfr_path is not None and nfr_path.is_file():
|
|
452
|
-
|
|
453
|
-
if
|
|
763
|
+
nfr_text = _read_text(nfr_path)
|
|
764
|
+
if nfr_text is None:
|
|
765
|
+
# A file that exists but cannot be read is the fail-closed contract's
|
|
766
|
+
# own case, not a crash: report it as its own failing signal rather
|
|
767
|
+
# than as the "field not found" one two branches down.
|
|
768
|
+
reasons.append(f"nfr file {nfr_path} could not be read; treated as failing")
|
|
769
|
+
failed = True
|
|
770
|
+
elif (nfr_status := _scan_nfr_overall_status(nfr_text)) == "FAIL":
|
|
454
771
|
reasons.append("nfr overallStatus is FAIL")
|
|
455
772
|
failed = True
|
|
456
773
|
elif nfr_status == "NOT_ASSESSED":
|
|
@@ -479,18 +796,26 @@ def apply_production_and(
|
|
|
479
796
|
failed = True
|
|
480
797
|
|
|
481
798
|
if review_path is not None and review_path.is_file():
|
|
482
|
-
review_text = review_path
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
reasons.append(f"test-review
|
|
487
|
-
failed = True
|
|
488
|
-
elif review_score < 80:
|
|
489
|
-
reasons.append(f"test-review score {review_score} < 80")
|
|
490
|
-
failed = True
|
|
491
|
-
if recommendation is not None and recommendation.lower() == "block":
|
|
492
|
-
reasons.append("test-review recommendation is Block")
|
|
799
|
+
review_text = _read_text(review_path)
|
|
800
|
+
if review_text is None:
|
|
801
|
+
# Same fail-closed case as the NFR read above: present but unreadable
|
|
802
|
+
# is a failing signal, not an exception out of this function.
|
|
803
|
+
reasons.append(f"test-review file {review_path} could not be read; treated as failing")
|
|
493
804
|
failed = True
|
|
805
|
+
else:
|
|
806
|
+
review_score = _scan_review_score(review_text)
|
|
807
|
+
recommendation = _scan_review_recommendation(review_text)
|
|
808
|
+
if review_score is None:
|
|
809
|
+
reasons.append(
|
|
810
|
+
f"test-review score not found in {review_path.name}; treated as failing"
|
|
811
|
+
)
|
|
812
|
+
failed = True
|
|
813
|
+
elif review_score < 80:
|
|
814
|
+
reasons.append(f"test-review score {review_score} < 80")
|
|
815
|
+
failed = True
|
|
816
|
+
if recommendation is not None and recommendation.lower() == "block":
|
|
817
|
+
reasons.append("test-review recommendation is Block")
|
|
818
|
+
failed = True
|
|
494
819
|
elif review_path is not None:
|
|
495
820
|
reasons.append(f"test-review file {review_path} not found; treated as failing")
|
|
496
821
|
failed = True
|
|
@@ -508,11 +833,212 @@ def apply_production_and(
|
|
|
508
833
|
return verdict, nfr_status, review_score
|
|
509
834
|
|
|
510
835
|
|
|
836
|
+
# The marker's scope line, written by execute.md steps 2/5 beside baseline=.
|
|
837
|
+
_SWEEP_SCOPE_RE = re.compile(r"^scope=(\S+)\s*$", re.MULTILINE)
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def apply_sweep_scope(
|
|
841
|
+
verdict: str, tests_ran_path: Path, reasons: list[str]
|
|
842
|
+
) -> tuple[str, str | None]:
|
|
843
|
+
"""Cap a story-moving verdict at reloop unless the sweep behind it was full.
|
|
844
|
+
|
|
845
|
+
Reads the story's ``.tests-ran-<story_id>`` marker for its
|
|
846
|
+
``scope=<full|scoped>`` line. ``full`` passes the verdict through;
|
|
847
|
+
``scoped`` is legitimate mid-loop evidence whose verdict is capped at
|
|
848
|
+
reloop - a scoped sweep must not impersonate a full one at the one place
|
|
849
|
+
the distinction pays. Unlike the production AND's floor, the cap moves
|
|
850
|
+
``defer`` as well as ``advance``, because defer's route ADVANCES the story
|
|
851
|
+
(ledger, then advance anyway), and a story moving forward off mid-loop
|
|
852
|
+
evidence is exactly what the cap exists to refuse.
|
|
853
|
+
|
|
854
|
+
FAIL-CLOSED like every other signal: a missing or unreadable marker, a
|
|
855
|
+
marker with no ``scope=`` line, and a value this reader does not recognise
|
|
856
|
+
all get the same cap, with a reason naming what was wrong. Only the OMITTED
|
|
857
|
+
FLAG is a no-op, and that is deliberate rather than a soft spot: making
|
|
858
|
+
omission failing would change the verdict of documented invocations that
|
|
859
|
+
predate the flag (docs/_internal/STABILITY.md carries the 2.0.0 lesson on
|
|
860
|
+
exactly that), so the always-pass-it rule is the instruction layer's to
|
|
861
|
+
state, not this reader's to retrofit.
|
|
862
|
+
|
|
863
|
+
Returns ``(verdict, sweep_scope)`` where ``sweep_scope`` is the recognised
|
|
864
|
+
value read from the marker, or None when none was.
|
|
865
|
+
"""
|
|
866
|
+
scope: str | None = None
|
|
867
|
+
problem: str | None = None
|
|
868
|
+
text = _read_text(tests_ran_path) if tests_ran_path.is_file() else None
|
|
869
|
+
if text is None:
|
|
870
|
+
problem = f"tests-ran marker {tests_ran_path} missing or unreadable"
|
|
871
|
+
else:
|
|
872
|
+
values = _SWEEP_SCOPE_RE.findall(text)
|
|
873
|
+
if not values:
|
|
874
|
+
problem = (
|
|
875
|
+
f"tests-ran marker {tests_ran_path.name} carries no scope= line "
|
|
876
|
+
"(execute.md step 2 writes scope=full|scoped beside baseline=)"
|
|
877
|
+
)
|
|
878
|
+
elif len(set(values)) > 1:
|
|
879
|
+
# Disagreeing duplicates are a marker that was APPENDED to rather
|
|
880
|
+
# than rewritten - and picking either line would be a guess. Taking
|
|
881
|
+
# the first is the trap specifically: a re-loop cycle appending its
|
|
882
|
+
# scoped line under a first pass's scope=full would read as full,
|
|
883
|
+
# which is the one impersonation this reader exists to refuse.
|
|
884
|
+
problem = (
|
|
885
|
+
f"tests-ran marker {tests_ran_path.name} carries "
|
|
886
|
+
f"{len(values)} disagreeing scope= lines ({', '.join(values)}); "
|
|
887
|
+
"rewrite the marker with exactly one"
|
|
888
|
+
)
|
|
889
|
+
elif values[0] not in ("full", "scoped"):
|
|
890
|
+
problem = (
|
|
891
|
+
f"tests-ran marker {tests_ran_path.name} carries unrecognised "
|
|
892
|
+
f"scope={values[0]!r}"
|
|
893
|
+
)
|
|
894
|
+
else:
|
|
895
|
+
scope = values[0]
|
|
896
|
+
|
|
897
|
+
if scope == "full":
|
|
898
|
+
return verdict, scope
|
|
899
|
+
if verdict in ("advance", "defer"):
|
|
900
|
+
reasons.append(
|
|
901
|
+
f"{problem}; treated as failing" if problem is not None else "sweep scope is scoped"
|
|
902
|
+
)
|
|
903
|
+
reasons.append(
|
|
904
|
+
f"{verdict} capped at reloop: a story only advances off a full-scope "
|
|
905
|
+
"sweep - re-run the full three-axis sweep, refresh the marker with "
|
|
906
|
+
"scope=full, and re-run the gate"
|
|
907
|
+
)
|
|
908
|
+
verdict = "reloop"
|
|
909
|
+
elif problem is not None:
|
|
910
|
+
# A verdict already at reloop/escalate is not moved, but the marker
|
|
911
|
+
# problem still gets a line - the re-loop should fix the marker too.
|
|
912
|
+
reasons.append(f"{problem} (verdict already {verdict}; not moved)")
|
|
913
|
+
else:
|
|
914
|
+
reasons.append(f"sweep scope scoped (verdict already {verdict}; not moved)")
|
|
915
|
+
return verdict, scope
|
|
916
|
+
|
|
917
|
+
|
|
918
|
+
# Severity order for the roll-up aggregate: the epic's status is its WORST
|
|
919
|
+
# story's status, and an unknown status is treated as the worst there is.
|
|
920
|
+
_ROLLUP_SEVERITY = ("PASS", "WAIVED", "CONCERNS", "FAIL", "NOT_EVALUATED")
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
def rollup_gate(
|
|
924
|
+
trace_output: Path, sprint_status: Path, epic: str, reasons: list[str]
|
|
925
|
+
) -> dict:
|
|
926
|
+
"""Aggregate the epic verdict from per-story artifacts, fail-closed.
|
|
927
|
+
|
|
928
|
+
The story set comes from sprint-status.yaml (the authority on which stories
|
|
929
|
+
the epic owes), never from a caller-typed list - a story omitted from a
|
|
930
|
+
typed list would simply not be aggregated, which is the fabrication channel
|
|
931
|
+
this reader exists to close. Refusals, all NOT_EVALUATED -> escalate:
|
|
932
|
+
an unreadable sprint-status, an epic with no stories, any story not `done`
|
|
933
|
+
(the roll-up's own trigger is every-story-done, so running it early is
|
|
934
|
+
refused rather than answered), and any done story whose own gate artifact
|
|
935
|
+
does not resolve - a `done` row nothing ever gated is exactly what this
|
|
936
|
+
aggregate must surface, not paper over.
|
|
937
|
+
"""
|
|
938
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
939
|
+
import preflight_check # noqa: PLC0415 (sibling script, path set above)
|
|
940
|
+
|
|
941
|
+
try:
|
|
942
|
+
text = sprint_status.read_text(encoding="utf-8")
|
|
943
|
+
except (OSError, UnicodeDecodeError):
|
|
944
|
+
reasons.append(f"sprint-status unreadable at {sprint_status}; roll-up refused")
|
|
945
|
+
return {
|
|
946
|
+
"gate_status": "NOT_EVALUATED",
|
|
947
|
+
"p0_status": None,
|
|
948
|
+
"p1_status": None,
|
|
949
|
+
"overall_status": None,
|
|
950
|
+
}
|
|
951
|
+
statuses = preflight_check._parse_development_status(text)
|
|
952
|
+
stories = [
|
|
953
|
+
key
|
|
954
|
+
for key in statuses
|
|
955
|
+
if preflight_check._is_story_key(key)
|
|
956
|
+
and preflight_check._epic_id_of(key) == str(epic)
|
|
957
|
+
]
|
|
958
|
+
refused = None
|
|
959
|
+
if not stories:
|
|
960
|
+
refused = f"no stories for epic {epic} in {sprint_status.name}; roll-up refused"
|
|
961
|
+
else:
|
|
962
|
+
not_done = [key for key in stories if statuses[key] != "done"]
|
|
963
|
+
if not_done:
|
|
964
|
+
refused = (
|
|
965
|
+
"roll-up refused: %d of %d stories not done (%s)"
|
|
966
|
+
% (len(not_done), len(stories), ", ".join(not_done[:5]))
|
|
967
|
+
)
|
|
968
|
+
if refused:
|
|
969
|
+
reasons.append(refused)
|
|
970
|
+
return {
|
|
971
|
+
"gate_status": "NOT_EVALUATED",
|
|
972
|
+
"p0_status": None,
|
|
973
|
+
"p1_status": None,
|
|
974
|
+
"overall_status": None,
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
# A generically-named directory (TEA's untouched single-story output) has
|
|
978
|
+
# exactly one gate artifact, and the unscoped fallback would hand that one
|
|
979
|
+
# file to EVERY story here - a multi-story epic advancing on a single
|
|
980
|
+
# artifact, each reason line asserting a per-story status the directory
|
|
981
|
+
# cannot support. One story may legitimately live in such a directory
|
|
982
|
+
# (gate_eval's documented isolated-dir rule); two or more cannot.
|
|
983
|
+
if len(stories) > 1 and not _is_per_story_named(trace_output):
|
|
984
|
+
reasons.append(
|
|
985
|
+
"roll-up refused: %s is not per-story-named, so one shared artifact "
|
|
986
|
+
"would decide all %d stories" % (trace_output, len(stories))
|
|
987
|
+
)
|
|
988
|
+
return {
|
|
989
|
+
"gate_status": "NOT_EVALUATED",
|
|
990
|
+
"p0_status": None,
|
|
991
|
+
"p1_status": None,
|
|
992
|
+
"overall_status": None,
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
worst = "PASS"
|
|
996
|
+
for key in stories:
|
|
997
|
+
sub: list[str] = []
|
|
998
|
+
gate = load_gate(trace_output, sub, key)
|
|
999
|
+
status = (gate["gate_status"] or "NOT_EVALUATED").upper()
|
|
1000
|
+
if status not in _ROLLUP_SEVERITY:
|
|
1001
|
+
reasons.append(
|
|
1002
|
+
f"story {key}: unrecognized gate_status {status!r}; treated as NOT_EVALUATED"
|
|
1003
|
+
)
|
|
1004
|
+
status = "NOT_EVALUATED"
|
|
1005
|
+
elif status == "NOT_EVALUATED":
|
|
1006
|
+
reasons.append(
|
|
1007
|
+
f"story {key}: gate_status NOT_EVALUATED - a done row with no "
|
|
1008
|
+
"resolvable gate artifact of its own"
|
|
1009
|
+
)
|
|
1010
|
+
else:
|
|
1011
|
+
reasons.append(f"story {key}: gate_status {status}")
|
|
1012
|
+
# The resolver's own record (which file decided, any refused hint) must
|
|
1013
|
+
# not be dropped by the aggregation - the trail has to agree with the
|
|
1014
|
+
# gate about which file decided. Surface it for every non-PASS story,
|
|
1015
|
+
# and always for a refused or honoured hint.
|
|
1016
|
+
for line in sub:
|
|
1017
|
+
if status != "PASS" or "hint" in line.lower():
|
|
1018
|
+
reasons.append(f"story {key}: {line}")
|
|
1019
|
+
if _ROLLUP_SEVERITY.index(status) > _ROLLUP_SEVERITY.index(worst):
|
|
1020
|
+
worst = status
|
|
1021
|
+
reasons.append(
|
|
1022
|
+
"roll-up over %d stories: worst gate_status %s" % (len(stories), worst)
|
|
1023
|
+
)
|
|
1024
|
+
return {
|
|
1025
|
+
"gate_status": worst,
|
|
1026
|
+
"p0_status": None,
|
|
1027
|
+
"p1_status": None,
|
|
1028
|
+
"overall_status": None,
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
|
|
511
1032
|
def evaluate(args: argparse.Namespace) -> dict:
|
|
512
1033
|
reasons: list[str] = []
|
|
513
1034
|
trace_output = Path(args.trace_output)
|
|
514
1035
|
|
|
515
|
-
|
|
1036
|
+
if getattr(args, "rollup", False):
|
|
1037
|
+
gate = rollup_gate(
|
|
1038
|
+
trace_output, Path(args.sprint_status), args.story, reasons
|
|
1039
|
+
)
|
|
1040
|
+
else:
|
|
1041
|
+
gate = load_gate(trace_output, reasons, getattr(args, "story", None))
|
|
516
1042
|
gate_status = (gate["gate_status"] or "NOT_EVALUATED").upper()
|
|
517
1043
|
verdict = GATE_VERDICT.get(gate_status, "escalate")
|
|
518
1044
|
if gate_status not in GATE_VERDICT:
|
|
@@ -540,7 +1066,30 @@ def evaluate(args: argparse.Namespace) -> dict:
|
|
|
540
1066
|
verdict, nfr_path, review_path, reasons
|
|
541
1067
|
)
|
|
542
1068
|
|
|
543
|
-
|
|
1069
|
+
# `is not None`, never truthiness: an explicitly-passed empty value is a
|
|
1070
|
+
# caller error to fail closed on (Path("") is never a file, so it caps),
|
|
1071
|
+
# not a silent way back to the pre-flag behaviour. Only true omission skips.
|
|
1072
|
+
tests_ran = getattr(args, "tests_ran", None)
|
|
1073
|
+
sweep_scope: str | None = None
|
|
1074
|
+
if tests_ran is not None:
|
|
1075
|
+
verdict, sweep_scope = apply_sweep_scope(verdict, Path(tests_ran), reasons)
|
|
1076
|
+
|
|
1077
|
+
# The delta cycle profile: a mid-loop gate that re-ran only the previously
|
|
1078
|
+
# failed assessor dimension(s) declares itself and is capped exactly as a
|
|
1079
|
+
# scoped sweep is - it may say "not yet, and here is why", never advance.
|
|
1080
|
+
# defer moves too, for the same reason as the sweep cap: defer's route
|
|
1081
|
+
# advances the story. The choices list means an unrecognised value is an
|
|
1082
|
+
# argparse error (exit 2), never a silently-full gate.
|
|
1083
|
+
cycle_profile = getattr(args, "cycle_profile", None)
|
|
1084
|
+
if cycle_profile == "delta" and verdict in ("advance", "defer"):
|
|
1085
|
+
reasons.append(
|
|
1086
|
+
f"{verdict} capped at reloop: a delta-profile gate re-ran only the "
|
|
1087
|
+
"previously failed dimension(s), so it may not move the story - "
|
|
1088
|
+
"run the full gate (every assessor, over a scope=full sweep) to advance"
|
|
1089
|
+
)
|
|
1090
|
+
verdict = "reloop"
|
|
1091
|
+
|
|
1092
|
+
result = {
|
|
544
1093
|
"verdict": verdict,
|
|
545
1094
|
"gate_status": gate_status,
|
|
546
1095
|
"p0_status": gate["p0_status"],
|
|
@@ -553,8 +1102,22 @@ def evaluate(args: argparse.Namespace) -> dict:
|
|
|
553
1102
|
# an advance that ANDed both production signals from one that skipped
|
|
554
1103
|
# them, and `nfr_status: null` alone does not say which.
|
|
555
1104
|
"epic_level": bool(getattr(args, "epic_level", False)),
|
|
556
|
-
"reasons": reasons,
|
|
557
1105
|
}
|
|
1106
|
+
if tests_ran is not None:
|
|
1107
|
+
# Present exactly when --tests-ran was supplied, absent otherwise - so
|
|
1108
|
+
# the printed shape of every pre-flag invocation is unchanged (the
|
|
1109
|
+
# stability policy classes a new key in a documented invocation's JSON
|
|
1110
|
+
# as a shape change; a caller only sees this one by opting in). The
|
|
1111
|
+
# value is the recognised scope= reading, or null when none was.
|
|
1112
|
+
result["sweep_scope"] = sweep_scope
|
|
1113
|
+
if cycle_profile is not None:
|
|
1114
|
+
# Same conditional-presence contract as sweep_scope, same reason.
|
|
1115
|
+
result["cycle_profile"] = cycle_profile
|
|
1116
|
+
# reasons goes last, after any conditional keys, so the printed object
|
|
1117
|
+
# reads verdict-first, evidence-last exactly as the documented examples
|
|
1118
|
+
# show. Key ORDER is cosmetic; the contract is the key SET.
|
|
1119
|
+
result["reasons"] = reasons
|
|
1120
|
+
return result
|
|
558
1121
|
|
|
559
1122
|
|
|
560
1123
|
def main(argv: list[str] | None = None) -> int:
|
|
@@ -566,8 +1129,45 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
566
1129
|
help="Current story id; scopes gate-file resolution to that story's "
|
|
567
1130
|
"artifacts in a shared multi-story trace dir. Omit for a single-story dir.",
|
|
568
1131
|
)
|
|
569
|
-
parser.add_argument(
|
|
570
|
-
|
|
1132
|
+
parser.add_argument(
|
|
1133
|
+
"--nfr",
|
|
1134
|
+
help="PATH to the story's nfr-assessment markdown (production only) — a file "
|
|
1135
|
+
"path, never a status word. The file must carry a literal "
|
|
1136
|
+
"'**Overall Status:** <PASS|CONCERNS|FAIL|NOT_ASSESSED>' line; a value "
|
|
1137
|
+
"passed here instead of a path is read as a missing file and treated as "
|
|
1138
|
+
"failing.",
|
|
1139
|
+
)
|
|
1140
|
+
parser.add_argument(
|
|
1141
|
+
"--test-review",
|
|
1142
|
+
help="PATH to the story's test-review markdown (production only) — a file "
|
|
1143
|
+
"path, never a score. The file must carry '**Quality Score**: N/100' "
|
|
1144
|
+
"(the /100 denominator is required) and a '**Recommendation**:' line; a "
|
|
1145
|
+
"value passed here instead of a path is read as a missing file and "
|
|
1146
|
+
"treated as failing.",
|
|
1147
|
+
)
|
|
1148
|
+
parser.add_argument(
|
|
1149
|
+
"--tests-ran",
|
|
1150
|
+
help="PATH to the story's .tests-ran-<story_id> marker (per-story gates "
|
|
1151
|
+
"only). The marker must carry a scope=<full|scoped> line naming which "
|
|
1152
|
+
"sweep form its green run was (execute.md steps 2/5 write it beside "
|
|
1153
|
+
"baseline=). scope=full passes the verdict through; scope=scoped - and, "
|
|
1154
|
+
"fail-closed, a missing marker, a missing scope= line, or an "
|
|
1155
|
+
"unrecognised value - caps advance/defer at reloop. Omitting the flag "
|
|
1156
|
+
"skips the check (invocations that predate the flag keep their "
|
|
1157
|
+
"verdicts); the skill instructions require it on every per-story gate. "
|
|
1158
|
+
"Cannot be combined with --epic-level.",
|
|
1159
|
+
)
|
|
1160
|
+
parser.add_argument(
|
|
1161
|
+
"--cycle-profile",
|
|
1162
|
+
choices=["full", "delta"],
|
|
1163
|
+
help="Which gate cycle this is (per-story gates only). 'delta' declares "
|
|
1164
|
+
"a mid-loop gate that re-ran in full only the previously failed "
|
|
1165
|
+
"assessor dimension(s), the others re-checking their prior artifact "
|
|
1166
|
+
"against the diff: its verdict is capped at reloop - advance/defer "
|
|
1167
|
+
"require the full gate. Omitting the flag is the full behaviour; the "
|
|
1168
|
+
"JSON carries cycle_profile exactly when the flag was supplied. Cannot "
|
|
1169
|
+
"be combined with --epic-level.",
|
|
1170
|
+
)
|
|
571
1171
|
parser.add_argument(
|
|
572
1172
|
"--epic-level",
|
|
573
1173
|
action="store_true",
|
|
@@ -575,8 +1175,31 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
575
1175
|
"NFR/test-review ANDs, which TEA writes per story and never per epic. "
|
|
576
1176
|
"Without it, --profile production requires both --nfr and --test-review.",
|
|
577
1177
|
)
|
|
1178
|
+
parser.add_argument(
|
|
1179
|
+
"--rollup",
|
|
1180
|
+
action="store_true",
|
|
1181
|
+
help="Compute the epic roll-up from the per-story gate artifacts and "
|
|
1182
|
+
"the sprint-status story set, instead of reading a single epic-named "
|
|
1183
|
+
"artifact. Requires --epic-level, --story <epic id> and "
|
|
1184
|
+
"--sprint-status. Fail-closed: any not-done story, and any done story "
|
|
1185
|
+
"with no resolvable gate artifact of its own, refuses to NOT_EVALUATED.",
|
|
1186
|
+
)
|
|
1187
|
+
parser.add_argument(
|
|
1188
|
+
"--sprint-status",
|
|
1189
|
+
help="PATH to sprint-status.yaml (rollup only): the authority on which "
|
|
1190
|
+
"stories the epic owes, read so the story set can never be a "
|
|
1191
|
+
"caller-typed list with a story quietly left off.",
|
|
1192
|
+
)
|
|
578
1193
|
args = parser.parse_args(argv)
|
|
579
1194
|
|
|
1195
|
+
if args.rollup and not (args.epic_level and args.story and args.sprint_status):
|
|
1196
|
+
parser.error(
|
|
1197
|
+
"--rollup is the epic roll-up computed from per-story artifacts: it "
|
|
1198
|
+
"requires --epic-level, --story <epic id> and --sprint-status"
|
|
1199
|
+
)
|
|
1200
|
+
if args.sprint_status and not args.rollup:
|
|
1201
|
+
parser.error("--sprint-status is only meaningful with --rollup")
|
|
1202
|
+
|
|
580
1203
|
# Mutually exclusive by meaning: --epic-level asserts there is no aggregate to
|
|
581
1204
|
# AND, so supplying one anyway is a contradiction, and the flag would win
|
|
582
1205
|
# silently - discarding a FAILING artifact the caller explicitly named.
|
|
@@ -589,6 +1212,28 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
589
1212
|
"per-story gate, or drop the paths for the epic roll-up"
|
|
590
1213
|
)
|
|
591
1214
|
|
|
1215
|
+
# Same lane as the hybrid above: the epic roll-up gates no sweep of its own
|
|
1216
|
+
# (every story proved its own scope before reaching `done`), so an
|
|
1217
|
+
# epic-level --tests-ran is a contradiction to refuse, never to evaluate.
|
|
1218
|
+
# `is not None`, not truthiness: --epic-level --tests-ran "" is the same
|
|
1219
|
+
# contradiction spelled with an empty value, not a way around the refusal.
|
|
1220
|
+
if args.epic_level and args.tests_ran is not None:
|
|
1221
|
+
parser.error(
|
|
1222
|
+
"--epic-level is the epic roll-up: it gates no sweep of its own, so "
|
|
1223
|
+
"it cannot be combined with --tests-ran; drop the flag for a "
|
|
1224
|
+
"per-story gate, or drop the path for the epic roll-up"
|
|
1225
|
+
)
|
|
1226
|
+
|
|
1227
|
+
# The roll-up is by definition the final full instrument over the per-story
|
|
1228
|
+
# record - a delta roll-up is a contradiction, and even an explicit 'full'
|
|
1229
|
+
# is per-story vocabulary the roll-up does not speak. Same refusal lane.
|
|
1230
|
+
if args.epic_level and args.cycle_profile is not None:
|
|
1231
|
+
parser.error(
|
|
1232
|
+
"--epic-level is the epic roll-up, always the full instrument: it "
|
|
1233
|
+
"cannot be combined with --cycle-profile; drop the flag for a "
|
|
1234
|
+
"per-story gate"
|
|
1235
|
+
)
|
|
1236
|
+
|
|
592
1237
|
result = evaluate(args)
|
|
593
1238
|
print(json.dumps(result, indent=2))
|
|
594
1239
|
return 0
|