@xenonbyte/req-2-plan 0.2.3 → 0.3.0

package/tools/workflow_cli/markdown.py

@@ -0,0 +1,160 @@
+"""Shared Markdown helpers used by gate and trace checks.
+
+The single source of truth for "read only the text outside fenced code
+blocks", so template/example snippets inside ``` ... ``` (or ~~~) fences
+do not register as real headings, trace IDs, or references.
+"""
+from __future__ import annotations
+
+import re
+
+# A fence opener/closer: up to 3 leading spaces, then 3+ backticks or tildes.
+_FENCE_MARKER_RE = re.compile(r"^[ \t]{0,3}(`{3,}|~{3,})")
+_READONLY_SECTION_RE = re.compile(
+    r"^[ \t]{0,3}(#{1,6})\s+"
+    r"(?:Upstream Summary|Project Context)\s+\(read-only\)\s*(?:#+\s*)?$",
+    re.IGNORECASE,
+)
+_READONLY_SECTION_END_RE = re.compile(
+    r"^[ \t]{0,3}<!--\s*/r2p-read-only\s*-->\s*$",
+    re.IGNORECASE,
+)
+
+
+def unfenced_markdown_lines(content: str):
+    """Yield (line, start, end) for lines outside Markdown fenced code blocks.
+
+    `start`/`end` are byte-less character offsets into the original `content`,
+    so callers can map a yielded line back to its position in the source.
+    """
+    fence_char = ""
+    fence_len = 0
+    offset = 0
+    for line in content.splitlines(keepends=True):
+        marker = _FENCE_MARKER_RE.match(line)
+        if fence_char:
+            if (
+                marker
+                and marker.group(1)[0] == fence_char
+                and len(marker.group(1)) >= fence_len
+                and not line[marker.end():].strip()
+            ):
+                fence_char = ""
+                fence_len = 0
+            offset += len(line)
+            continue
+
+        if marker:
+            fence_char = marker.group(1)[0]
+            fence_len = len(marker.group(1))
+            offset += len(line)
+            continue
+
+        start = offset
+        offset += len(line)
+        yield line, start, offset
+
+
+def unfenced_markdown_text(content: str) -> str:
+    return "".join(line for line, _, _ in unfenced_markdown_lines(content))
+
+
+def strip_readonly_sections(content: str) -> str:
+    """Remove seeded read-only Markdown sections before structural validation."""
+    lines = list(unfenced_markdown_lines(content))
+    headings = [
+        (start, level)
+        for line, start, _ in lines
+        if (level := heading_level(line)) is not None
+    ]
+    markers = [
+        (start, len(match.group(1)))
+        for line, start, _ in lines
+        if (match := _READONLY_SECTION_RE.match(line))
+    ]
+    removals: list[tuple[int, int]] = []
+    for marker_start, marker_level in markers:
+        next_marker_start = next(
+            (start for start, _ in markers if start > marker_start),
+            len(content),
+        )
+        # Seeded payloads can contain copied documents with their own #/##
+        # headings, so prefer the explicit terminator when the seed provides it.
+        explicit_end = next(
+            (
+                line_end
+                for line, start, line_end in lines
+                if marker_start < start < next_marker_start
+                and _READONLY_SECTION_END_RE.match(line)
+            ),
+            None,
+        )
+        if explicit_end is not None:
+            removals.append((marker_start, explicit_end))
+            continue
+
+        end = len(content)
+        for heading_start, heading_level_ in headings:
+            if heading_start <= marker_start:
+                continue
+            if heading_level_ <= marker_level:
+                end = heading_start
+                break
+        removals.append((marker_start, end))
+
+    if not removals:
+        return content
+
+    merged: list[tuple[int, int]] = []
+    for start, end in sorted(removals):
+        if not merged or start > merged[-1][1]:
+            merged.append((start, end))
+        else:
+            prev_start, prev_end = merged[-1]
+            merged[-1] = (prev_start, max(prev_end, end))
+
+    pieces: list[str] = []
+    cursor = 0
+    for start, end in merged:
+        pieces.append(content[cursor:start])
+        cursor = end
+    pieces.append(content[cursor:])
+    return "".join(pieces)
+
+
+def heading_level(line: str) -> int | None:
+    """ATX heading level (count of leading '#'), or None when not a heading."""
+    stripped = line.lstrip()
+    if not stripped.startswith("#"):
+        return None
+    return len(stripped) - len(stripped.lstrip("#"))
+
+
+def heading_bounded_bodies(content: str, is_start):
+    """Yield each section whose heading line satisfies `is_start(line)`.
+
+    A section runs from its heading to the next heading at the same or higher
+    level, so a later sibling section cannot bleed into it. Headings are
+    located outside fenced code; each yielded body is a slice of the original
+    `content` (fences within the body are preserved for the caller to handle).
+    """
+    lines = list(unfenced_markdown_lines(content))
+    starts = [
+        (start, level)
+        for line, start, _ in lines
+        if (level := heading_level(line)) is not None and is_start(line)
+    ]
+    headings = [
+        (start, level)
+        for line, start, _ in lines
+        if (level := heading_level(line)) is not None
+    ]
+    for start, level in starts:
+        end = len(content)
+        for heading_start, heading_level_ in headings:
+            if heading_start <= start:
+                continue
+            if heading_level_ <= level:
+                end = heading_start
+                break
+        yield content[start:end]

package/tools/workflow_cli/stage_schema.py

@@ -0,0 +1,57 @@
+import re
+
+from tools.workflow_cli.models import Stage, TierBase
+
+# Structural fields every PLAN-TASK anchor must carry (gate + trace share these).
+PLAN_TASK_FIELDS = (
+    "Spec References",
+    "Change Type",
+    "TDD Applicable",
+    "Files",
+    "Skeleton",
+    "Steps",
+    "Verification",
+)
+# Matches a line that opens one of those fields, e.g. "Files: src/a.py".
+PLAN_TASK_FIELD_RE = re.compile(r"^(" + "|".join(PLAN_TASK_FIELDS) + r"):")
+
+# Headings MUST stay byte-identical to any existing gate regex they overlap.
+# Notably "## External Documentation Checked" matches gates.py:_EXTERNAL_DOCS_RE.
+STAGE_SCHEMA: dict = {
+    Stage.REQUIREMENT_BRIEF: {
+        TierBase.LIGHT: ["## Goal", "## In-Scope", "## Out-of-Scope", "## Acceptance Criteria"],
+        TierBase.STANDARD: [
+            "## Goal", "## In-Scope", "## Out-of-Scope", "## Non-Goals",
+            "## Assumptions", "## Acceptance Criteria", "## Open Questions", "## Sources",
+        ],
+    },
+    Stage.RISK_DISCOVERY: {
+        TierBase.LIGHT: ["## Risks", "## Boundaries"],
+        TierBase.STANDARD: ["## Risks", "## Boundaries", "## Scope Overflow Risks", "## Mitigations"],
+    },
+    Stage.DESIGN: {
+        TierBase.LIGHT: ["## Design Summary", "## Chosen Design", "## SPEC Handoff"],
+        TierBase.STANDARD: [
+            "## Design Summary", "## Current Code Evidence", "## Requirements Coverage",
+            "## Options Considered", "## Chosen Design", "## Rollback",
+            "## Observability", "## SPEC Handoff",
+        ],
+    },
+    Stage.SPEC: {
+        TierBase.LIGHT: ["## Behavior Contracts", "## External Documentation Checked", "## PLAN Handoff"],
+        TierBase.STANDARD: [
+            "## Behavior Contracts", "## API / Data / Config Contracts",
+            "## External Documentation Checked", "## Test Matrix", "## Non-goals", "## PLAN Handoff",
+        ],
+    },
+    Stage.PLAN: {
+        # PLAN's substantive checks (task coverage, fields) live in R5, not R2.
+        TierBase.LIGHT: ["## Tasks"],
+        TierBase.STANDARD: ["## Tasks"],
+    },
+}
+
+
+def required_headings(stage: Stage, tier_base: TierBase) -> list[str]:
+    """Required top-level headings for a stage at a tier base; [] if unschema'd."""
+    return list(STAGE_SCHEMA.get(stage, {}).get(tier_base, []))

package/tools/workflow_cli/stage_templates.py

@@ -0,0 +1,55 @@
+"""Render STAGE_SCHEMA into per-stage, per-tier seed templates.
+
+The CLI writes these into the stage content file so the agent starts from a
+structured skeleton, not a blank page. Templates carry no semantic claims —
+only headings, required gate anchors, example ID shapes, and a static trace-table skeleton.
+"""
+from __future__ import annotations
+
+from tools.workflow_cli.models import Stage, TierBase
+from tools.workflow_cli.stage_schema import required_headings
+
+_TRACE_SKELETON = (
+    "## Trace\n"
+    "<!-- Map this stage's IDs to upstream/downstream. R3 derives & checks closure. -->\n"
+    "| This ID | Upstream | Status |\n"
+    "|---|---|---|\n"
+)
+
+
+_HEADING_BODY = {
+    (Stage.REQUIREMENT_BRIEF, "## In-Scope"): "- SCOPE-IN-001 <!-- fill in -->\n",
+    (Stage.REQUIREMENT_BRIEF, "## Out-of-Scope"): "- SCOPE-OUT-001 <!-- fill in -->\n",
+    (Stage.RISK_DISCOVERY, "## Risks"): "### RISK-SEC-001 <!-- fill in -->\nStatus: <!-- fill in -->\n",
+    (Stage.DESIGN, "## Chosen Design"): "### DES-ARCH-001 <!-- fill in -->\n",
+    (Stage.SPEC, "## Behavior Contracts"): "### SPEC-BEHAVIOR-001 <!-- fill in -->\n",
+    (Stage.PLAN, "## Tasks"): (
+        "### PLAN-TASK-001 <!-- fill in -->\n"
+        "Spec References: SPEC-BEHAVIOR-001\n"
+        "Change Type: modify\n"
+        "TDD Applicable: yes\n"
+        "Files:\n"
+        "- <!-- fill in -->\n"
+        "Skeleton:\n"
+        "```python\n"
+        "# <!-- fill in -->\n"
+        "```\n"
+        "Steps:\n"
+        "- [ ] <!-- fill in -->\n"
+        "Verification: <!-- fill in -->\n"
+    ),
+}
+
+
+def _body_for(stage: Stage, heading: str) -> str:
+    return _HEADING_BODY.get((stage, heading), "<!-- fill in -->\n")
+
+
+def template_for(stage: Stage, tier_base: TierBase) -> str:
+    headings = required_headings(stage, tier_base)
+    title = stage.value.replace("_", " ").title()
+    parts = [f"# {title}\n"]
+    for h in headings:
+        parts.append(f"{h}\n{_body_for(stage, h)}")
+    parts.append(_TRACE_SKELETON)
+    return "\n".join(parts)

package/tools/workflow_cli/trace.py

@@ -0,0 +1,224 @@
+"""Cross-stage trace coverage (R3). Derived from artifacts; no stored matrix."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from tools.workflow_cli.markdown import (
+    heading_bounded_bodies,
+    heading_level,
+    strip_readonly_sections,
+    unfenced_markdown_lines,
+    unfenced_markdown_text,
+)
+from tools.workflow_cli.models import STAGE_ARTIFACT_MAP, Stage
+from tools.workflow_cli.stage_schema import PLAN_TASK_FIELD_RE
+
+# REQ-AUTH-001 / RISK-SEC-001 / DES-AUTH-001 / SPEC-AUTH-001 / SCOPE-IN-001 / SCOPE-OUT-001 / PLAN-TASK-001
+_ID_RE = re.compile(r"(?:REQ|RISK|DES|SPEC)-[A-Z]+-\d+|SCOPE-(?:IN|OUT)-\d+|PLAN-TASK-\d+")
+_PLAN_TASK_HEADING_RE = re.compile(r"^###\s+PLAN-TASK-\d+\b")
+_NATIVE_HEADING_ID_PREFIXES: dict[Stage, tuple[str, ...]] = {
+    Stage.RAW_REQUIREMENT: ("REQ-",),
+    Stage.RISK_DISCOVERY: ("RISK-",),
+    Stage.DESIGN: ("DES-",),
+    Stage.SPEC: ("SPEC-",),
+    Stage.PLAN: ("PLAN-TASK-",),
+}
+
+
+@dataclass
+class TraceModel:
+    defined: dict = field(default_factory=dict)     # id -> stage value where first defined
+
+
+def _scope_ids_defined_in_brief(stage: Stage, content: str) -> set[str]:
+    """SCOPE-* ids are defined by bullet entries in the brief scope sections."""
+    if stage != Stage.REQUIREMENT_BRIEF:
+        return set()
+    ids: set[str] = set()
+    capture = False
+    capture_level = 0
+    for line, _, _ in unfenced_markdown_lines(content):
+        stripped = line.strip()
+        level = heading_level(line)
+        if stripped in {"## In-Scope", "## Out-of-Scope"}:
+            capture = True
+            capture_level = level or 0
+            continue
+        if capture and level is not None and level <= capture_level:
+            capture = False
+        if capture:
+            ids.update(m.group(0) for m in _ID_RE.finditer(line) if m.group(0).startswith("SCOPE-"))
+    return ids
+
+
+def _native_heading_ids(stage: Stage, content: str) -> set[str]:
+    prefixes = _NATIVE_HEADING_ID_PREFIXES.get(stage, ())
+    if not prefixes:
+        return set()
+    ids: set[str] = set()
+    for line, _, _ in unfenced_markdown_lines(content):
+        if line.lstrip().startswith("#"):
+            for match in _ID_RE.finditer(line):
+                id_ = match.group(0)
+                if id_.startswith(prefixes):
+                    ids.add(id_)
+    return ids
+
+
+def _artifact_text(run_dir: Path, stage: Stage) -> str:
+    path = run_dir / STAGE_ARTIFACT_MAP[stage]
+    return strip_readonly_sections(path.read_text(encoding="utf-8")) if path.exists() else ""
+
+
+def _plan_task_bodies(plan_content: str):
+    return heading_bounded_bodies(plan_content, _PLAN_TASK_HEADING_RE.match)
+
+
+def _find_plan_task_field(body: str, field: str):
+    field_re = re.compile(rf"^{re.escape(field)}:\s*(.*)$")
+    for line, start, _ in unfenced_markdown_lines(body):
+        m = field_re.match(line)
+        if m:
+            return m, start
+    return None
+
+
+def _find_next_plan_task_field_start(body: str, after: int) -> int | None:
+    for line, start, _ in unfenced_markdown_lines(body):
+        if start >= after and PLAN_TASK_FIELD_RE.match(line):
+            return start
+    return None
+
+
+def _plan_task_field_value(body: str, field: str) -> str:
+    found = _find_plan_task_field(body, field)
+    if found is None:
+        return ""
+    match, line_start = found
+    body_start = line_start + match.end()
+    next_start = _find_next_plan_task_field_start(body, body_start)
+    end = next_start if next_start is not None else len(body)
+    return f"{match.group(1)}\n{body[body_start:end]}".strip()
+
+
+def plan_consumed_spec_ids(run_dir: Path) -> set[str]:
+    """SPEC IDs consumed by PLAN-TASK Spec References fields, not merely mentioned."""
+    plan = _artifact_text(run_dir, Stage.PLAN)
+    consumed: set[str] = set()
+    for body in _plan_task_bodies(plan):
+        consumed.update(m.group(0) for m in _ID_RE.finditer(_plan_task_field_value(body, "Spec References"))
+                        if m.group(0).startswith("SPEC-"))
+    return consumed
+
+
+def _heading_blocks(content: str, id_pattern: str) -> dict[str, str]:
+    """Map each `id_pattern` ID defined in a heading to its section text.
+
+    A block runs from its heading to the next same-or-higher heading, so a
+    later sibling section cannot bleed its references into the block. Fenced
+    code is ignored.
+    """
+    content = unfenced_markdown_text(content)
+    starts = list(re.finditer(rf"(?m)^(#+)\s+.*?\b({id_pattern})\b", content))
+    headings = list(re.finditer(r"(?m)^(#+)\s+", content))
+    blocks: dict[str, str] = {}
+    for match in starts:
+        level = len(match.group(1))
+        end = len(content)
+        for heading in headings:
+            if heading.start() <= match.start():
+                continue
+            if len(heading.group(1)) <= level:
+                end = heading.start()
+                break
+        blocks[match.group(2)] = content[match.start():end]
+    return blocks
+
+
+def _spec_blocks(spec_content: str) -> dict[str, str]:
+    return _heading_blocks(spec_content, r"SPEC-[A-Z]+-\d+")
+
+
+def _risk_blocks(content: str) -> dict[str, str]:
+    return _heading_blocks(content, r"RISK-[A-Z]+-\d+")
+
+
+def scope_in_not_closed(run_dir: Path) -> list[str]:
+    """SCOPE-IN closes only when a PLAN-TASK carries it or consumes a SPEC carrying it."""
+    model = build_trace(run_dir)
+    plan_text = _artifact_text(run_dir, Stage.PLAN)
+    plan_task_text = "\n".join(unfenced_markdown_text(body) for body in _plan_task_bodies(plan_text))
+    spec_blocks = _spec_blocks(_artifact_text(run_dir, Stage.SPEC))
+    consumed_specs = plan_consumed_spec_ids(run_dir)
+    issues: list[str] = []
+    for id_ in sorted(i for i in model.defined if i.startswith("SCOPE-IN-")):
+        if id_ in plan_task_text:
+            continue
+        if any(id_ in spec_blocks.get(spec_id, "") for spec_id in consumed_specs):
+            continue
+        issues.append(id_)
+    return issues
+
+
+def risk_ids_not_closed(run_dir: Path) -> list[str]:
+    """RISK-* blocks must declare Status: mitigated|deferred|out_of_scope|out-of-scope.
+
+    Closure is read only from the risk definition block in RISK_DISCOVERY.
+    A block stops at the next same-or-higher Markdown heading, so unrelated
+    downstream Status fields cannot close an open risk.
+    """
+    model = build_trace(run_dir)
+    blocks = _risk_blocks(_artifact_text(run_dir, Stage.RISK_DISCOVERY))
+    open_risks: list[str] = []
+    for id_ in sorted(i for i in model.defined if i.startswith("RISK-")):
+        if not re.search(r"(?m)^Status:\s*(mitigated|deferred|out(?:_of_scope|-of-scope))\s*$", blocks.get(id_, "")):
+            open_risks.append(id_)
+    return open_risks
+
+
+def build_trace(run_dir: Path) -> TraceModel:
+    model = TraceModel()
+    for stage, filename in STAGE_ARTIFACT_MAP.items():
+        path = run_dir / filename
+        if not path.exists():
+            continue
+        content = _artifact_text(run_dir, stage)
+        heading_ids = _native_heading_ids(stage, content)
+        definition_ids = heading_ids | _scope_ids_defined_in_brief(stage, content)
+        for id_ in definition_ids:
+            model.defined.setdefault(id_, stage.value)
+    return model
+
+
+def spec_ids_not_consumed(run_dir: Path) -> list[str]:
+    """SPEC-* defined but not consumed by PLAN-TASK Spec References fields."""
+    model = build_trace(run_dir)
+    consumed = plan_consumed_spec_ids(run_dir)
+    return sorted(id_ for id_, _ in model.defined.items()
+                  if id_.startswith("SPEC-") and id_ not in consumed)
+
+
+def scope_out_violations(run_dir: Path) -> list[str]:
+    """SCOPE-OUT-* ids that executable PLAN-TASK bodies reference — a scope overflow (R8)."""
+    plan_text = _artifact_text(run_dir, Stage.PLAN)
+    plan_task_text = "\n".join(unfenced_markdown_text(body) for body in _plan_task_bodies(plan_text))
+    return sorted(
+        {
+            m.group(0)
+            for m in _ID_RE.finditer(plan_task_text)
+            if m.group(0).startswith("SCOPE-OUT-")
+        }
+    )
+
+
+def check_trace_closure(run_dir: Path) -> list[str]:
+    issues: list[str] = []
+    for id_ in spec_ids_not_consumed(run_dir):
+        issues.append(f"SPEC {id_} is not consumed by any PLAN-TASK (coverage gap).")
+    for id_ in scope_in_not_closed(run_dir):
+        issues.append(f"In-scope item {id_} is not carried into PLAN consumption (scope not closed).")
+    for id_ in risk_ids_not_closed(run_dir):
+        issues.append(f"Risk {id_} is not mitigated, deferred, or marked out-of-scope (risk not closed).")
+    return issues

package/tools/workflow_cli/version.py

@@ -1 +1 @@
-R2P_VERSION = "0.2.3"
+R2P_VERSION = "0.3.0"