bmad-module-skill-forge 1.9.0 → 2.0.1

package/src/shared/scripts/skf-comention-pairs.py

@@ -0,0 +1,350 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.10"
+# dependencies = []
+# ///
+"""SKF Co-mention Pairs — compute compose-mode co-mention pairs from an
+architecture document with the same precision guards the prompt used to run
+by hand.
+
+`detect-integrations.md` §2 (compose branch) asks the LLM to scan an
+architecture markdown document for skill-name co-mentions using three
+deterministic guards:
+
+  1. Word-boundary matching (`\\b{skill_name}\\b`, case-insensitive) — no
+     substring false positives (`react` must not match inside `reactive`).
+  2. Section filtering — paragraphs governed by an H1/H2 header that
+     normalises to one of {introduction, overview, glossary, table of
+     contents, references, appendix, index} are excluded, and heading text
+     itself is never a co-mention source.
+  3. Two-paragraph minimum — a pair (A, B) qualifies only when at least two
+     distinct body paragraphs each contain word-boundary matches for both A
+     and B.
+
+Given a fixed architecture doc and skill-name set the qualifying-pair set is
+fully determined, so this belongs in a deterministic helper rather than
+in-prompt regex/counting. It mirrors the co-import subprocess the code-mode
+branch already delegates and the `skf-pair-intersect.py` pre-pass. The LLM
+still does the semantic composition (integration-section authoring, tier
+inheritance, VS-verdict parsing) — the script only emits the qualifying pairs
+and their evidence paragraphs.
+
+Subcommand:
+  comention --doc <md-file-or-'-'> --skills <json-file-or-'-'>
+      --doc     path to the architecture markdown document, or '-' for stdin.
+      --skills  path to a JSON array of loaded skill names
+                (e.g. `["react", "express"]`), or '-' for stdin.
+                (--doc and --skills cannot both read from stdin.)
+
+      Emit JSON:
+        {
+          "pairs": [
+            {
+              "a": "<skill>", "b": "<skill>",
+              "paragraph_count": N,
+              "evidence": [{"header": "<governing-header-or-null>",
+                            "excerpt": "<collapsed paragraph text>"}, ...]
+            },
+            ...
+          ],
+          "excluded_section_count": M
+        }
+
+      Only unordered pairs (a < b lexicographically) with paragraph_count >= 2
+      are emitted. `pairs[]` is sorted by paragraph_count DESC, then (a, b)
+      ASC — stable, mirroring skf-pair-intersect.py. `evidence[]` is in
+      document order. `excluded_section_count` is the number of DISTINCT
+      excluded H1/H2 sections (by normalised header) that governed at least
+      one body paragraph — a diagnostic for the caller, not a gate.
+
+CLI examples:
+  uv run skf-comention-pairs.py comention --doc arch.md --skills skills.json
+  echo '["react","express"]' | uv run skf-comention-pairs.py comention \\
+      --doc arch.md --skills -
+
+Exit codes:
+  0  — operation succeeded (including: no qualifying pairs → empty list)
+  1  — user error (bad JSON, missing/unreadable file, both inputs on stdin,
+       malformed skills array)
+  2  — unexpected internal error
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+
+# Closed exclusion set — governing H1/H2 headers that normalise to one of
+# these drop their paragraphs from co-mention analysis (they typically
+# enumerate all libraries without describing an integration).
+EXCLUDED_HEADERS = frozenset({
+    "introduction",
+    "overview",
+    "glossary",
+    "table of contents",
+    "references",
+    "appendix",
+    "index",
+})
+
+# ATX header line: 1-6 leading hashes followed by at least one space/tab.
+# A line like `#foo` (no space) is NOT a header per CommonMark and is treated
+# as body text. Capture group 1 = hashes (level), group 2 = header text.
+_HEADER_RE = re.compile(r"^(#{1,6})[ \t]+(.*)$")
+
+_EXCERPT_LIMIT = 240
+
+
+class UserError(Exception):
+    """A caller-facing input error → exit code 1."""
+
+
+# --------------------------------------------------------------------------
+# Markdown parsing
+# --------------------------------------------------------------------------
+
+
+def _strip_closing_hashes(text: str) -> str:
+    """Remove a closed-ATX trailing run of hashes (e.g. `Overview ##`)."""
+    return re.sub(r"[ \t]*#+[ \t]*$", "", text)
+
+
+def normalize_header(text: str) -> str:
+    """Normalise a header for exclusion-set comparison: lowercase, collapse
+    internal whitespace, strip trailing punctuation."""
+    t = text.strip().lower()
+    t = re.sub(r"\s+", " ", t).strip()
+    t = t.rstrip(".,:;!?").strip()
+    return t
+
+
+def parse_body_paragraphs(doc_text: str) -> list[tuple[str | None, str]]:
+    """Split the document into body paragraphs, each tagged with the raw text
+    of its governing (nearest-preceding) H1/H2 header (or None).
+
+    Paragraphs are maximal runs of consecutive non-blank, non-header lines,
+    separated by blank lines. Header lines of ANY level are never emitted as
+    paragraphs (headings are not a co-mention source). Only H1/H2 headers
+    update the governing header; H3-H6 leave it unchanged.
+    """
+    paragraphs: list[tuple[str | None, str]] = []
+    governing: str | None = None
+    buf: list[str] = []
+
+    def _flush() -> None:
+        if buf:
+            text = "\n".join(buf).strip()
+            if text:
+                paragraphs.append((governing, text))
+            buf.clear()
+
+    for raw_line in doc_text.splitlines():
+        m = _HEADER_RE.match(raw_line)
+        if m is not None:
+            # A header terminates the current paragraph; the header line
+            # itself is not body content.
+            _flush()
+            level = len(m.group(1))
+            header_text = _strip_closing_hashes(m.group(2)).strip()
+            if level <= 2:
+                governing = header_text if header_text else None
+            continue
+        if raw_line.strip() == "":
+            _flush()
+            continue
+        buf.append(raw_line)
+    _flush()
+    return paragraphs
+
+
+# --------------------------------------------------------------------------
+# Co-mention analysis
+# --------------------------------------------------------------------------
+
+
+def _make_excerpt(text: str, limit: int = _EXCERPT_LIMIT) -> str:
+    """Collapse whitespace and truncate for a stable, compact evidence quote."""
+    collapsed = re.sub(r"\s+", " ", text).strip()
+    if len(collapsed) > limit:
+        return collapsed[: limit - 3].rstrip() + "..."
+    return collapsed
+
+
+def analyze(doc_text: str, skill_names: list[str]) -> dict:
+    """Compute qualifying co-mention pairs and their evidence.
+
+    Deterministic: same (doc_text, skill_names) → identical output.
+    """
+    names = sorted(set(skill_names))
+    patterns = {
+        n: re.compile(r"\b" + re.escape(n) + r"\b", re.IGNORECASE)
+        for n in names
+    }
+    paragraphs = parse_body_paragraphs(doc_text)
+
+    excluded_sections: set[str] = set()
+    # (a, b) -> list of evidence dicts, one per qualifying paragraph
+    pair_evidence: dict[tuple[str, str], list[dict]] = {}
+
+    for governing, text in paragraphs:
+        if governing is not None:
+            norm = normalize_header(governing)
+            if norm in EXCLUDED_HEADERS:
+                excluded_sections.add(norm)
+                continue
+        # names is sorted, so `present` is sorted → pairs are (a < b)
+        present = [n for n in names if patterns[n].search(text)]
+        if len(present) < 2:
+            continue
+        excerpt = _make_excerpt(text)
+        evidence_entry = {"header": governing, "excerpt": excerpt}
+        for i in range(len(present)):
+            for j in range(i + 1, len(present)):
+                key = (present[i], present[j])
+                pair_evidence.setdefault(key, []).append(evidence_entry)
+
+    pairs: list[dict] = []
+    for (a, b), evidence in pair_evidence.items():
+        if len(evidence) >= 2:
+            pairs.append({
+                "a": a,
+                "b": b,
+                "paragraph_count": len(evidence),
+                "evidence": evidence,
+            })
+    pairs.sort(key=lambda p: (-p["paragraph_count"], p["a"], p["b"]))
+
+    return {
+        "pairs": pairs,
+        "excluded_section_count": len(excluded_sections),
+    }
+
+
+# --------------------------------------------------------------------------
+# CLI
+# --------------------------------------------------------------------------
+
+
+def _read_source(source: str, label: str) -> str:
+    """Read text from a file path or stdin (if source == '-')."""
+    if source == "-":
+        try:
+            return sys.stdin.read()
+        except OSError as exc:
+            raise UserError(f"failed to read {label} from stdin: {exc}") from exc
+    path = Path(source)
+    if not path.is_file():
+        raise UserError(f"{label} file not found: {path}")
+    try:
+        return path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise UserError(f"failed to read {label} file {path}: {exc}") from exc
+
+
+def parse_skills(raw_text: str) -> list[str]:
+    """Parse and validate the --skills JSON array of non-empty strings."""
+    try:
+        data = json.loads(raw_text)
+    except json.JSONDecodeError as exc:
+        raise UserError(f"malformed JSON in --skills input: {exc}") from exc
+    if not isinstance(data, list):
+        raise UserError(
+            "--skills input must be a JSON array of strings; "
+            f"got {type(data).__name__}"
+        )
+    names: list[str] = []
+    for idx, item in enumerate(data):
+        if not isinstance(item, str) or not item:
+            raise UserError(
+                f"--skills[{idx}] must be a non-empty string; got {item!r}"
+            )
+        names.append(item)
+    return names
+
+
+def _cmd_comention(args: argparse.Namespace) -> int:
+    if args.doc == "-" and args.skills == "-":
+        raise UserError(
+            "--doc and --skills cannot both read from stdin ('-')"
+        )
+    # Read the file-backed input first so a single stdin source stays intact.
+    if args.doc == "-":
+        skills_text = _read_source(args.skills, "--skills")
+        doc_text = _read_source(args.doc, "--doc")
+    else:
+        doc_text = _read_source(args.doc, "--doc")
+        skills_text = _read_source(args.skills, "--skills")
+    skill_names = parse_skills(skills_text)
+
+    result = analyze(doc_text, skill_names)
+    if args.verbose:
+        print(
+            f"analyzed {len(set(skill_names))} distinct skill names; "
+            f"{len(result['pairs'])} qualifying pairs; "
+            f"{result['excluded_section_count']} excluded sections",
+            file=sys.stderr,
+        )
+    json.dump(result, sys.stdout, indent=2)
+    sys.stdout.write("\n")
+    return 0
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="skf-comention-pairs",
+        description=(
+            "Compute compose-mode co-mention pairs from an architecture "
+            "document (detect-integrations.md §2 compose branch)."
+        ),
+    )
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    p_cm = sub.add_parser(
+        "comention",
+        help=(
+            "emit qualifying co-mention pairs (>=2 body paragraphs, "
+            "word-boundary, section-filtered) as JSON"
+        ),
+    )
+    p_cm.add_argument(
+        "--doc",
+        required=True,
+        help="path to the architecture markdown document, or '-' for stdin",
+    )
+    p_cm.add_argument(
+        "--skills",
+        required=True,
+        help=(
+            "path to a JSON array of loaded skill names, or '-' for stdin. "
+            "Shape: [\"<skill>\", ...]"
+        ),
+    )
+    p_cm.add_argument(
+        "--verbose",
+        action="store_true",
+        help="print a one-line summary to stderr",
+    )
+    p_cm.set_defaults(func=_cmd_comention)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return args.func(args)
+    except UserError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+    except Exception as exc:  # noqa: BLE001 — last-resort guard → exit 2
+        print(f"internal error: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/src/shared/scripts/skf-count-tokens.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.10"
+# ///
+"""SKF Count Tokens — Deterministic per-artifact token/word metrics for a skill package.
+
+Pre-pass that measures the exported artifacts of a single skill package and the
+shared managed section, so the export skill's token report (references/token-report.md)
+and snippet token check (references/generate-snippet.md §4) render exact numbers from
+JSON instead of re-reading every reference file to word-count it in-prompt.
+
+Metrics:
+  - words:  whitespace-split word count (`len(text.split())`) — the report's "Words" column
+  - tokens: char-over-four estimate (`len(text) // 4`) — the SKF-wide convention already
+            used by skf-validate-output.py, so every SKF token number agrees
+
+Measured artifacts (all relative to the positional skill-package dir):
+  - context-snippet.md   — the compressed passive-context snippet (if present)
+  - SKILL.md             — the active skill document
+  - metadata.json        — machine-readable metadata
+  - references/*         — every file under references/ (recursively), summed as references_total
+  - managed section      — the complete <!-- SKF:BEGIN ... --> ... <!-- SKF:END --> block,
+                           extracted from the first --target-file that contains one
+                           (shared across the batch; measured once)
+
+package_total sums context-snippet.md + SKILL.md + metadata.json + references_total and
+DELIBERATELY EXCLUDES the managed-section row — the managed section is a shared all-skills
+cost reported separately, and including it would double-count this skill's own snippet
+(see references/token-report.md §1).
+
+CLI:
+  python3 skf-count-tokens.py <skill-package-dir>
+  python3 skf-count-tokens.py <skill-package-dir> --target-file CLAUDE.md
+  python3 skf-count-tokens.py <skill-package-dir> --target-file CLAUDE.md --target-file AGENTS.md
+  python3 skf-count-tokens.py <skill-package-dir> -o report.json
+
+Exit codes: 0 = metrics emitted, 2 = error (skill package dir missing / not a directory).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+# Mirrors skf-rebuild-managed-sections.py: a BEGIN marker may carry attributes
+# (e.g. `updated:2026-07-13`) before its closing `-->`; the END marker is fixed.
+MANAGED_SECTION_RE = re.compile(
+    r"<!-- SKF:BEGIN[^>]*-->.*?<!-- SKF:END -->",
+    re.DOTALL,
+)
+
+# Core per-skill artifacts, in report order. (references/ is handled separately.)
+CORE_ARTIFACTS = [
+    ("context-snippet.md", "context-snippet"),
+    ("SKILL.md", "skill-md"),
+    ("metadata.json", "metadata"),
+]
+
+
+def estimate_tokens(text: str) -> int:
+    """Char-over-four token estimate — the SKF-wide convention (see skf-validate-output.py)."""
+    return len(text) // 4
+
+
+def count_words(text: str) -> int:
+    """Whitespace-split word count — the token report's 'Words' column."""
+    return len(text.split())
+
+
+def measure(text: str) -> dict:
+    """Return the {words, tokens} metrics for a piece of text."""
+    return {"words": count_words(text), "tokens": estimate_tokens(text)}
+
+
+def _read_text(path: Path) -> str:
+    # encoding is pinned to utf-8 so non-ASCII content is not mojibaked on
+    # Windows (cp1252 default).
+    return path.read_text(encoding="utf-8")
+
+
+def extract_managed_section(text: str) -> str | None:
+    """Return the complete BEGIN..END managed-section span (markers included), or None."""
+    m = MANAGED_SECTION_RE.search(text)
+    return m.group(0) if m else None
+
+
+def measure_reference_files(references_dir: Path) -> tuple[list[dict], dict]:
+    """Measure every file under references/ (recursively, sorted). Returns (per-file rows, total)."""
+    rows: list[dict] = []
+    total_words = 0
+    total_tokens = 0
+    if references_dir.is_dir():
+        for path in sorted(references_dir.rglob("*"), key=lambda p: p.as_posix()):
+            if not path.is_file():
+                continue
+            text = _read_text(path)
+            metrics = measure(text)
+            rows.append(
+                {
+                    "path": path.relative_to(references_dir.parent).as_posix(),
+                    "role": "reference",
+                    "exists": True,
+                    **metrics,
+                }
+            )
+            total_words += metrics["words"]
+            total_tokens += metrics["tokens"]
+    total = {"count": len(rows), "words": total_words, "tokens": total_tokens}
+    return rows, total
+
+
+def count_package(skill_dir: Path, target_files: list[Path] | None = None) -> dict:
+    """Measure a skill package's artifacts and the shared managed section.
+
+    Returns the metrics dict (see module docstring for the shape). Missing core
+    artifacts are reported with exists=False and zero metrics so package_total is
+    always well-defined and callers can render 'N/A' rows without guessing.
+    """
+    skill_dir = Path(skill_dir)
+    target_files = target_files or []
+
+    files: list[dict] = []
+
+    # Core per-skill artifacts.
+    for name, role in CORE_ARTIFACTS:
+        path = skill_dir / name
+        if path.is_file():
+            metrics = measure(_read_text(path))
+            files.append({"path": name, "role": role, "exists": True, **metrics})
+        else:
+            files.append({"path": name, "role": role, "exists": False, "words": 0, "tokens": 0})
+
+    # references/ — per-file rows + summed total.
+    ref_rows, references_total = measure_reference_files(skill_dir / "references")
+    files.extend(ref_rows)
+
+    # package_total = snippet + SKILL.md + metadata + references_total; EXCLUDES managed section.
+    core_by_role = {row["role"]: row for row in files if row["role"] in {"context-snippet", "skill-md", "metadata"}}
+    package_total = {
+        "words": sum(core_by_role[r]["words"] for r in core_by_role) + references_total["words"],
+        "tokens": sum(core_by_role[r]["tokens"] for r in core_by_role) + references_total["tokens"],
+    }
+
+    # Managed section — first --target-file that actually contains a block wins.
+    managed_section = {"present": False, "source_file": None, "words": 0, "tokens": 0}
+    for tf in target_files:
+        tf = Path(tf)
+        if not tf.is_file():
+            continue
+        section = extract_managed_section(_read_text(tf))
+        if section is not None:
+            managed_section = {
+                "present": True,
+                "source_file": tf.as_posix(),
+                **measure(section),
+            }
+            break
+
+    return {
+        "status": "ok",
+        "skill_package": skill_dir.as_posix(),
+        "files": files,
+        "references_total": references_total,
+        "managed_section": managed_section,
+        "package_total": package_total,
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="skf-count-tokens.py",
+        description=(
+            "Deterministic per-artifact word/token metrics for a skill package. Emits JSON "
+            "with per-file rows, references_total, the shared managed-section block, and a "
+            "package_total that excludes the managed section (see references/token-report.md §1). "
+            "tokens = len(text)//4 (SKF-wide convention); words = whitespace-split count."
+        ),
+    )
+    parser.add_argument(
+        "skill_package",
+        help="Resolved skill-package directory (holds context-snippet.md, SKILL.md, metadata.json, references/)",
+    )
+    parser.add_argument(
+        "--target-file",
+        action="append",
+        default=[],
+        metavar="PATH",
+        help="Context file (CLAUDE.md / AGENTS.md / .cursorrules) to extract the managed "
+        "section from; repeatable — the first file with a BEGIN..END block is measured.",
+    )
+    parser.add_argument("-o", "--output", metavar="PATH", help="Write JSON here (default: stdout)")
+    parser.add_argument("--verbose", action="store_true", help="Emit diagnostics to stderr")
+    args = parser.parse_args(argv)
+
+    skill_dir = Path(args.skill_package)
+    if not skill_dir.is_dir():
+        print(f"error: skill package directory not found: {skill_dir}", file=sys.stderr)
+        return 2
+
+    if args.verbose:
+        print(f"measuring skill package: {skill_dir}", file=sys.stderr)
+        if args.target_file:
+            print(f"managed-section candidates: {args.target_file}", file=sys.stderr)
+
+    result = count_package(skill_dir, [Path(p) for p in args.target_file])
+
+    payload = json.dumps(result, indent=2)
+    if args.output:
+        Path(args.output).write_text(payload + "\n", encoding="utf-8")
+        if args.verbose:
+            print(f"wrote {args.output}", file=sys.stderr)
+    else:
+        print(payload)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/src/shared/scripts/skf-derive-assembly-shape.py

@@ -0,0 +1,107 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = ["pyyaml"]
+# ///
+"""SKF Derive Assembly Shape — decide whether a brief is a whole-language reference.
+
+create-skill's SKILL.md assembly is mostly LLM-driven prose, but the BRANCH
+SELECTION — does this skill get the whole-language "Language Guide" treatment
+(prose foregrounded, compiler internals demoted) or the standard
+library-export layout? — must be deterministic. This helper computes that one
+decision from the brief so the assembler (and the doc-fetch step) branch on a
+machine value instead of grepping free-text `scope.notes`.
+
+The gate is a single, structured, schema-validated condition: the brief's
+`doc_urls` contains at least one entry with `source: "language-registry"`
+(issue #432). Registry-sourced corpora are stamped ONLY by
+skf-language-corpora.py, which is invoked ONLY for a whole-language reference
+(a grammar_file:/tree_triad: signal) in step-auto-scope.md §6b. So the gate
+fires exactly when there is registry-guaranteed prose to foreground, and never
+for an ordinary library, a parser library (pest), a component-library, a
+reference-app, or a docs-only brief — none of which carry a language-registry
+entry. A registry-miss (N==0) whole-language reference carries no such entry
+and is intentionally NOT gated: there is no canonical prose to foreground, so
+it keeps the standard layout (and its scope.notes DEGRADED caveat stands).
+
+Gating on the structured `source` field — not the operator-editable
+`scope.notes` substring — means a hand-edited or interactive brief cannot
+silently disable (or misfire) the override.
+
+CLI:
+  uv run skf-derive-assembly-shape.py <path/to/skill-brief.yaml>
+  cat brief.yaml | uv run skf-derive-assembly-shape.py -
+
+Output (JSON on stdout):
+  {
+    "assembly_shape": "whole-language-reference" | "standard",
+    "gate_signal":    "language-registry" | null
+  }
+
+Exit codes:
+  0  success (shape emitted)
+  2  error (missing/unreadable/invalid brief)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def derive_assembly_shape(brief: dict[str, Any]) -> dict[str, Any]:
+    """Return the assembly-shape decision for a parsed brief."""
+    doc_urls = brief.get("doc_urls")
+    if isinstance(doc_urls, list):
+        for entry in doc_urls:
+            if isinstance(entry, dict) and entry.get("source") == "language-registry":
+                return {
+                    "assembly_shape": "whole-language-reference",
+                    "gate_signal": "language-registry",
+                }
+    return {"assembly_shape": "standard", "gate_signal": None}
+
+
+def _die(message: str) -> None:
+    json.dump({"error": message}, sys.stderr)
+    sys.stderr.write("\n")
+    sys.exit(2)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="skf-derive-assembly-shape",
+        description="Decide whether a skill brief is a whole-language reference.",
+    )
+    parser.add_argument(
+        "path",
+        help="path to skill-brief.yaml; pass `-` to read YAML from stdin",
+    )
+    args = parser.parse_args(argv)
+
+    if args.path == "-":
+        text = sys.stdin.read()
+    else:
+        p = Path(args.path)
+        if not p.is_file():
+            _die(f"Brief not found at {p.as_posix()}")
+        text = p.read_text(encoding="utf-8")
+
+    try:
+        brief = yaml.safe_load(text)
+    except yaml.YAMLError as exc:
+        _die(f"Brief is not valid YAML: {exc}")
+    if not isinstance(brief, dict):
+        _die("Brief must be a YAML mapping")
+
+    json.dump(derive_assembly_shape(brief), sys.stdout)
+    sys.stdout.write("\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())