bmad-module-skill-forge 2.1.0 → 3.0.0

package/src/shared/scripts/skf-merge-doc-urls.py

@@ -1,5 +1,5 @@
 # /// script
-# requires-python = ">=3.9"
+# requires-python = ">=3.11"
 # dependencies = []
 # ///
 """SKF Merge Doc URLs — merge corpora-seeded + detected docs for a skill brief.

package/src/shared/scripts/skf-names-present.py

@@ -0,0 +1,221 @@
+# /// script
+# requires-python = ">=3.11"
+# dependencies = []
+# ///
+"""SKF Names Present: the provenance entries whose export name a skill package never writes.
+
+skf-test-skill credits a provenance entry toward a stack's Export Coverage
+only when the package writes its `export_name`: reconcile-coverage.py
+(`load_doc_text`, `names_present`) joins the text of `SKILL.md` and of each
+`.md` file directly in `references/` (sorted, read as UTF-8, no subfolder)
+and looks for the name in it as a fixed string, case-sensitive, never inside
+a longer identifier: on a side where the name ends in a letter, digit, `_`
+or `$`, the next character may not be one of those, so `get` is not found in
+`target` or `getAll` (skf-test-skill's validate-inventory.py keeps that
+rule; NAME_CHARS here must stay the same). A name holding
+`::` (an impl-block method, `Type::method`) rolls up under its type and is
+never looked up, and an entry with no name is no cited contract.
+create-stack-skill (generate-output section 8) runs this helper on a
+compose-mode map before the commit, so a stack never commits a name
+skf-test-skill will not credit. The rule here is the scorer's:
+test/test-skf-stack-step-rules.py runs both on the same package.
+
+Usage:
+  uv run skf-names-present.py --provenance <provenance-map.json> \\
+      --skill-dir <dir> [--drop-absent]
+
+Output (stdout):
+  {
+    "checked": <int>,           # entries whose export_name was looked up
+    "skipped": <int>,           # entries with a `::` name or no name
+    "absent": [                 # in entries[] order
+      {"entry_index": <int>, "export_name": "<name>",
+       "source_library": "<library>" | null},
+      ...
+    ],
+    "dropped": [...],           # --drop-absent: the absent entries removed
+    "provenance_written": <bool>
+  }
+
+--drop-absent removes every absent entry from the map and rewrites it
+through skf-atomic-write.py beside this script (JSON indented by two
+spaces, non-ASCII kept, a final newline). `absent[]` still lists what the
+lookup found, and `dropped[]` repeats it. `entry_index` is the position in
+the map as it was read.
+
+Exit codes:
+  0  no entry is absent, or --drop-absent removed every absent entry
+  1  an entry is absent (absent[] lists it)
+  2  usage, file, JSON or write error (one line on stderr, no JSON)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+ATOMIC_WRITE_HELPER = "skf-atomic-write.py"
+ATOMIC_WRITE_TIMEOUT_SEC = 60
+METHOD_SEPARATOR = "::"
+# Characters that continue an identifier in the languages SKF documents (`$`
+# for JS/TS names such as `$state`): skf-test-skill's IDENT_CHARS.
+NAME_CHARS = r"\w$"
+_NAME_CHAR_RE = re.compile(f"[{NAME_CHARS}]")
+
+
+def load_doc_text(skill_dir: Path) -> str:
+    """The text skf-test-skill looks names up in: SKILL.md, then each .md file
+    directly in references/ in sorted order, joined by newlines. Raises
+    OSError, or UnicodeDecodeError for a file that is not UTF-8."""
+    parts = []
+    skill_md = skill_dir / "SKILL.md"
+    if skill_md.is_file():
+        parts.append(skill_md.read_text(encoding="utf-8"))
+    refs = skill_dir / "references"
+    if refs.is_dir():
+        for ref in sorted(refs.glob("*.md")):
+            parts.append(ref.read_text(encoding="utf-8"))
+    return "\n".join(parts)
+
+
+def name_written(name: str, doc_text: str) -> bool:
+    """True when `doc_text` holds `name` as skf-test-skill's scorer finds it."""
+    body = re.escape(name)
+    if _NAME_CHAR_RE.match(name[0]):
+        body = f"(?<![{NAME_CHARS}])" + body
+    if _NAME_CHAR_RE.match(name[-1]):
+        body += f"(?![{NAME_CHARS}])"
+    return re.search(body, doc_text) is not None
+
+
+def looked_up_name(entry: object) -> str | None:
+    """The name skf-test-skill looks up for an entry, or None when it looks
+    up none: no string name, or a `Type::method` name."""
+    if not isinstance(entry, dict):
+        return None
+    name = entry.get("export_name")
+    if not isinstance(name, str) or not name or METHOD_SEPARATOR in name:
+        return None
+    return name
+
+
+def find_absent(entries: list, doc_text: str) -> tuple[int, list[dict]]:
+    """(entries looked up, the absent ones in entries[] order)."""
+    checked, absent = 0, []
+    for index, entry in enumerate(entries):
+        name = looked_up_name(entry)
+        if name is None:
+            continue
+        checked += 1
+        if not name_written(name, doc_text):
+            library = entry.get("source_library")
+            absent.append({
+                "entry_index": index,
+                "export_name": name,
+                "source_library": library if isinstance(library, str) else None,
+            })
+    return checked, absent
+
+
+def atomic_write(target: Path, data: bytes) -> None:
+    """Write `data` to `target` through skf-atomic-write.py beside this
+    script. Raises OSError when the helper is missing or fails."""
+    helper = Path(__file__).resolve().parent / ATOMIC_WRITE_HELPER
+    if not helper.is_file():
+        raise OSError(f"{ATOMIC_WRITE_HELPER} not found beside {Path(__file__).name}; nothing written")
+    try:
+        result = subprocess.run(
+            [sys.executable, str(helper), "write", "--target", str(target)],
+            input=data,
+            capture_output=True,
+            timeout=ATOMIC_WRITE_TIMEOUT_SEC,
+            check=False,
+        )
+    except (OSError, subprocess.SubprocessError) as exc:
+        raise OSError(f"atomic write of {target} failed: {exc}") from exc
+    if result.returncode != 0:
+        detail = (result.stderr or b"").decode("utf-8", errors="replace").strip()
+        raise OSError(f"atomic write of {target} failed: {detail}")
+
+
+def _fail(message: str) -> int:
+    print(f"error: {' '.join(message.split())}", file=sys.stderr)
+    return 2
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="skf-names-present",
+        description=(
+            "List the provenance entries whose export_name the skill package "
+            "never writes, by skf-test-skill's Export Coverage rule "
+            "(SKILL.md and references/*.md, case-sensitive substring, `::` "
+            "names skipped); with --drop-absent, remove them from the map."
+        ),
+    )
+    parser.add_argument("--provenance", required=True, help="path to provenance-map.json")
+    parser.add_argument(
+        "--skill-dir",
+        required=True,
+        help="skill package folder holding the SKILL.md and references/ to search",
+    )
+    parser.add_argument(
+        "--drop-absent",
+        action="store_true",
+        help="remove every absent entry and rewrite the map through skf-atomic-write.py",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+    prov_path = Path(args.provenance)
+    skill_dir = Path(args.skill_dir)
+    if not prov_path.is_file():
+        return _fail(f"provenance map not found: {prov_path}")
+    # A wrong folder would read every name as absent, and --drop-absent
+    # would then empty the map: require the package's SKILL.md.
+    if not (skill_dir / "SKILL.md").is_file():
+        return _fail(f"skill dir has no SKILL.md: {skill_dir}")
+    try:
+        prov = json.loads(prov_path.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        return _fail(f"cannot read the provenance map {prov_path}: {exc}")
+    entries = prov.get("entries") if isinstance(prov, dict) else None
+    if not isinstance(entries, list):
+        return _fail(f"the provenance map {prov_path} has no entries[] list")
+    try:
+        doc_text = load_doc_text(skill_dir)
+    except (OSError, ValueError) as exc:
+        return _fail(f"cannot read the skill package {skill_dir}: {exc}")
+
+    checked, absent = find_absent(entries, doc_text)
+    dropped: list[dict] = []
+    if args.drop_absent and absent:
+        gone = {item["entry_index"] for item in absent}
+        prov["entries"] = [entry for index, entry in enumerate(entries) if index not in gone]
+        data = (json.dumps(prov, indent=2, ensure_ascii=False) + "\n").encode("utf-8")
+        try:
+            atomic_write(prov_path, data)
+        except OSError as exc:
+            return _fail(str(exc))
+        dropped = absent
+
+    result = {
+        "checked": checked,
+        "skipped": len(entries) - checked,
+        "absent": absent,
+        "dropped": dropped,
+        "provenance_written": bool(dropped),
+    }
+    # ASCII escapes: a Windows console's code page cannot encode every name.
+    sys.stdout.write(json.dumps(result, indent=2) + "\n")
+    return 0 if not absent or dropped else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/src/shared/scripts/skf-pair-intersect.py

@@ -1,15 +1,15 @@
 # /// script
-# requires-python = ">=3.10"
+# requires-python = ">=3.11"
 # dependencies = []
 # ///
 """SKF Pair Intersect — compute file-list intersections between library pairs.
 
-`detect-integrations.md` §1 ("File-list intersection fast path") asks the LLM
-to prune the N*(N-1)/2 pair-grep matrix by intersecting the per-library file
-lists that step 3 import-count extraction already produced. At N≈21 this
-collapses ~210 prescribed pair greps to a handful of non-empty-intersection
-pairs in typical codebases; the LLM only needs to run further grep passes
-against those.
+`detect-integrations.md` §1 prunes the N*(N-1)/2 library pairs by
+intersecting the per-library file lists that step 3's `skf-count-imports.py`
+run already produced. At N≈21 this collapses ~210 pairs to a handful of
+non-empty-intersection pairs in typical codebases, and each pair's `files`
+are its co-import files: §2 reads them, with their import lines from step
+3's counts, and greps nothing again.
 
 This script extracts that intersection from prose-orchestrated work to a
 deterministic helper. The LLM still composes the user-facing warning when
@@ -17,12 +17,23 @@ the Top-K cap kicks in — the script just flags `truncated: true` so the
 caller knows there's something to warn about.
 
 Subcommand:
-  intersect --libraries <json-file-or-'-'> [--top-k N]
-      Read a libraries JSON (either from <json-file> or stdin if '-') of the
-      shape:
-        [{"name": "<lib>", "files": ["<path>", ...]}, ...]
-      where each library entry lists the project files that import it (the
-      per-library file enumeration from detect-integrations.md step 3).
+  intersect (--libraries <json-file-or-'-'> | --counts <json-file-or-'-'>)
+            [--only a,b] [--top-k N]
+      Read the libraries (from the file, or stdin if '-') in one of two
+      shapes:
+        --libraries  [{"name": "<lib>", "files": ["<path>", ...]}, ...]
+                     where each library entry lists the project files that
+                     import it (`skf-count-imports.py count --format
+                     libraries` prints this list).
+        --counts     the `skf-count-imports.py count` envelope as it is
+                     (its default `full` format): each `dependencies[]`
+                     entry, then each `units[]` entry, is a library whose
+                     files are its `files[]` ({path, line}). Entries that
+                     share a name are one library, with the first line each
+                     file imports it on.
+      --only names the libraries to pair, comma-separated (trimmed, empty
+      items dropped); every other library is left out, and a name the input
+      does not hold has no files, so it joins no pair.
 
       Emit JSON:
         {
@@ -34,11 +45,15 @@ Subcommand:
           "truncated": false,
           "total_pairs": <full non-empty-intersection count before cap>
         }
+      With --counts, each `files` entry is instead
+        {"path": "<rel-path-forward-slash>", "line_a": N, "line_b": N}
+      where `line_a` and `line_b` are the first lines of that file that
+      import `a` and `b`: the co-import files of the pair, with their lines.
 
       Pairs are computed for each unordered pair (a, b) with a < b
       lexicographically; only non-empty intersections are emitted. The pairs
       list is sorted by intersection_count DESC, then by (a, b) ASC for
-      stable, reproducible ordering.
+      stable, reproducible ordering; the files of a pair are sorted by path.
 
       Top-K cap: default --top-k 20 (matches detect-integrations.md §1's
       "Top-K cap at 20 with explicit warning"). If the full count exceeds
@@ -49,11 +64,14 @@ CLI examples:
   uv run skf-pair-intersect.py intersect --libraries libs.json
   cat libs.json | uv run skf-pair-intersect.py intersect --libraries -
   uv run skf-pair-intersect.py intersect --libraries libs.json --top-k 50
+  uv run skf-count-imports.py count . --deps manifests.json \\
+    | uv run skf-pair-intersect.py intersect --counts - --only "react,zustand"
 
 Exit codes:
-  0  — operation succeeded (including: empty input → empty pairs list)
-  1  — user error (bad JSON, missing required `name`/`files` field on any
-       library entry, file not readable)
+  0  - operation succeeded (including: empty input → empty pairs list)
+  1  - user error (bad JSON, missing required `name`/`files` field on any
+       library entry, a --counts input that is not the count envelope or
+       a file entry that is not a {path, line} object, file not readable)
 """
 
 from __future__ import annotations
@@ -111,11 +129,62 @@ def validate_libraries(libraries: object) -> list[dict]:
     return normalized
 
 
+def libraries_from_counts(counts: object) -> list[dict]:
+    """The libraries of a `skf-count-imports.py count` envelope.
+
+    Each `dependencies[]` entry, then each `units[]` entry, becomes a
+    {name, files, lines} record: `lines` maps each forward-slash path to the
+    first line that imports the library. Entries that share a name merge,
+    keeping the lower line. Raises ValueError on any structural defect.
+    """
+    if not isinstance(counts, dict) or not isinstance(counts.get("dependencies"), list):
+        raise ValueError(
+            "counts input must be the skf-count-imports.py count envelope "
+            "(an object with a `dependencies` array)"
+        )
+    units = counts.get("units", [])
+    if not isinstance(units, list):
+        raise ValueError("counts `units` is not an array")
+    merged: dict[str, dict[str, int]] = {}
+    for idx, entry in enumerate(counts["dependencies"] + units):
+        if not isinstance(entry, dict):
+            raise ValueError(f"counts entry at index {idx} is not an object: {entry!r}")
+        name = entry.get("name")
+        if not isinstance(name, str) or not name:
+            raise ValueError(f"counts entry at index {idx} missing required `name` field")
+        files = entry.get("files")
+        if not isinstance(files, list):
+            raise ValueError(f"counts entry {name!r} missing required `files` array")
+        lines = merged.setdefault(name, {})
+        for f_idx, f in enumerate(files):
+            line = f.get("line") if isinstance(f, dict) else None
+            if (not isinstance(f, dict) or not isinstance(f.get("path"), str)
+                    or not isinstance(line, int) or isinstance(line, bool)):
+                raise ValueError(
+                    f"counts entry {name!r} files[{f_idx}] is not a {{path, line}} object: {f!r}"
+                )
+            path = f["path"].replace("\\", "/")
+            if path not in lines or line < lines[path]:
+                lines[path] = line
+    return [
+        {"name": name, "files": sorted(lines), "lines": lines}
+        for name, lines in merged.items()
+    ]
+
+
+def only(libraries: list[dict], names: list[str]) -> list[dict]:
+    """The libraries `names` lists; a listed name with no library adds nothing."""
+    keep = set(names)
+    return [lib for lib in libraries if lib["name"] in keep]
+
+
 def compute_pairs(libraries: list[dict]) -> list[dict]:
     """For each unordered (a, b) with a < b lexicographically, compute the
     file-list intersection. Emit only pairs with non-empty intersections.
 
     The returned list is sorted by intersection_count DESC, then (a, b) ASC.
+    When both libraries carry `lines` (the --counts input), each file is a
+    {path, line_a, line_b} object instead of a path.
     """
     # sort libraries by name for stable iteration order
     sorted_libs = sorted(libraries, key=lambda lib: lib["name"])
@@ -135,11 +204,17 @@ def compute_pairs(libraries: list[dict]) -> list[dict]:
             intersection = sorted(set_a & set_b)
             if not intersection:
                 continue
+            files: list = intersection
+            if "lines" in a and "lines" in b:
+                files = [
+                    {"path": path, "line_a": a["lines"][path], "line_b": b["lines"][path]}
+                    for path in intersection
+                ]
             pairs.append({
                 "a": a["name"],
                 "b": b["name"],
                 "intersection_count": len(intersection),
-                "files": intersection,
+                "files": files,
             })
     pairs.sort(key=lambda p: (-p["intersection_count"], p["a"], p["b"]))
     return pairs
@@ -167,34 +242,38 @@ def intersect(libraries: list[dict], top_k: int = DEFAULT_TOP_K) -> dict:
 # --------------------------------------------------------------------------
 
 
-def _read_libraries_source(source: str) -> object:
-    """Read libraries JSON from a file path or stdin (if source == '-')."""
+def _read_libraries_source(source: str, what: str = "libraries") -> object:
+    """Read the input JSON from a file path or stdin (if source == '-')."""
     if source == "-":
         try:
             text = sys.stdin.read()
         except OSError as exc:
-            raise ValueError(f"failed to read libraries JSON from stdin: {exc}") from exc
+            raise ValueError(f"failed to read {what} JSON from stdin: {exc}") from exc
     else:
         path = Path(source)
         if not path.is_file():
-            raise ValueError(f"libraries file not found: {path}")
+            raise ValueError(f"{what} file not found: {path}")
         try:
             text = path.read_text(encoding="utf-8")
         except OSError as exc:
-            raise ValueError(f"failed to read libraries file {path}: {exc}") from exc
+            raise ValueError(f"failed to read {what} file {path}: {exc}") from exc
     try:
         return json.loads(text)
     except json.JSONDecodeError as exc:
-        raise ValueError(f"malformed JSON in libraries input: {exc}") from exc
+        raise ValueError(f"malformed JSON in {what} input: {exc}") from exc
 
 
 def _cmd_intersect(args: argparse.Namespace) -> int:
     try:
-        raw = _read_libraries_source(args.libraries)
-        libraries = validate_libraries(raw)
+        if args.counts is not None:
+            libraries = libraries_from_counts(_read_libraries_source(args.counts, "counts"))
+        else:
+            libraries = validate_libraries(_read_libraries_source(args.libraries))
     except ValueError as exc:
         print(f"error: {exc}", file=sys.stderr)
         return 1
+    if args.only is not None:
+        libraries = only(libraries, [n for n in (part.strip() for part in args.only.split(",")) if n])
     if args.top_k < 0:
         print(f"error: --top-k must be >= 0; got {args.top_k}", file=sys.stderr)
         return 1
@@ -218,14 +297,26 @@ def _build_parser() -> argparse.ArgumentParser:
         "intersect",
         help="emit non-empty-intersection pairs as JSON, sorted DESC by count",
     )
-    p_int.add_argument(
+    source = p_int.add_mutually_exclusive_group(required=True)
+    source.add_argument(
         "--libraries",
-        required=True,
         help=(
             "path to libraries JSON, or '-' to read from stdin. "
             "Shape: [{\"name\": \"<lib>\", \"files\": [\"<path>\", ...]}, ...]"
         ),
     )
+    source.add_argument(
+        "--counts",
+        help=(
+            "path to a skf-count-imports.py count envelope, or '-' to read from "
+            "stdin; each pair's files then carry line_a and line_b"
+        ),
+    )
+    p_int.add_argument(
+        "--only",
+        metavar="NAMES",
+        help="comma-separated library names to pair; every other library is left out",
+    )
     p_int.add_argument(
         "--top-k",
         type=int,