bmad-module-skill-forge 2.2.0 → 4.0.0

package/src/shared/scripts/skf-merge-doc-urls.py

@@ -1,18 +1,25 @@
 # /// script
-# requires-python = ">=3.9"
-# dependencies = []
+# requires-python = ">=3.11"
+# dependencies = ["pyyaml"]
 # ///
 """SKF Merge Doc URLs — merge corpora-seeded + detected docs for a skill brief.
 
 Deterministic merge of an upstream brief's `doc_urls` (registry-seeded
 companion corpora, source: language-registry) with the docs freshly
 discovered by `skf-detect-docs.py` (mapped to the brief `{url, label, source}`
-contract). Replaces the prose dedup/suppression that lived inline in
-`skf-brief-skill/references/step-auto-brief.md` §3 — that logic is fiddly
+contract). Replaces the prose mapping, dedup and suppression that lived inline
+in `skf-brief-skill/references/step-auto-brief.md` §3. That logic is fiddly
 (normalized-URL collisions, non-corpus path segments, locale duplicates) and
 must behave identically on every run, so it belongs in a tested script.
 
-Three operations, in order:
+With --detected-file, each skf-detect-docs.py entry {url, detected_via,
+content_type} is first mapped to {url, label, source}: `source` from
+`detected_via` and `label` from `content_type`, by the two tables below (a
+content_type they do not list gives the label ""). An entry whose
+`detected_via` they do not list is never merged: `suppressed` lists it with
+reason unknown-detected-via.
+
+Three merge operations, in order:
 
   1. Normalized-URL dedup — lowercase host, strip a trailing `/index.html`
      and any trailing `/` before comparing, so a seeded `…/book/` and a
@@ -41,10 +48,18 @@ DEGRADED case is intentionally out of scope (there is no canonical corpus host
 to anchor suppression against).
 
 CLI:
+  uv run src/shared/scripts/skf-merge-doc-urls.py --scope-type <type> \
+      --detected-file <detect-docs.json> --existing-brief <skill-brief.yaml>
   echo '{"scope_type": "...", "existing": [...], "detected": [...]}' \
     | uv run src/shared/scripts/skf-merge-doc-urls.py
 
-Input (JSON object on stdin):
+Files form (stdin is not read): --detected-file holds skf-detect-docs.py's
+JSON array, mapped as above; --existing-brief is a skill-brief.yaml whose
+`doc_urls` are the existing entries (none when it has no doc_urls), read as
+skf-resolve-authoritative-files.py reads a brief; --scope-type is the scope
+type. Each flag is optional: an absent one gives no entries (or "").
+
+Input of the stdin form (a JSON object), whose entries are merged as given:
   {
     "scope_type": "full-library",
     "existing":   [{"url": "...", "label": "...", "source": "language-registry"}, ...],
@@ -54,22 +69,42 @@ Input (JSON object on stdin):
 Output (JSON object on stdout):
   {
     "doc_urls":   [{"url": "...", "label": "...", "source": "..."}, ...],
-    "suppressed": [{"url": "...", "reason": "non-corpus-segment|non-primary-locale-dup"}, ...]
+    "suppressed": [{"url": "...", "reason":
+                    "unknown-detected-via|non-corpus-segment|non-primary-locale-dup"}, ...]
   }
 
 Exit codes:
   0  success (merged list emitted)
-  2  error (bad JSON, wrong input shape)
+  2  error (bad JSON, wrong input shape, a file that cannot be read, a brief
+     that is not a YAML mapping)
 """
 
 from __future__ import annotations
 
+import argparse
+import importlib.util
 import json
 import re
 import sys
+from pathlib import Path
 from typing import Any
 from urllib.parse import urlsplit
 
+# skf-detect-docs.py's `detected_via` -> the brief's coarse provenance, the
+# skill-brief.v1.json doc_urls[].source enum (a test pins it to the schema).
+_SOURCE_BY_DETECTED_VIA = {
+    "homepageUrl": "homepage",
+    "readme_link": "readme-detection",
+    "pages_api": "pages-api",
+    "docs_folder": "docs-folder",
+}
+# skf-detect-docs.py's `content_type` -> the entry's display label.
+_LABEL_BY_CONTENT_TYPE = {
+    "api-docs": "API Documentation",
+    "guide": "Guide",
+    "reference": "Reference",
+}
+
 # Path components (case-insensitive) that mark a non-corpus page on a doc host:
 # changelogs/news, contribution guides, wikis. Matched as whole path components
 # (or component + a separator), never as a bare substring.
@@ -153,6 +188,28 @@ def _matches_non_corpus_segment(path: str) -> bool:
     return False
 
 
+def map_detected(raw: list[Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    """skf-detect-docs.py entries as brief doc_urls entries {url, label, source}.
+
+    Returns (mapped, suppressed): an entry whose detected_via maps to no
+    source is suppressed with reason unknown-detected-via; one with no url is
+    skipped, as the merge skips it.
+    """
+    mapped: list[dict[str, Any]] = []
+    suppressed: list[dict[str, Any]] = []
+    for d in raw:
+        if not isinstance(d, dict) or not d.get("url"):
+            continue
+        via, content_type = d.get("detected_via"), d.get("content_type")
+        source = _SOURCE_BY_DETECTED_VIA.get(via) if isinstance(via, str) else None
+        if source is None:
+            suppressed.append({"url": d["url"], "reason": "unknown-detected-via"})
+            continue
+        label = _LABEL_BY_CONTENT_TYPE.get(content_type, "") if isinstance(content_type, str) else ""
+        mapped.append({"url": d["url"], "label": label, "source": source})
+    return mapped, suppressed
+
+
 def merge_doc_urls(
     scope_type: str,
     existing: list[dict[str, Any]],
@@ -232,30 +289,96 @@ def _force_utf8(*streams) -> None:
                 stream.reconfigure(encoding="utf-8", errors=errors)
 
 
-def main() -> int:
-    _force_utf8(sys.stdin, sys.stdout)
+class InputError(Exception):
+    """An input the merge cannot read (exit 2)."""
+
+
+def _read_detected(path: str) -> list[Any]:
+    """The JSON array skf-detect-docs.py wrote to `path`."""
+    try:
+        data = json.loads(Path(path).read_bytes().decode("utf-8-sig"))
+    except OSError as exc:
+        raise InputError(f"cannot read --detected-file {path}: {exc.strerror or exc}") from exc
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise InputError(f"--detected-file {path} is not JSON: {exc}") from exc
+    if not isinstance(data, list):
+        raise InputError(f"--detected-file {path} must hold a JSON array (skf-detect-docs.py output)")
+    return data
+
+
+def _brief_doc_urls(path: str) -> list[Any]:
+    """The doc_urls of the skill brief at `path`, read by
+    skf-resolve-authoritative-files.py's parse_brief_yaml (beside this file)."""
+    resolver_path = Path(__file__).resolve().parent / "skf-resolve-authoritative-files.py"
+    spec = importlib.util.spec_from_file_location("skf_resolve_authoritative_files", resolver_path)
+    if spec is None or spec.loader is None:
+        raise InputError(f"cannot load {resolver_path.name} beside {Path(__file__).name}")
+    resolver = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(resolver)
+    except (OSError, ImportError, SyntaxError) as exc:
+        raise InputError(f"cannot load {resolver_path.name} beside {Path(__file__).name}: {exc}") from exc
+    try:
+        brief = resolver.parse_brief_yaml(Path(path).read_bytes().decode("utf-8-sig"))
+    except OSError as exc:
+        raise InputError(f"cannot read --existing-brief {path}: {exc.strerror or exc}") from exc
+    except (UnicodeDecodeError, resolver.yaml.YAMLError) as exc:
+        raise InputError(f"--existing-brief {path} is not YAML: {exc}") from exc
+    if not isinstance(brief, dict):
+        raise InputError(f"--existing-brief {path} holds no brief mapping")
+    doc_urls = brief.get("doc_urls") or []
+    if not isinstance(doc_urls, list):
+        raise InputError(f"the doc_urls of --existing-brief {path} must be a list")
+    return doc_urls
+
+
+def _stdin_payload() -> tuple[str, list[Any], list[Any]]:
+    """(scope_type, existing, detected) of the stdin form."""
     raw = sys.stdin.read()
     if not raw or not raw.strip():
-        sys.stderr.write("skf-merge-doc-urls: empty stdin (expected JSON object)\n")
-        return 2
+        raise InputError("empty stdin (expected JSON object)")
     try:
         payload = json.loads(raw)
     except json.JSONDecodeError as exc:
-        sys.stderr.write(f"skf-merge-doc-urls: invalid JSON on stdin: {exc}\n")
-        return 2
+        raise InputError(f"invalid JSON on stdin: {exc}") from exc
     if not isinstance(payload, dict):
-        sys.stderr.write("skf-merge-doc-urls: input must be a JSON object\n")
-        return 2
-
-    scope_type = payload.get("scope_type") or ""
+        raise InputError("input must be a JSON object")
     existing = payload.get("existing") or []
     detected = payload.get("detected") or []
     if not isinstance(existing, list) or not isinstance(detected, list):
-        sys.stderr.write("skf-merge-doc-urls: 'existing' and 'detected' must be arrays\n")
+        raise InputError("'existing' and 'detected' must be arrays")
+    return payload.get("scope_type") or "", existing, detected
+
+
+def main(argv: list[str] | None = None) -> int:
+    _force_utf8(sys.stdin, sys.stdout)
+    parser = argparse.ArgumentParser(
+        prog="skf-merge-doc-urls",
+        description="Merge a brief's doc_urls with the docs skf-detect-docs.py found. With no flag, read the "
+                    "{scope_type, existing, detected} JSON object on stdin.",
+    )
+    parser.add_argument("--detected-file", metavar="PATH",
+                        help="skf-detect-docs.py's JSON array, mapped to {url, label, source}")
+    parser.add_argument("--existing-brief", metavar="PATH",
+                        help="the skill-brief.yaml whose doc_urls are the existing entries")
+    parser.add_argument("--scope-type", metavar="TYPE", help="the brief's scope.type")
+    args = parser.parse_args(argv)
+    unknown: list[dict[str, Any]] = []
+    try:
+        if args.detected_file is None and args.existing_brief is None and args.scope_type is None:
+            scope_type, existing, detected = _stdin_payload()
+        else:
+            scope_type = args.scope_type or ""
+            existing = _brief_doc_urls(args.existing_brief) if args.existing_brief is not None else []
+            detected = []
+            if args.detected_file is not None:
+                detected, unknown = map_detected(_read_detected(args.detected_file))
+    except InputError as exc:
+        sys.stderr.write(f"skf-merge-doc-urls: {exc}\n")
         return 2
 
     doc_urls, suppressed = merge_doc_urls(scope_type, existing, detected)
-    json.dump({"doc_urls": doc_urls, "suppressed": suppressed}, sys.stdout, ensure_ascii=False)
+    json.dump({"doc_urls": doc_urls, "suppressed": unknown + suppressed}, sys.stdout, ensure_ascii=False)
     sys.stdout.write("\n")
     return 0
 

package/src/shared/scripts/skf-names-present.py

@@ -0,0 +1,221 @@
+# /// script
+# requires-python = ">=3.11"
+# dependencies = []
+# ///
+"""SKF Names Present: the provenance entries whose export name a skill package never writes.
+
+skf-test-skill credits a provenance entry toward a stack's Export Coverage
+only when the package writes its `export_name`: reconcile-coverage.py
+(`load_doc_text`, `names_present`) joins the text of `SKILL.md` and of each
+`.md` file directly in `references/` (sorted, read as UTF-8, no subfolder)
+and looks for the name in it as a fixed string, case-sensitive, never inside
+a longer identifier: on a side where the name ends in a letter, digit, `_`
+or `$`, the next character may not be one of those, so `get` is not found in
+`target` or `getAll` (skf-test-skill's validate-inventory.py keeps that
+rule; NAME_CHARS here must stay the same). A name holding
+`::` (an impl-block method, `Type::method`) rolls up under its type and is
+never looked up, and an entry with no name is no cited contract.
+create-stack-skill (generate-output section 8) runs this helper on a
+compose-mode map before the commit, so a stack never commits a name
+skf-test-skill will not credit. The rule here is the scorer's:
+test/test-skf-stack-step-rules.py runs both on the same package.
+
+Usage:
+  uv run skf-names-present.py --provenance <provenance-map.json> \\
+      --skill-dir <dir> [--drop-absent]
+
+Output (stdout):
+  {
+    "checked": <int>,           # entries whose export_name was looked up
+    "skipped": <int>,           # entries with a `::` name or no name
+    "absent": [                 # in entries[] order
+      {"entry_index": <int>, "export_name": "<name>",
+       "source_library": "<library>" | null},
+      ...
+    ],
+    "dropped": [...],           # --drop-absent: the absent entries removed
+    "provenance_written": <bool>
+  }
+
+--drop-absent removes every absent entry from the map and rewrites it
+through skf-atomic-write.py beside this script (JSON indented by two
+spaces, non-ASCII kept, a final newline). `absent[]` still lists what the
+lookup found, and `dropped[]` repeats it. `entry_index` is the position in
+the map as it was read.
+
+Exit codes:
+  0  no entry is absent, or --drop-absent removed every absent entry
+  1  an entry is absent (absent[] lists it)
+  2  usage, file, JSON or write error (one line on stderr, no JSON)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+ATOMIC_WRITE_HELPER = "skf-atomic-write.py"
+ATOMIC_WRITE_TIMEOUT_SEC = 60
+METHOD_SEPARATOR = "::"
+# Characters that continue an identifier in the languages SKF documents (`$`
+# for JS/TS names such as `$state`): skf-test-skill's IDENT_CHARS.
+NAME_CHARS = r"\w$"
+_NAME_CHAR_RE = re.compile(f"[{NAME_CHARS}]")
+
+
+def load_doc_text(skill_dir: Path) -> str:
+    """The text skf-test-skill looks names up in: SKILL.md, then each .md file
+    directly in references/ in sorted order, joined by newlines. Raises
+    OSError, or UnicodeDecodeError for a file that is not UTF-8."""
+    parts = []
+    skill_md = skill_dir / "SKILL.md"
+    if skill_md.is_file():
+        parts.append(skill_md.read_text(encoding="utf-8"))
+    refs = skill_dir / "references"
+    if refs.is_dir():
+        for ref in sorted(refs.glob("*.md")):
+            parts.append(ref.read_text(encoding="utf-8"))
+    return "\n".join(parts)
+
+
+def name_written(name: str, doc_text: str) -> bool:
+    """True when `doc_text` holds `name` as skf-test-skill's scorer finds it."""
+    body = re.escape(name)
+    if _NAME_CHAR_RE.match(name[0]):
+        body = f"(?<![{NAME_CHARS}])" + body
+    if _NAME_CHAR_RE.match(name[-1]):
+        body += f"(?![{NAME_CHARS}])"
+    return re.search(body, doc_text) is not None
+
+
+def looked_up_name(entry: object) -> str | None:
+    """The name skf-test-skill looks up for an entry, or None when it looks
+    up none: no string name, or a `Type::method` name."""
+    if not isinstance(entry, dict):
+        return None
+    name = entry.get("export_name")
+    if not isinstance(name, str) or not name or METHOD_SEPARATOR in name:
+        return None
+    return name
+
+
+def find_absent(entries: list, doc_text: str) -> tuple[int, list[dict]]:
+    """(entries looked up, the absent ones in entries[] order)."""
+    checked, absent = 0, []
+    for index, entry in enumerate(entries):
+        name = looked_up_name(entry)
+        if name is None:
+            continue
+        checked += 1
+        if not name_written(name, doc_text):
+            library = entry.get("source_library")
+            absent.append({
+                "entry_index": index,
+                "export_name": name,
+                "source_library": library if isinstance(library, str) else None,
+            })
+    return checked, absent
+
+
+def atomic_write(target: Path, data: bytes) -> None:
+    """Write `data` to `target` through skf-atomic-write.py beside this
+    script. Raises OSError when the helper is missing or fails."""
+    helper = Path(__file__).resolve().parent / ATOMIC_WRITE_HELPER
+    if not helper.is_file():
+        raise OSError(f"{ATOMIC_WRITE_HELPER} not found beside {Path(__file__).name}; nothing written")
+    try:
+        result = subprocess.run(
+            [sys.executable, str(helper), "write", "--target", str(target)],
+            input=data,
+            capture_output=True,
+            timeout=ATOMIC_WRITE_TIMEOUT_SEC,
+            check=False,
+        )
+    except (OSError, subprocess.SubprocessError) as exc:
+        raise OSError(f"atomic write of {target} failed: {exc}") from exc
+    if result.returncode != 0:
+        detail = (result.stderr or b"").decode("utf-8", errors="replace").strip()
+        raise OSError(f"atomic write of {target} failed: {detail}")
+
+
+def _fail(message: str) -> int:
+    print(f"error: {' '.join(message.split())}", file=sys.stderr)
+    return 2
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="skf-names-present",
+        description=(
+            "List the provenance entries whose export_name the skill package "
+            "never writes, by skf-test-skill's Export Coverage rule "
+            "(SKILL.md and references/*.md, case-sensitive substring, `::` "
+            "names skipped); with --drop-absent, remove them from the map."
+        ),
+    )
+    parser.add_argument("--provenance", required=True, help="path to provenance-map.json")
+    parser.add_argument(
+        "--skill-dir",
+        required=True,
+        help="skill package folder holding the SKILL.md and references/ to search",
+    )
+    parser.add_argument(
+        "--drop-absent",
+        action="store_true",
+        help="remove every absent entry and rewrite the map through skf-atomic-write.py",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+    prov_path = Path(args.provenance)
+    skill_dir = Path(args.skill_dir)
+    if not prov_path.is_file():
+        return _fail(f"provenance map not found: {prov_path}")
+    # A wrong folder would read every name as absent, and --drop-absent
+    # would then empty the map: require the package's SKILL.md.
+    if not (skill_dir / "SKILL.md").is_file():
+        return _fail(f"skill dir has no SKILL.md: {skill_dir}")
+    try:
+        prov = json.loads(prov_path.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        return _fail(f"cannot read the provenance map {prov_path}: {exc}")
+    entries = prov.get("entries") if isinstance(prov, dict) else None
+    if not isinstance(entries, list):
+        return _fail(f"the provenance map {prov_path} has no entries[] list")
+    try:
+        doc_text = load_doc_text(skill_dir)
+    except (OSError, ValueError) as exc:
+        return _fail(f"cannot read the skill package {skill_dir}: {exc}")
+
+    checked, absent = find_absent(entries, doc_text)
+    dropped: list[dict] = []
+    if args.drop_absent and absent:
+        gone = {item["entry_index"] for item in absent}
+        prov["entries"] = [entry for index, entry in enumerate(entries) if index not in gone]
+        data = (json.dumps(prov, indent=2, ensure_ascii=False) + "\n").encode("utf-8")
+        try:
+            atomic_write(prov_path, data)
+        except OSError as exc:
+            return _fail(str(exc))
+        dropped = absent
+
+    result = {
+        "checked": checked,
+        "skipped": len(entries) - checked,
+        "absent": absent,
+        "dropped": dropped,
+        "provenance_written": bool(dropped),
+    }
+    # ASCII escapes: a Windows console's code page cannot encode every name.
+    sys.stdout.write(json.dumps(result, indent=2) + "\n")
+    return 0 if not absent or dropped else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())