bmad-module-skill-forge 2.2.0 → 4.0.0

package/src/shared/scripts/skf-compare-file-hashes.py

@@ -1,34 +1,62 @@
 # /// script
-# requires-python = ">=3.10"
-# dependencies = []
+# requires-python = ">=3.11"
+# dependencies = ["pyyaml"]
 # ///
-"""SKF Compare File Hashes — script/asset/doc drift detection for audit-skill.
+"""SKF Compare File Hashes: script/asset/doc drift detection for audit-skill.
 
 This script replaces the per-file hash-compute-and-classify prose at
 `src/skf-audit-skill/references/structural-diff.md` §4b (Script/Asset Drift).
 It compares the `file_entries[]` block of a provenance map against the
 current state of the source tree and emits three drift categories:
 
-  - **added**:   files present in the source tree but NOT recorded in
-                 `file_entries[]`. Walk is restricted to standard script/
-                 asset/doc directories (scripts/, bin/, tools/, cli/,
-                 assets/, templates/, schemas/, configs/, examples/) and
-                 excludes binary extensions and generated paths — same
-                 path discipline as `skf-detect-scripts-assets.py`.
+  - **added**:   with --brief, files present in the source tree but NOT
+                 recorded in `file_entries[]` that the skill brief takes.
+                 The walk finds scripts and assets with the detector
+                 create-skill and update-skill use
+                 (skf-detect-scripts-assets.py, loaded from this folder:
+                 its iter_files, is_binary_path, entry_point_script_paths
+                 and file_kind), so it takes what update-skill's Category D
+                 takes: a file under a script or asset folder at any depth,
+                 a shebang file or a `package.json` `bin` entry point in
+                 any folder, and a pattern asset (`*.schema.json`,
+                 `openapi.*`, `*.example`, ...) anywhere, skipping binary
+                 extensions, generated paths and symbolic links, and
+                 leaving out a module of a Python package named like a
+                 script folder, which is code unless it runs on its own.
+                 It also takes each docs/authoritative/ file, a document.
+                 A file under `scripts/[MANUAL]/` or `assets/[MANUAL]/` is
+                 user-authored, never added (Category D sets it aside too).
+                 Without --brief, or with a brief it cannot read, no new
+                 file is looked for: `added` is empty and
+                 `added_not_checked` says why.
   - **removed**: files in `file_entries[]` whose `source_file` is gone
                  from disk under `<source-root>`.
   - **changed**: files in `file_entries[]` present on disk but with a
                  different content hash than the stored hash.
 
+--brief <skill-brief.yaml> keeps a walked file as added only when the
+brief's scope takes it, by the one brief-scope test of
+skf-classify-changed-files.py (load_scope, loaded from this folder only
+when --brief is given), which update-skill's Category A and Category D
+and create-skill's script and asset detector use too: with the glob rules
+of skf-resolve-authoritative-files.py, and, for an empty `scope.include`,
+the extensions of the map's `entries[]` files and of the brief's
+`language`. A file the map's `entries[]` cite is code, never an added file
+(update-skill's `tracked_code`). A walked file is a script or an asset by
+the detector's file_kind, a script first (a shebang file under an asset
+folder is a script), except a docs/authoritative/ file, a document
+whatever the detector says: a script or an asset whose `scripts_intent` or
+`assets_intent` is `none` is no added file, and no intent filters a
+document.
+
 Distinct from `skf-hash-content.py compare`: that script classifies entries
 that ARE in the provenance map (UNCHANGED / MODIFIED_FILE / DELETED_FILE).
-audit-skill additionally needs the **inverse walk** — what new
+audit-skill additionally needs the **inverse walk**: what new
 script/asset/doc files have appeared in the source tree since the skill
-was created? That's `added[]`. We delegate the path-resolution and
-hash-comparison logic for entries IN the map by duplicating the small
-sha256 helper here (15 lines) rather than importing across files — keeps
-the runtime import surface flat and matches the existing canonical pattern
-in `skf-detect-scripts-assets.py:216`.
+was created? That's `added[]`, the set update-skill's Category D pipes
+the detector into skf-new-file-diff.py for. The sha256 helper is the one
+skf-detect-scripts-assets.py has, kept here so hashing, and so removed and
+changed, need no other file.
 
 Hash-prefix normalization (writer-vs-reader compatibility): stored hashes
 in `file_entries[].content_hash` carry a `sha256:` prefix by SKF convention
@@ -40,10 +68,11 @@ the original form. A bare-hex hash from a future writer would pass through
 unchanged.
 
 Subcommand:
-  compare <provenance-map.json> <source-root>
+  compare <provenance-map.json> <source-root> [--brief <skill-brief.yaml>]
       Emit JSON:
         {
           "added": ["<rel-path, forward-slash>", ...],
+          "added_not_checked": null | "<reason>",
           "removed": ["<rel-path, forward-slash>", ...],
           "changed": [
             {"path": "<rel-path>", "stored_hash": "sha256:...",
@@ -54,55 +83,100 @@ Subcommand:
 
       All paths are forward-slash relative to `<source-root>`. Stable sort
       on the lists for deterministic output (lexicographic on path).
+      `added_not_checked` is null when the walk ran, else the reason no
+      new file was looked for, on one line: `no skill brief` without
+      --brief (or with an empty one), the error the brief gave
+      (`brief not found: <path>`, or a brief that is not valid YAML or no
+      mapping), or `cannot load <helper> beside skf-compare-file-hashes.py:
+      ...; re-install SKF` when a helper of this folder the walk needs
+      cannot be loaded or is out of date. `stats.added` counts `added`.
 
 CLI examples:
-  uv run skf-compare-file-hashes.py compare prov-map.json /path/to/src
+  uv run skf-compare-file-hashes.py compare prov-map.json /path/to/src \\
+      --brief skill-brief.yaml
 
 Exit codes:
-  0  — comparison succeeded (including empty/well-formed input that yields
-       all-empty drift lists)
-  1  — user error (missing files, malformed JSON, unreadable paths)
+  0  comparison succeeded (including empty/well-formed input that yields
+     all-empty drift lists, and a brief or a helper of this folder it
+     cannot read or load: see `added_not_checked`)
+  1  user error (missing files, malformed JSON, unreadable paths)
 """
 
 from __future__ import annotations
 
 import argparse
 import hashlib
+import importlib.util
 import json
 import re
 import sys
 from pathlib import Path
-from typing import Iterable
+from typing import Callable, Iterable
 
 
 # --------------------------------------------------------------------------
-# Constants — mirror skf-detect-scripts-assets.py so the inverse walk
-# explores the same set of trees the writer side would have considered.
+# Constants: the walk's script and asset rules are the detector's
+# (skf-detect-scripts-assets.py), loaded from this folder, not copies.
 # --------------------------------------------------------------------------
 
 
-SCRIPT_DIRS = {"scripts", "bin", "tools", "cli"}
-ASSET_DIRS = {"assets", "templates", "schemas", "configs", "examples"}
 DOC_DIR_PREFIXES = ("docs/authoritative/",)  # synthetic namespace from create-skill §6
 
-# Path-segment names that mark generated/vendored output trees — pruned
-# from the walk so the inverse never reports build-tree artifacts as added.
-EXCLUDED_DIR_NAMES = {
-    "node_modules", "__pycache__", "dist", "build", ".webpack",
-    "target", ".next", ".nuxt", "out", "coverage", ".git",
-    ".venv", "venv", ".tox", ".mypy_cache", ".pytest_cache",
-    ".ruff_cache", ".gradle", ".idea", ".vscode",
-}
+# scripts/[MANUAL]/... or assets/[MANUAL]/... anywhere in the (posix) path:
+# a copy of skf-new-file-diff.py's _MANUAL_RE (update-skill's Category D),
+# which a test keeps equal to it.
+_MANUAL_RE = re.compile(r"(?:^|/)(?:scripts|assets)/\[MANUAL\]/")
 
-BINARY_EXTS = {
-    ".so", ".dll", ".jar", ".wasm", ".exe", ".dylib", ".a", ".o",
-    ".pyc", ".class", ".png", ".jpg", ".jpeg", ".gif", ".ico",
-    ".pdf", ".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".7z",
-    ".woff", ".woff2", ".ttf", ".otf", ".eot",
-}
+_HASH_PREFIX_RE = re.compile(r"^[a-z0-9]+:")
 
+DETECTOR = "skf-detect-scripts-assets.py"    # the walk (--brief): see _detector
+CLASSIFIER = "skf-classify-changed-files.py"  # load_scope: the brief's scope (--brief)
+NO_BRIEF = "no skill brief"
 
-_HASH_PREFIX_RE = re.compile(r"^[a-z0-9]+:")
+_SIBLINGS: dict[str, object] = {}
+
+
+class HelperError(Exception):
+    """A helper beside this script that cannot be loaded, lacks what this
+    script calls or fails in it: no new file can be looked for."""
+
+
+def _helper_error(filename: str, exc: BaseException) -> HelperError:
+    return HelperError(f"cannot load {filename} beside {Path(__file__).name}: {exc}; re-install SKF")
+
+
+def _sibling(filename: str):
+    """A helper installed beside this script, loaded once as a module.
+    Raises HelperError when it cannot be loaded."""
+    module = _SIBLINGS.get(filename)
+    if module is None:
+        path = Path(__file__).resolve().parent / filename
+        name = "skf_" + filename.removeprefix("skf-").removesuffix(".py").replace("-", "_")
+        try:
+            spec = importlib.util.spec_from_file_location(name, path)
+            if spec is None or spec.loader is None or not path.is_file():
+                raise ImportError(f"{filename} not found")
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+        except Exception as exc:  # whatever the helper raises, the run reports it
+            raise _helper_error(filename, exc) from exc
+        _SIBLINGS[filename] = module
+    return module
+
+
+def _posix(p: str) -> str:
+    """A path as skf-resolve-authoritative-files.py's normalize_rel_path
+    writes it: stripped, forward slashes, no leading `./`."""
+    p = p.strip().replace("\\", "/")
+    while p.startswith("./"):
+        p = p[2:]
+    return p
+
+
+def _one_line(message: str) -> str:
+    """Each line break of `message` and the space around it as one space
+    (a YAML error spans several lines)."""
+    return re.sub(r"\s*[\r\n]\s*", " ", message.strip("\r\n"))
 
 
 # --------------------------------------------------------------------------
@@ -136,16 +210,8 @@ def normalize_hash(value: str | None) -> str | None:
 # --------------------------------------------------------------------------
 
 
-def load_file_entries(provenance_path: Path) -> list[dict]:
-    """Extract `file_entries[]` from a provenance map. Accepts:
-      - top-level object with a `file_entries` key (canonical)
-      - top-level array (already extracted)
-      - top-level object with NO `file_entries` field → empty list
-        (a single-skill with no scripts/assets/docs may omit the field
-        entirely per skill-sections.md §file_entries)
-
-    Raises ValueError on read failure or structural defects.
-    """
+def _read_map(provenance_path: Path) -> object:
+    """The provenance map's JSON. Raises ValueError when it cannot be read."""
     try:
         text = provenance_path.read_text(encoding="utf-8")
     except OSError as exc:
@@ -153,12 +219,27 @@ def load_file_entries(provenance_path: Path) -> list[dict]:
             f"failed to read provenance file {provenance_path}: {exc}"
         ) from exc
     try:
-        data = json.loads(text)
+        return json.loads(text)
     except json.JSONDecodeError as exc:
         raise ValueError(
             f"malformed JSON in provenance file {provenance_path}: {exc}"
         ) from exc
 
+
+def load_file_entries(provenance_path: Path) -> list[dict]:
+    """Extract `file_entries[]` from a provenance map. Accepts:
+      - top-level object with a `file_entries` key (canonical)
+      - top-level array (already extracted)
+      - top-level object with NO `file_entries` field → empty list
+        (a single-skill with no scripts/assets/docs may omit the field
+        entirely per skill-sections.md §file_entries)
+
+    Raises ValueError on read failure or structural defects.
+    """
+    return _file_entries(_read_map(provenance_path), provenance_path)
+
+
+def _file_entries(data: object, provenance_path: Path) -> list[dict]:
     if isinstance(data, list):
         return list(data)
     if isinstance(data, dict):
@@ -176,69 +257,96 @@ def load_file_entries(provenance_path: Path) -> list[dict]:
     )
 
 
-# --------------------------------------------------------------------------
-# Walk — the inverse direction (source tree → candidate set)
-# --------------------------------------------------------------------------
-
-
-def _segment_in_excluded(rel_parts: tuple[str, ...]) -> bool:
-    return any(seg in EXCLUDED_DIR_NAMES for seg in rel_parts)
+def _cited(data: object) -> set[str]:
+    """The map's `entries[].source_file`, the code files it cites (none for
+    a bare array, or an `entries` that is not a list)."""
+    rows = data.get("entries") if isinstance(data, dict) else None
+    if not isinstance(rows, list):
+        return set()
+    return {
+        _posix(row["source_file"])
+        for row in rows
+        if isinstance(row, dict) and isinstance(row.get("source_file"), str) and _posix(row["source_file"])
+    }
 
 
-def _segment_in_tracked_dir(rel_parts: tuple[str, ...]) -> bool:
-    """True if any segment of the relative path is a tracked script/asset
-    directory."""
-    for seg in rel_parts:
-        if seg in SCRIPT_DIRS or seg in ASSET_DIRS:
-            return True
-    return False
+# --------------------------------------------------------------------------
+# Walk: the inverse direction (source tree to candidate set)
+# --------------------------------------------------------------------------
 
 
 def _matches_doc_prefix(rel_posix: str) -> bool:
     return any(rel_posix.startswith(p) for p in DOC_DIR_PREFIXES)
 
 
+def _detector(source_root: Path) -> tuple:
+    """(iter_files, is_binary_path, file_kind,
+    entry_point_script_paths(source_root)) from the detector, read before
+    any file is walked. Raises HelperError when the detector cannot be
+    loaded, lacks one of these four functions or fails in
+    entry_point_script_paths."""
+    detector = _sibling(DETECTOR)
+    try:
+        return (detector.iter_files, detector.is_binary_path, detector.file_kind,
+                detector.entry_point_script_paths(source_root))
+    except Exception as exc:  # an out-of-date detector, or one that fails
+        raise _helper_error(DETECTOR, exc) from exc
+
+
+def _walk(source_root: Path, keep: Callable[[str], bool] | None = None) -> Iterable[tuple[str, str]]:
+    """(POSIX relative path, kind) of each file under source_root that could
+    be tracked as a script, asset or doc file_entries row, by the detector's
+    walk (iter_files, is_binary_path): a docs/authoritative/ file is a
+    `doc`, whatever the detector says, and any other file the kind the
+    detector's file_kind gives it, both kinds on (`script` or `asset`), or
+    left out. `keep`, a test of the relative path, runs before the doc rule
+    and file_kind, so a file it refuses is never read. The detector is
+    loaded when the walk starts. Raises HelperError when it cannot be
+    loaded, lacks a function the walk calls or fails in one, and lets
+    through a HelperError `keep` raises."""
+    iter_files, is_binary_path, file_kind, entry_points = _detector(source_root)
+    try:
+        for entry in iter_files(source_root):
+            if is_binary_path(entry):
+                continue
+            rel = entry.relative_to(source_root).as_posix()
+            if keep is not None and not keep(rel):
+                continue
+            if _matches_doc_prefix(rel):
+                yield rel, "doc"
+                continue
+            found = file_kind(entry, source_root, entry_points)
+            if found is not None:
+                yield rel, found[0]
+    except HelperError:
+        raise
+    except Exception as exc:  # the detector failing on a file
+        raise _helper_error(DETECTOR, exc) from exc
+
+
 def candidate_source_files(source_root: Path) -> Iterable[str]:
     """Yield POSIX relative paths under source_root that could be tracked
-    as script / asset / doc file_entries.
+    as script / asset / doc file_entries (see _walk).
 
     Selection rules:
-      - File is under a SCRIPT_DIRS or ASSET_DIRS directory at any depth, OR
-      - File path matches a DOC_DIR_PREFIXES synthetic namespace.
+      - A script or an asset by skf-detect-scripts-assets.py's file_kind:
+        a file under a script or asset folder at any depth, a shebang file
+        or a `package.json` `bin` entry point in any folder, a pattern
+        asset (`*.schema.json`, `openapi.*`, `*.example`, ...) anywhere;
+        a module of a Python package named like a script folder is code,
+        not a script, unless an asset folder holds it or its name is an
+        asset pattern.
+      - Or a file under a DOC_DIR_PREFIXES synthetic namespace.
       - Binary extensions are excluded.
       - Generated/vendored directories are pruned from the walk.
 
-    This is deliberately narrower than "every file" — audit-skill §4b only
+    This is deliberately narrower than "every file": audit-skill §4b only
     tracks files matching script/asset patterns. Treating every random
     source file as a candidate `added` row would drown real drift in noise.
+    Raises HelperError when the detector cannot be loaded.
     """
-    stack: list[Path] = [source_root]
-    while stack:
-        current = stack.pop()
-        try:
-            entries = list(current.iterdir())
-        except (PermissionError, FileNotFoundError):
-            continue
-        for entry in entries:
-            if entry.is_symlink():
-                continue
-            if entry.is_dir():
-                if entry.name in EXCLUDED_DIR_NAMES:
-                    continue
-                stack.append(entry)
-                continue
-            if not entry.is_file():
-                continue
-            if entry.suffix.lower() in BINARY_EXTS:
-                continue
-            try:
-                rel = entry.relative_to(source_root)
-            except ValueError:
-                continue
-            rel_parts = rel.parts[:-1]
-            rel_posix = rel.as_posix()
-            if _segment_in_tracked_dir(rel_parts) or _matches_doc_prefix(rel_posix):
-                yield rel_posix
+    for rel, _kind_ in _walk(source_root):
+        yield rel
 
 
 # --------------------------------------------------------------------------
@@ -246,14 +354,19 @@ def candidate_source_files(source_root: Path) -> Iterable[str]:
 # --------------------------------------------------------------------------
 
 
-def compare(source_root: Path, provenance_path: Path) -> dict:
+def compare(source_root: Path, provenance_path: Path, brief_path: Path | None = None) -> dict:
     """Compute added / removed / changed / unchanged for tracked file_entries.
 
     Stored hashes are normalized before comparison; the diff record carries
     the original stored value (writer-form) plus the freshly computed
-    current value (always prefixed sha256:) so reviewers see both.
+    current value (always prefixed sha256:) so reviewers see both. Added
+    files are looked for only with a brief it can read (`brief_path`):
+    otherwise `added` is empty and `added_not_checked` says why, as it
+    does when a helper the walk needs cannot be loaded. Raises ValueError
+    for a map it cannot read.
     """
-    entries = load_file_entries(provenance_path)
+    data = _read_map(provenance_path)
+    entries = _file_entries(data, provenance_path)
     # Build the {posix_path -> stored_hash} index from the provenance.
     stored: dict[str, str | None] = {}
     for entry in entries:
@@ -266,9 +379,6 @@ def compare(source_root: Path, provenance_path: Path) -> dict:
             entry.get("content_hash"), str
         ) else None
 
-    # Walk the source tree once; track which provenance entries are matched.
-    candidates_on_disk = set(candidate_source_files(source_root))
-
     removed: list[str] = []
     changed: list[dict] = []
     unchanged_count = 0
@@ -288,11 +398,11 @@ def compare(source_root: Path, provenance_path: Path) -> dict:
                 "current_hash": current,
             })
 
-    # Anything on disk but NOT in the provenance store is "added".
-    added = sorted(candidates_on_disk - set(stored.keys()))
+    added, not_checked = _added(source_root, data, stored, brief_path)
 
     return {
         "added": added,
+        "added_not_checked": not_checked,
         "removed": sorted(removed),
         "changed": sorted(changed, key=lambda r: r["path"]),
         "stats": {
@@ -304,6 +414,50 @@ def compare(source_root: Path, provenance_path: Path) -> dict:
     }
 
 
+def _added(source_root: Path, data: object, stored: dict, brief_path: Path | None) -> tuple[list[str], str | None]:
+    """(added, added_not_checked): the walked files the map does not track
+    and the brief takes, or ([], the reason) without a brief it can read or
+    a helper the walk needs. removed and changed need neither. The brief's
+    scope is loaded before the detector, and the path rules (tracked or
+    cited, [MANUAL], the brief's scope) run before the detector reads a
+    file."""
+    if brief_path is None:
+        return [], NO_BRIEF
+    cited = _cited(data)
+    try:
+        try:
+            scope = _sibling(CLASSIFIER).load_scope(brief_path, cited)
+        except ValueError as exc:  # a brief it cannot find or read
+            return [], _one_line(str(exc))
+        except HelperError:
+            raise
+        except Exception as exc:  # an out-of-date classifier, or a resolver beside it that cannot load
+            raise _helper_error(CLASSIFIER, exc) from exc
+        known = {_posix(path) for path in stored}
+
+        def keep(rel: str) -> bool:
+            if rel in known or rel in cited:  # tracked, or code the map's entries[] cite
+                return False
+            if _MANUAL_RE.search(rel):  # user-authored, as Category D's skipped_manual
+                return False
+            return _scope_call(scope.takes, rel)
+
+        added = [rel for rel, kind in _walk(source_root, keep)
+                 if kind == "doc" or _scope_call(scope.wants, kind)]  # no intent filters a document
+    except HelperError as exc:
+        return [], _one_line(str(exc))
+    return sorted(added), None
+
+
+def _scope_call(method, value) -> bool:
+    """A Scope method's answer. Raises HelperError when it fails (the
+    resolver it calls cannot be loaded or is out of date)."""
+    try:
+        return bool(method(value))
+    except Exception as exc:
+        raise _helper_error(CLASSIFIER, exc) from exc
+
+
 # --------------------------------------------------------------------------
 # CLI
 # --------------------------------------------------------------------------
@@ -318,8 +472,9 @@ def _cmd_compare(args: argparse.Namespace) -> int:
     if not source_root.is_dir():
         print(f"error: source root not a directory: {source_root}", file=sys.stderr)
         return 1
+    brief = Path(args.brief) if args.brief is not None and args.brief.strip() else None
     try:
-        result = compare(source_root, provenance)
+        result = compare(source_root, provenance, brief)
     except ValueError as exc:
         print(f"error: {exc}", file=sys.stderr)
         return 1
@@ -343,6 +498,13 @@ def _build_parser() -> argparse.ArgumentParser:
     )
     p_cmp.add_argument("provenance_map", help="path to provenance-map.json")
     p_cmp.add_argument("source_root", help="path to the source tree root")
+    p_cmp.add_argument(
+        "--brief",
+        default=None,
+        metavar="BRIEF",
+        help="the skill brief (skill-brief.yaml): look for added files only in its scope "
+             "(none, or one it cannot read: no added file is looked for)",
+    )
     p_cmp.set_defaults(func=_cmd_compare)
     return parser