bmad-module-skill-forge 3.0.0 → 4.0.0

package/src/skf-update-skill/scripts/skf-new-file-diff.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # /// script
 # requires-python = ">=3.11"
+# dependencies = ["pyyaml"]
 # ///
 """Deterministic NEW_FILE detection for skf-update-skill (detect-changes Category D).
 
@@ -10,39 +11,70 @@ file that is present in source yet absent from the provenance map: a NEW_FILE.
 Deriving that set is a set-difference plus a filter with exactly one correct
 answer per input: take every `source_file` in the scripts/assets inventory
 (emitted by `skf-detect-scripts-assets detect`), subtract the paths already in
-`file_entries[].source_file`, and set aside any user-authored `[MANUAL]` path.
-Doing that subtraction in the prompt drifts across runs; doing it here is
-byte-stable.
+`file_entries[].source_file`, and set aside any user-authored `[MANUAL]` path,
+any path the map's `entries[]` cite (code Category A and B track) and, with
+--brief, any path the skill brief leaves out. Doing that subtraction in the
+prompt drifts across runs; doing it here is byte-stable.
 
 Usage:
-  skf-detect-scripts-assets.py detect <source-root> \
-    | skf-new-file-diff.py <provenance-map-path>
+  skf-detect-scripts-assets.py detect <source-root> \\
+    | skf-new-file-diff.py <provenance-map-path> [--brief <brief-path>]
 
 Reads the detect JSON on stdin (needs `scripts_inventory[]` and
 `assets_inventory[]`, each row carrying `source_file`). Reads the provenance
-map at the path argument (canonical `{file_entries: [...]}` object, or a bare
-array of entries).
+map at the path argument (canonical `{file_entries: [...], entries: [...]}`
+object, or a bare array of file entries). A map with no `file_entries`, or a
+null one, tracks no file (a skill with no script, asset or promoted document,
+or a map written before the field existed); a `file_entries` or `entries`
+that is neither null nor an array is bad input.
+
+--brief <brief-path> (the skill's skill-brief.yaml) keeps a detected path
+only when the brief takes it, by update-skill Category A's in-scope test
+(skf-classify-changed-files.py load_scope, loaded from the shared
+scripts folder, the one brief-scope test the script and asset detector's
+and audit-skill's --brief use too, with the glob rules of
+skf-resolve-authoritative-files.py): each `scope.include` and
+`scope.exclude` pattern normalized (forward slashes, no leading `./`),
+and a path in scope when an include
+matches it and no exclude does. With an empty `scope.include`, a path is in
+scope when no exclude matches it and its extension is one a file the map's
+`entries[]` cite has, or one the brief's `language` uses. The brief's
+`scripts_intent` and `assets_intent` are honoured too: `none` drops every
+detected path of that kind, and an absent or free-text intent detects.
+An empty --brief (an unbound brief path) is a usage error, never an
+unscoped run. Every path, the map's and the detector's, is compared
+stripped, with forward slashes and no leading `./`.
 
 Output JSON (stdout):
   {
     "new_files":       [ {"source_file": "...", "kind": "script|asset"}, ... ],
     "skipped_manual":  [ "...", ... ],   # under scripts/[MANUAL]/ or assets/[MANUAL]/
     "already_tracked": [ "...", ... ],   # present in file_entries
+    "tracked_code":    [ "...", ... ],   # cited by entries[]: code, not a new file
+    "out_of_scope":    [ "...", ... ],   # --brief: outside the brief's scope
+    "intent_none":     [ "...", ... ],   # --brief: a kind whose intent is none
     "stats": {"inventory_total": N, "new": N,
               "skipped_manual": N, "already_tracked": N}
   }
 
-All three arrays are sorted by source_file; new_files carries the kind of the
-inventory it came from (scripts -> "script", assets -> "asset"). A path found
-in both inventories is counted once, resolved as "script".
+Each detected path lands in exactly one list, tested in this order:
+already_tracked, skipped_manual, tracked_code, out_of_scope, intent_none,
+and new_files for the rest. All arrays are sorted by
+source_file; new_files carries the kind of the inventory it came from
+(scripts -> "script", assets -> "asset"). A path found in both inventories is
+counted once, resolved as "script". The three set-aside lists (tracked_code,
+out_of_scope, intent_none) are never counted in `stats`, whose
+inventory_total still counts every detected path.
 
 Exit codes:
   0  success (and --help)
   2  bad input (a usage error, unreadable/invalid stdin JSON, missing/invalid
-     provenance map)
+     provenance map, a missing or unreadable brief, or a shared helper that
+     cannot be loaded)
 """
 
 import argparse
+import importlib.util
 import json
 import re
 import sys
@@ -51,6 +83,13 @@ from pathlib import Path
 # scripts/[MANUAL]/... or assets/[MANUAL]/... anywhere in the (posix) path
 _MANUAL_RE = re.compile(r"(?:^|/)(?:scripts|assets)/\[MANUAL\]/")
 
+# The shared helper that holds the scope test, installed beside this skill's
+# folder: it loads skf-resolve-authoritative-files.py beside it.
+SHARED_SCRIPTS = Path(__file__).resolve().parent.parent.parent / "shared" / "scripts"
+CLASSIFIER = "skf-classify-changed-files.py"
+
+_SHARED: dict[str, object] = {}
+
 
 def _fail(msg: str) -> "NoReturn":  # type: ignore[valid-type]
     print(json.dumps({"error": msg}), file=sys.stderr)
@@ -58,37 +97,88 @@ def _fail(msg: str) -> "NoReturn":  # type: ignore[valid-type]
 
 
 def _posix(p: str) -> str:
-    return p.replace("\\", "/")
+    """A path as skf-resolve-authoritative-files.py's normalize_rel_path
+    writes it: stripped, forward slashes, no leading `./`."""
+    p = p.strip().replace("\\", "/")
+    while p.startswith("./"):
+        p = p[2:]
+    return p
+
+
+def _shared(filename: str):
+    """A script of SHARED_SCRIPTS, loaded once; a failure to load it is bad input (exit 2)."""
+    module = _SHARED.get(filename)
+    if module is None:
+        path = SHARED_SCRIPTS / filename
+        try:
+            spec = importlib.util.spec_from_file_location("skf_" + filename[:-3].replace("-", "_"), path)
+            if spec is None or spec.loader is None or not path.is_file():
+                raise ImportError(f"{filename} not found in {SHARED_SCRIPTS}")
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+        except Exception as exc:  # whatever the script raises, the run reports it
+            _fail(f"cannot load {filename} from {SHARED_SCRIPTS}: {exc}; re-install SKF")
+        _SHARED[filename] = module
+    return module
+
+
+def _source_files(rows: list) -> set[str]:
+    found: set[str] = set()
+    for entry in rows:
+        if isinstance(entry, dict):
+            sf = entry.get("source_file")
+            if isinstance(sf, str) and _posix(sf):
+                found.add(_posix(sf))
+    return found
 
 
-def load_tracked_source_files(provenance_path: Path) -> set[str]:
-    """Return the set of file_entries[].source_file already in the provenance map."""
+def load_map_paths(provenance_path: Path) -> tuple[set[str], set[str]]:
+    """(file_entries[].source_file, entries[].source_file) of the provenance map.
+
+    A missing or null `file_entries` (or `entries`) is an empty list; one
+    that is neither null nor an array is bad input. A bare array is the
+    file entries themselves.
+    """
     try:
         data = json.loads(provenance_path.read_text(encoding="utf-8"))
     except (json.JSONDecodeError, OSError) as exc:
         _fail(f"failed to read provenance map {provenance_path}: {exc}")
 
     if isinstance(data, list):
-        entries = data
-    elif isinstance(data, dict):
-        entries = data.get("file_entries")
-        if entries is None:
-            _fail(f"provenance map {provenance_path} has no `file_entries` field")
-        if not isinstance(entries, list):
-            _fail(f"`file_entries` in {provenance_path} is not an array")
-    else:
+        return _source_files(data), set()
+    if not isinstance(data, dict):
         _fail(
             f"provenance map {provenance_path} must be an object or array; "
             f"got {type(data).__name__}"
         )
+    rows = {}
+    for key in ("file_entries", "entries"):
+        value = data.get(key)
+        if value is None:
+            value = []  # a map that tracks no such row
+        if not isinstance(value, list):
+            _fail(f"`{key}` in {provenance_path} is not an array")
+        rows[key] = value
+    return _source_files(rows["file_entries"]), _source_files(rows["entries"])
 
-    tracked: set[str] = set()
-    for entry in entries:
-        if isinstance(entry, dict):
-            sf = entry.get("source_file")
-            if isinstance(sf, str) and sf:
-                tracked.add(_posix(sf))
-    return tracked
+
+def load_tracked_source_files(provenance_path: Path) -> set[str]:
+    """Return the set of file_entries[].source_file already in the provenance map."""
+    return load_map_paths(provenance_path)[0]
+
+
+def load_scope(brief_path: Path, cited: set[str]):
+    """The brief's Scope (skf-classify-changed-files.py load_scope): its
+    takes(path) is Category A's in-scope test and its wants(kind) the
+    brief's intent. A brief it cannot find or read, or a shared helper it
+    cannot load, is bad input (exit 2)."""
+    classifier = _shared(CLASSIFIER)
+    try:
+        return classifier.load_scope(brief_path, cited)
+    except ValueError as exc:  # a brief it cannot find or read
+        _fail(str(exc))
+    except Exception as exc:  # an out-of-date classifier, or a resolver beside it that cannot load
+        _fail(f"cannot load the brief's scope test from {SHARED_SCRIPTS}: {exc}; re-install SKF")
 
 
 def collect_inventory(detect: dict) -> list[tuple[str, str]]:
@@ -114,38 +204,53 @@ def collect_inventory(detect: dict) -> list[tuple[str, str]]:
     return sorted(seen.items())
 
 
-def diff(detect: dict, tracked: set[str]) -> dict:
+def diff(detect: dict, tracked: set[str], cited: set[str] | frozenset = frozenset(),
+         scope=None) -> dict:
+    """Sort the detected paths into the output lists; `scope` is load_scope's
+    Scope (--brief) or None."""
     new_files: list[dict] = []
-    skipped_manual: list[str] = []
-    already_tracked: list[str] = []
+    lists: dict[str, list[str]] = {name: [] for name in (
+        "skipped_manual", "already_tracked", "tracked_code", "out_of_scope", "intent_none")}
 
     inventory = collect_inventory(detect)
     for sf, kind in inventory:
         if sf in tracked:
-            already_tracked.append(sf)
+            lists["already_tracked"].append(sf)
         elif _MANUAL_RE.search(sf):
-            skipped_manual.append(sf)
+            lists["skipped_manual"].append(sf)
+        elif sf in cited:
+            lists["tracked_code"].append(sf)
+        elif scope is not None and not scope.takes(sf):
+            lists["out_of_scope"].append(sf)
+        elif scope is not None and not scope.wants(kind):
+            lists["intent_none"].append(sf)
         else:
             new_files.append({"source_file": sf, "kind": kind})
 
     return {
         "new_files": new_files,
-        "skipped_manual": sorted(skipped_manual),
-        "already_tracked": sorted(already_tracked),
+        "skipped_manual": sorted(lists["skipped_manual"]),
+        "already_tracked": sorted(lists["already_tracked"]),
+        "tracked_code": sorted(lists["tracked_code"]),
+        "out_of_scope": sorted(lists["out_of_scope"]),
+        "intent_none": sorted(lists["intent_none"]),
         "stats": {
             "inventory_total": len(inventory),
             "new": len(new_files),
-            "skipped_manual": len(skipped_manual),
-            "already_tracked": len(already_tracked),
+            "skipped_manual": len(lists["skipped_manual"]),
+            "already_tracked": len(lists["already_tracked"]),
         },
     }
 
 
+USAGE = "usage: skf-new-file-diff.py <provenance-map-path> [--brief <brief-path>]  (detect JSON on stdin)"
+
+
 class _JsonErrorParser(argparse.ArgumentParser):
     """Report a usage error as the JSON error and exit 2 every other bad input gets."""
 
     def error(self, message: str) -> "NoReturn":  # type: ignore[valid-type]
-        _fail(f"usage: skf-new-file-diff.py <provenance-map-path>  (detect JSON on stdin): {message}")
+        _fail(f"{USAGE}: {message}")
 
 
 def _build_parser() -> argparse.ArgumentParser:
@@ -153,17 +258,26 @@ def _build_parser() -> argparse.ArgumentParser:
         prog="skf-new-file-diff.py",
         description=__doc__,
         formatter_class=argparse.RawDescriptionHelpFormatter,
+        allow_abbrev=False,
     )
     parser.add_argument(
         "provenance_map",
         metavar="provenance-map-path",
         help="the skill's provenance-map.json (the detect JSON is read on stdin)",
     )
+    parser.add_argument(
+        "--brief",
+        metavar="brief-path",
+        help="the skill brief: keep only the paths its scope takes and its intents want",
+    )
     return parser
 
 
 def main(argv: list[str]) -> int:
-    args = _build_parser().parse_args(argv[1:])
+    parser = _build_parser()
+    args = parser.parse_args(argv[1:])
+    if args.brief is not None and not args.brief.strip():
+        parser.error("--brief is empty (an unbound brief path?): pass the brief, or leave --brief out")
 
     provenance_path = Path(args.provenance_map)
     if not provenance_path.exists():
@@ -179,8 +293,9 @@ def main(argv: list[str]) -> int:
     if not isinstance(detect, dict):
         _fail(f"detect JSON must be an object; got {type(detect).__name__}")
 
-    tracked = load_tracked_source_files(provenance_path)
-    result = diff(detect, tracked)
+    tracked, cited = load_map_paths(provenance_path)
+    scope = load_scope(Path(args.brief), cited) if args.brief is not None else None
+    result = diff(detect, tracked, cited, scope)
     print(json.dumps(result, indent=2))
     return 0