bmad-module-skill-forge 3.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/docs/_data/pinned.yaml +1 -1
- package/docs/examples.md +1 -1
- package/docs/troubleshooting.md +8 -0
- package/docs/verifying-a-skill.md +7 -3
- package/docs/workflows.md +3 -3
- package/package.json +1 -1
- package/src/knowledge/qmd-registry.md +1 -1
- package/src/knowledge/version-paths.md +1 -1
- package/src/shared/health-check.md +2 -50
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +2 -2
- package/src/shared/scripts/skf-atomic-write.py +212 -59
- package/src/shared/scripts/skf-build-change-manifest.py +593 -47
- package/src/shared/scripts/skf-classify-changed-files.py +78 -0
- package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
- package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
- package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
- package/src/shared/scripts/skf-extract-public-api.py +1309 -157
- package/src/shared/scripts/skf-extraction-inventory.py +580 -46
- package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
- package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
- package/src/shared/scripts/skf-github-probe.py +47 -8
- package/src/shared/scripts/skf-hash-content.py +14 -8
- package/src/shared/scripts/skf-load-provenance.py +3 -1
- package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
- package/src/shared/scripts/skf-preflight.py +15 -1
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
- package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
- package/src/shared/scripts/skf-structural-diff.py +133 -18
- package/src/shared/scripts/skf-update-active-symlink.py +115 -62
- package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
- package/src/shared/scripts/skf-verify-provenance-completeness.py +977 -49
- package/src/shared/scripts/skf-write-skill-brief.py +41 -8
- package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
- package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
- package/src/skf-audit-skill/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +2 -1
- package/src/skf-audit-skill/references/re-index.md +14 -9
- package/src/skf-audit-skill/references/report.md +2 -2
- package/src/skf-audit-skill/references/semantic-diff.md +15 -15
- package/src/skf-audit-skill/references/structural-diff.md +10 -5
- package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
- package/src/skf-brief-skill/SKILL.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +1 -1
- package/src/skf-brief-skill/assets/skill-brief-schema.md +2 -9
- package/src/skf-brief-skill/references/analyze-target.md +6 -6
- package/src/skf-brief-skill/references/confirm-brief.md +1 -1
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
- package/src/skf-brief-skill/references/version-resolution.md +5 -9
- package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
- package/src/skf-create-skill/assets/skill-sections.md +6 -6
- package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
- package/src/skf-create-skill/references/compile.md +31 -60
- package/src/skf-create-skill/references/component-extraction.md +3 -3
- package/src/skf-create-skill/references/enrich.md +2 -2
- package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
- package/src/skf-create-skill/references/extract.md +22 -19
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
- package/src/skf-create-skill/references/extraction-patterns.md +8 -9
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +16 -5
- package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
- package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
- package/src/skf-drop-skill/references/execute.md +5 -1
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +2 -2
- package/src/skf-drop-skill/references/select.md +7 -3
- package/src/skf-test-skill/references/coherence-check.md +1 -1
- package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
- package/src/skf-test-skill/references/coverage-check.md +39 -57
- package/src/skf-test-skill/references/init.md +3 -5
- package/src/skf-test-skill/references/report.md +10 -8
- package/src/skf-test-skill/references/scoring-rules.md +6 -5
- package/src/skf-test-skill/references/source-access-protocol.md +14 -17
- package/src/skf-test-skill/scripts/build-result-context.py +114 -21
- package/src/skf-test-skill/scripts/gap-ledger.py +315 -30
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +534 -96
- package/src/skf-test-skill/scripts/reconcile-coverage.py +276 -2
- package/src/skf-test-skill/scripts/score-signatures.py +36 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
- package/src/skf-update-skill/references/detect-changes.md +32 -13
- package/src/skf-update-skill/references/gap-driven.md +3 -3
- package/src/skf-update-skill/references/init.md +6 -2
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +2 -2
- package/src/skf-update-skill/references/re-extract.md +2 -2
- package/src/skf-update-skill/references/report.md +3 -1
- package/src/skf-update-skill/references/write.md +43 -23
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
|
@@ -103,6 +103,21 @@ Modes, from the statuses:
|
|
|
103
103
|
|
|
104
104
|
An unknown --language value adds an `unknown-language` warning.
|
|
105
105
|
|
|
106
|
+
Library use: the one brief-scope test for script and asset paths.
|
|
107
|
+
skf-new-file-diff.py (update-skill Category D, --brief),
|
|
108
|
+
skf-detect-scripts-assets.py detect --brief (create-skill) and
|
|
109
|
+
skf-compare-file-hashes.py compare --brief (audit-skill) load this file by
|
|
110
|
+
its path and call load_scope(brief_path, cited). It returns a Scope:
|
|
111
|
+
takes(path) is the in-scope test above (`_in_scope`), with the brief's
|
|
112
|
+
`scope.include` and `scope.exclude` normalized as classify normalizes them
|
|
113
|
+
and, for an empty `scope.include`, the extensions of the `cited` paths
|
|
114
|
+
(the provenance map's `entries[].source_file`) and of the brief's
|
|
115
|
+
`language`; wants(kind) is False for a `script` or `asset` whose
|
|
116
|
+
`scripts_intent` or `assets_intent` is `none` (an absent or free-text
|
|
117
|
+
intent detects). load_scope raises ValueError, `brief not found: <path>`
|
|
118
|
+
for a path that names no file and load_brief's message for a brief it
|
|
119
|
+
cannot read; each caller keeps its own exit code.
|
|
120
|
+
|
|
106
121
|
Output JSON (stdout):
|
|
107
122
|
|
|
108
123
|
{
|
|
@@ -354,6 +369,69 @@ def _in_scope(rel: str, includes: list[str], excludes: list[str], extensions: se
|
|
|
354
369
|
return PurePosixPath(rel).suffix in extensions
|
|
355
370
|
|
|
356
371
|
|
|
372
|
+
# --------------------------------------------------------------------------
|
|
373
|
+
# The brief's scope for script and asset paths (Library use)
|
|
374
|
+
# --------------------------------------------------------------------------
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
# kind -> the brief field that holds its intent
|
|
378
|
+
INTENT_FIELDS = {"script": "scripts_intent", "asset": "assets_intent"}
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
class Scope:
|
|
382
|
+
"""A skill brief's scope and script and asset intents, by this helper's in-scope test."""
|
|
383
|
+
|
|
384
|
+
def __init__(self, includes: list[str], excludes: list[str], extensions: set[str],
|
|
385
|
+
intents: dict[str, str]):
|
|
386
|
+
self.includes = includes
|
|
387
|
+
self.excludes = excludes
|
|
388
|
+
self.extensions = extensions
|
|
389
|
+
self.intents = intents
|
|
390
|
+
|
|
391
|
+
def takes(self, rel: str) -> bool:
|
|
392
|
+
"""True when the brief's scope takes `rel`, a path relative to the source root."""
|
|
393
|
+
return _in_scope(_resolver().normalize_rel_path(rel), self.includes, self.excludes, self.extensions)
|
|
394
|
+
|
|
395
|
+
def wants(self, kind: str) -> bool:
|
|
396
|
+
"""False when the brief's intent for `kind` (`script` or `asset`) is `none`."""
|
|
397
|
+
return self.intents.get(kind) != "none"
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _intent(value: object) -> str:
|
|
401
|
+
"""`none` for a `none` intent; `detect` for an absent or free-text one."""
|
|
402
|
+
if isinstance(value, str) and value.strip().lower() == "none":
|
|
403
|
+
return "none"
|
|
404
|
+
return "detect"
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def load_scope(brief_path: Path, cited: set[str] | frozenset = frozenset()) -> Scope:
|
|
408
|
+
"""The brief's Scope: its include and exclude patterns, normalized as
|
|
409
|
+
classify normalizes them, the extensions an empty include falls back to
|
|
410
|
+
(the cited paths' and the brief language's) and the script and asset
|
|
411
|
+
intents. Raises ValueError for a brief path that names no file or a
|
|
412
|
+
brief it cannot read."""
|
|
413
|
+
try:
|
|
414
|
+
found = brief_path.is_file()
|
|
415
|
+
except OSError as exc: # a folder on the way that cannot be searched
|
|
416
|
+
raise ValueError(f"cannot read the skill brief {brief_path}: {exc}") from exc
|
|
417
|
+
if not found:
|
|
418
|
+
raise ValueError(f"brief not found: {brief_path}")
|
|
419
|
+
resolver = _resolver()
|
|
420
|
+
brief = resolver.load_brief(brief_path)
|
|
421
|
+
includes, excludes, _ = resolver.extract_scope(brief)
|
|
422
|
+
norm = resolver.normalize_rel_path
|
|
423
|
+
includes = [norm(p) for p in includes]
|
|
424
|
+
excludes = [norm(p) for p in excludes]
|
|
425
|
+
extensions: set[str] = set()
|
|
426
|
+
if not includes:
|
|
427
|
+
extensions = {PurePosixPath(p).suffix for p in cited if PurePosixPath(p).suffix}
|
|
428
|
+
language = brief.get("language")
|
|
429
|
+
if isinstance(language, str) and language.strip():
|
|
430
|
+
extensions |= language_extensions([language])[0]
|
|
431
|
+
intents = {kind: _intent(brief.get(field)) for kind, field in INTENT_FIELDS.items()}
|
|
432
|
+
return Scope(includes, excludes, extensions, intents)
|
|
433
|
+
|
|
434
|
+
|
|
357
435
|
# --------------------------------------------------------------------------
|
|
358
436
|
# Same-content moves (diff mode)
|
|
359
437
|
# --------------------------------------------------------------------------
|
|
@@ -1,34 +1,62 @@
|
|
|
1
1
|
# /// script
|
|
2
2
|
# requires-python = ">=3.11"
|
|
3
|
-
# dependencies = []
|
|
3
|
+
# dependencies = ["pyyaml"]
|
|
4
4
|
# ///
|
|
5
|
-
"""SKF Compare File Hashes
|
|
5
|
+
"""SKF Compare File Hashes: script/asset/doc drift detection for audit-skill.
|
|
6
6
|
|
|
7
7
|
This script replaces the per-file hash-compute-and-classify prose at
|
|
8
8
|
`src/skf-audit-skill/references/structural-diff.md` §4b (Script/Asset Drift).
|
|
9
9
|
It compares the `file_entries[]` block of a provenance map against the
|
|
10
10
|
current state of the source tree and emits three drift categories:
|
|
11
11
|
|
|
12
|
-
- **added**: files present in the source tree but NOT
|
|
13
|
-
`file_entries[]
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
- **added**: with --brief, files present in the source tree but NOT
|
|
13
|
+
recorded in `file_entries[]` that the skill brief takes.
|
|
14
|
+
The walk finds scripts and assets with the detector
|
|
15
|
+
create-skill and update-skill use
|
|
16
|
+
(skf-detect-scripts-assets.py, loaded from this folder:
|
|
17
|
+
its iter_files, is_binary_path, entry_point_script_paths
|
|
18
|
+
and file_kind), so it takes what update-skill's Category D
|
|
19
|
+
takes: a file under a script or asset folder at any depth,
|
|
20
|
+
a shebang file or a `package.json` `bin` entry point in
|
|
21
|
+
any folder, and a pattern asset (`*.schema.json`,
|
|
22
|
+
`openapi.*`, `*.example`, ...) anywhere, skipping binary
|
|
23
|
+
extensions, generated paths and symbolic links, and
|
|
24
|
+
leaving out a module of a Python package named like a
|
|
25
|
+
script folder, which is code unless it runs on its own.
|
|
26
|
+
It also takes each docs/authoritative/ file, a document.
|
|
27
|
+
A file under `scripts/[MANUAL]/` or `assets/[MANUAL]/` is
|
|
28
|
+
user-authored, never added (Category D sets it aside too).
|
|
29
|
+
Without --brief, or with a brief it cannot read, no new
|
|
30
|
+
file is looked for: `added` is empty and
|
|
31
|
+
`added_not_checked` says why.
|
|
18
32
|
- **removed**: files in `file_entries[]` whose `source_file` is gone
|
|
19
33
|
from disk under `<source-root>`.
|
|
20
34
|
- **changed**: files in `file_entries[]` present on disk but with a
|
|
21
35
|
different content hash than the stored hash.
|
|
22
36
|
|
|
37
|
+
--brief <skill-brief.yaml> keeps a walked file as added only when the
|
|
38
|
+
brief's scope takes it, by the one brief-scope test of
|
|
39
|
+
skf-classify-changed-files.py (load_scope, loaded from this folder only
|
|
40
|
+
when --brief is given), which update-skill's Category A and Category D
|
|
41
|
+
and create-skill's script and asset detector use too: with the glob rules
|
|
42
|
+
of skf-resolve-authoritative-files.py, and, for an empty `scope.include`,
|
|
43
|
+
the extensions of the map's `entries[]` files and of the brief's
|
|
44
|
+
`language`. A file the map's `entries[]` cite is code, never an added file
|
|
45
|
+
(update-skill's `tracked_code`). A walked file is a script or an asset by
|
|
46
|
+
the detector's file_kind, a script first (a shebang file under an asset
|
|
47
|
+
folder is a script), except a docs/authoritative/ file, a document
|
|
48
|
+
whatever the detector says: a script or an asset whose `scripts_intent` or
|
|
49
|
+
`assets_intent` is `none` is no added file, and no intent filters a
|
|
50
|
+
document.
|
|
51
|
+
|
|
23
52
|
Distinct from `skf-hash-content.py compare`: that script classifies entries
|
|
24
53
|
that ARE in the provenance map (UNCHANGED / MODIFIED_FILE / DELETED_FILE).
|
|
25
|
-
audit-skill additionally needs the **inverse walk
|
|
54
|
+
audit-skill additionally needs the **inverse walk**: what new
|
|
26
55
|
script/asset/doc files have appeared in the source tree since the skill
|
|
27
|
-
was created? That's `added[]
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
in `skf-detect-scripts-assets.py:216`.
|
|
56
|
+
was created? That's `added[]`, the set update-skill's Category D pipes
|
|
57
|
+
the detector into skf-new-file-diff.py for. The sha256 helper is the one
|
|
58
|
+
skf-detect-scripts-assets.py has, kept here so hashing, and so removed and
|
|
59
|
+
changed, need no other file.
|
|
32
60
|
|
|
33
61
|
Hash-prefix normalization (writer-vs-reader compatibility): stored hashes
|
|
34
62
|
in `file_entries[].content_hash` carry a `sha256:` prefix by SKF convention
|
|
@@ -40,10 +68,11 @@ the original form. A bare-hex hash from a future writer would pass through
|
|
|
40
68
|
unchanged.
|
|
41
69
|
|
|
42
70
|
Subcommand:
|
|
43
|
-
compare <provenance-map.json> <source-root>
|
|
71
|
+
compare <provenance-map.json> <source-root> [--brief <skill-brief.yaml>]
|
|
44
72
|
Emit JSON:
|
|
45
73
|
{
|
|
46
74
|
"added": ["<rel-path, forward-slash>", ...],
|
|
75
|
+
"added_not_checked": null | "<reason>",
|
|
47
76
|
"removed": ["<rel-path, forward-slash>", ...],
|
|
48
77
|
"changed": [
|
|
49
78
|
{"path": "<rel-path>", "stored_hash": "sha256:...",
|
|
@@ -54,55 +83,100 @@ Subcommand:
|
|
|
54
83
|
|
|
55
84
|
All paths are forward-slash relative to `<source-root>`. Stable sort
|
|
56
85
|
on the lists for deterministic output (lexicographic on path).
|
|
86
|
+
`added_not_checked` is null when the walk ran, else the reason no
|
|
87
|
+
new file was looked for, on one line: `no skill brief` without
|
|
88
|
+
--brief (or with an empty one), the error the brief gave
|
|
89
|
+
(`brief not found: <path>`, or a brief that is not valid YAML or no
|
|
90
|
+
mapping), or `cannot load <helper> beside skf-compare-file-hashes.py:
|
|
91
|
+
...; re-install SKF` when a helper of this folder the walk needs
|
|
92
|
+
cannot be loaded or is out of date. `stats.added` counts `added`.
|
|
57
93
|
|
|
58
94
|
CLI examples:
|
|
59
|
-
uv run skf-compare-file-hashes.py compare prov-map.json /path/to/src
|
|
95
|
+
uv run skf-compare-file-hashes.py compare prov-map.json /path/to/src \\
|
|
96
|
+
--brief skill-brief.yaml
|
|
60
97
|
|
|
61
98
|
Exit codes:
|
|
62
|
-
0
|
|
63
|
-
|
|
64
|
-
|
|
99
|
+
0 comparison succeeded (including empty/well-formed input that yields
|
|
100
|
+
all-empty drift lists, and a brief or a helper of this folder it
|
|
101
|
+
cannot read or load: see `added_not_checked`)
|
|
102
|
+
1 user error (missing files, malformed JSON, unreadable paths)
|
|
65
103
|
"""
|
|
66
104
|
|
|
67
105
|
from __future__ import annotations
|
|
68
106
|
|
|
69
107
|
import argparse
|
|
70
108
|
import hashlib
|
|
109
|
+
import importlib.util
|
|
71
110
|
import json
|
|
72
111
|
import re
|
|
73
112
|
import sys
|
|
74
113
|
from pathlib import Path
|
|
75
|
-
from typing import Iterable
|
|
114
|
+
from typing import Callable, Iterable
|
|
76
115
|
|
|
77
116
|
|
|
78
117
|
# --------------------------------------------------------------------------
|
|
79
|
-
# Constants
|
|
80
|
-
#
|
|
118
|
+
# Constants: the walk's script and asset rules are the detector's
|
|
119
|
+
# (skf-detect-scripts-assets.py), loaded from this folder, not copies.
|
|
81
120
|
# --------------------------------------------------------------------------
|
|
82
121
|
|
|
83
122
|
|
|
84
|
-
SCRIPT_DIRS = {"scripts", "bin", "tools", "cli"}
|
|
85
|
-
ASSET_DIRS = {"assets", "templates", "schemas", "configs", "examples"}
|
|
86
123
|
DOC_DIR_PREFIXES = ("docs/authoritative/",) # synthetic namespace from create-skill §6
|
|
87
124
|
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
"target", ".next", ".nuxt", "out", "coverage", ".git",
|
|
93
|
-
".venv", "venv", ".tox", ".mypy_cache", ".pytest_cache",
|
|
94
|
-
".ruff_cache", ".gradle", ".idea", ".vscode",
|
|
95
|
-
}
|
|
125
|
+
# scripts/[MANUAL]/... or assets/[MANUAL]/... anywhere in the (posix) path:
|
|
126
|
+
# a copy of skf-new-file-diff.py's _MANUAL_RE (update-skill's Category D),
|
|
127
|
+
# which a test keeps equal to it.
|
|
128
|
+
_MANUAL_RE = re.compile(r"(?:^|/)(?:scripts|assets)/\[MANUAL\]/")
|
|
96
129
|
|
|
97
|
-
|
|
98
|
-
".so", ".dll", ".jar", ".wasm", ".exe", ".dylib", ".a", ".o",
|
|
99
|
-
".pyc", ".class", ".png", ".jpg", ".jpeg", ".gif", ".ico",
|
|
100
|
-
".pdf", ".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".7z",
|
|
101
|
-
".woff", ".woff2", ".ttf", ".otf", ".eot",
|
|
102
|
-
}
|
|
130
|
+
_HASH_PREFIX_RE = re.compile(r"^[a-z0-9]+:")
|
|
103
131
|
|
|
132
|
+
DETECTOR = "skf-detect-scripts-assets.py" # the walk (--brief): see _detector
|
|
133
|
+
CLASSIFIER = "skf-classify-changed-files.py" # load_scope: the brief's scope (--brief)
|
|
134
|
+
NO_BRIEF = "no skill brief"
|
|
104
135
|
|
|
105
|
-
|
|
136
|
+
_SIBLINGS: dict[str, object] = {}
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class HelperError(Exception):
|
|
140
|
+
"""A helper beside this script that cannot be loaded, lacks what this
|
|
141
|
+
script calls or fails in it: no new file can be looked for."""
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _helper_error(filename: str, exc: BaseException) -> HelperError:
|
|
145
|
+
return HelperError(f"cannot load {filename} beside {Path(__file__).name}: {exc}; re-install SKF")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _sibling(filename: str):
|
|
149
|
+
"""A helper installed beside this script, loaded once as a module.
|
|
150
|
+
Raises HelperError when it cannot be loaded."""
|
|
151
|
+
module = _SIBLINGS.get(filename)
|
|
152
|
+
if module is None:
|
|
153
|
+
path = Path(__file__).resolve().parent / filename
|
|
154
|
+
name = "skf_" + filename.removeprefix("skf-").removesuffix(".py").replace("-", "_")
|
|
155
|
+
try:
|
|
156
|
+
spec = importlib.util.spec_from_file_location(name, path)
|
|
157
|
+
if spec is None or spec.loader is None or not path.is_file():
|
|
158
|
+
raise ImportError(f"{filename} not found")
|
|
159
|
+
module = importlib.util.module_from_spec(spec)
|
|
160
|
+
spec.loader.exec_module(module)
|
|
161
|
+
except Exception as exc: # whatever the helper raises, the run reports it
|
|
162
|
+
raise _helper_error(filename, exc) from exc
|
|
163
|
+
_SIBLINGS[filename] = module
|
|
164
|
+
return module
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _posix(p: str) -> str:
|
|
168
|
+
"""A path as skf-resolve-authoritative-files.py's normalize_rel_path
|
|
169
|
+
writes it: stripped, forward slashes, no leading `./`."""
|
|
170
|
+
p = p.strip().replace("\\", "/")
|
|
171
|
+
while p.startswith("./"):
|
|
172
|
+
p = p[2:]
|
|
173
|
+
return p
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _one_line(message: str) -> str:
|
|
177
|
+
"""Each line break of `message` and the space around it as one space
|
|
178
|
+
(a YAML error spans several lines)."""
|
|
179
|
+
return re.sub(r"\s*[\r\n]\s*", " ", message.strip("\r\n"))
|
|
106
180
|
|
|
107
181
|
|
|
108
182
|
# --------------------------------------------------------------------------
|
|
@@ -136,16 +210,8 @@ def normalize_hash(value: str | None) -> str | None:
|
|
|
136
210
|
# --------------------------------------------------------------------------
|
|
137
211
|
|
|
138
212
|
|
|
139
|
-
def
|
|
140
|
-
"""
|
|
141
|
-
- top-level object with a `file_entries` key (canonical)
|
|
142
|
-
- top-level array (already extracted)
|
|
143
|
-
- top-level object with NO `file_entries` field → empty list
|
|
144
|
-
(a single-skill with no scripts/assets/docs may omit the field
|
|
145
|
-
entirely per skill-sections.md §file_entries)
|
|
146
|
-
|
|
147
|
-
Raises ValueError on read failure or structural defects.
|
|
148
|
-
"""
|
|
213
|
+
def _read_map(provenance_path: Path) -> object:
|
|
214
|
+
"""The provenance map's JSON. Raises ValueError when it cannot be read."""
|
|
149
215
|
try:
|
|
150
216
|
text = provenance_path.read_text(encoding="utf-8")
|
|
151
217
|
except OSError as exc:
|
|
@@ -153,12 +219,27 @@ def load_file_entries(provenance_path: Path) -> list[dict]:
|
|
|
153
219
|
f"failed to read provenance file {provenance_path}: {exc}"
|
|
154
220
|
) from exc
|
|
155
221
|
try:
|
|
156
|
-
|
|
222
|
+
return json.loads(text)
|
|
157
223
|
except json.JSONDecodeError as exc:
|
|
158
224
|
raise ValueError(
|
|
159
225
|
f"malformed JSON in provenance file {provenance_path}: {exc}"
|
|
160
226
|
) from exc
|
|
161
227
|
|
|
228
|
+
|
|
229
|
+
def load_file_entries(provenance_path: Path) -> list[dict]:
|
|
230
|
+
"""Extract `file_entries[]` from a provenance map. Accepts:
|
|
231
|
+
- top-level object with a `file_entries` key (canonical)
|
|
232
|
+
- top-level array (already extracted)
|
|
233
|
+
- top-level object with NO `file_entries` field → empty list
|
|
234
|
+
(a single-skill with no scripts/assets/docs may omit the field
|
|
235
|
+
entirely per skill-sections.md §file_entries)
|
|
236
|
+
|
|
237
|
+
Raises ValueError on read failure or structural defects.
|
|
238
|
+
"""
|
|
239
|
+
return _file_entries(_read_map(provenance_path), provenance_path)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _file_entries(data: object, provenance_path: Path) -> list[dict]:
|
|
162
243
|
if isinstance(data, list):
|
|
163
244
|
return list(data)
|
|
164
245
|
if isinstance(data, dict):
|
|
@@ -176,69 +257,96 @@ def load_file_entries(provenance_path: Path) -> list[dict]:
|
|
|
176
257
|
)
|
|
177
258
|
|
|
178
259
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
return
|
|
260
|
+
def _cited(data: object) -> set[str]:
|
|
261
|
+
"""The map's `entries[].source_file`, the code files it cites (none for
|
|
262
|
+
a bare array, or an `entries` that is not a list)."""
|
|
263
|
+
rows = data.get("entries") if isinstance(data, dict) else None
|
|
264
|
+
if not isinstance(rows, list):
|
|
265
|
+
return set()
|
|
266
|
+
return {
|
|
267
|
+
_posix(row["source_file"])
|
|
268
|
+
for row in rows
|
|
269
|
+
if isinstance(row, dict) and isinstance(row.get("source_file"), str) and _posix(row["source_file"])
|
|
270
|
+
}
|
|
186
271
|
|
|
187
272
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
for seg in rel_parts:
|
|
192
|
-
if seg in SCRIPT_DIRS or seg in ASSET_DIRS:
|
|
193
|
-
return True
|
|
194
|
-
return False
|
|
273
|
+
# --------------------------------------------------------------------------
|
|
274
|
+
# Walk: the inverse direction (source tree to candidate set)
|
|
275
|
+
# --------------------------------------------------------------------------
|
|
195
276
|
|
|
196
277
|
|
|
197
278
|
def _matches_doc_prefix(rel_posix: str) -> bool:
|
|
198
279
|
return any(rel_posix.startswith(p) for p in DOC_DIR_PREFIXES)
|
|
199
280
|
|
|
200
281
|
|
|
282
|
+
def _detector(source_root: Path) -> tuple:
|
|
283
|
+
"""(iter_files, is_binary_path, file_kind,
|
|
284
|
+
entry_point_script_paths(source_root)) from the detector, read before
|
|
285
|
+
any file is walked. Raises HelperError when the detector cannot be
|
|
286
|
+
loaded, lacks one of these four functions or fails in
|
|
287
|
+
entry_point_script_paths."""
|
|
288
|
+
detector = _sibling(DETECTOR)
|
|
289
|
+
try:
|
|
290
|
+
return (detector.iter_files, detector.is_binary_path, detector.file_kind,
|
|
291
|
+
detector.entry_point_script_paths(source_root))
|
|
292
|
+
except Exception as exc: # an out-of-date detector, or one that fails
|
|
293
|
+
raise _helper_error(DETECTOR, exc) from exc
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _walk(source_root: Path, keep: Callable[[str], bool] | None = None) -> Iterable[tuple[str, str]]:
|
|
297
|
+
"""(POSIX relative path, kind) of each file under source_root that could
|
|
298
|
+
be tracked as a script, asset or doc file_entries row, by the detector's
|
|
299
|
+
walk (iter_files, is_binary_path): a docs/authoritative/ file is a
|
|
300
|
+
`doc`, whatever the detector says, and any other file the kind the
|
|
301
|
+
detector's file_kind gives it, both kinds on (`script` or `asset`), or
|
|
302
|
+
left out. `keep`, a test of the relative path, runs before the doc rule
|
|
303
|
+
and file_kind, so a file it refuses is never read. The detector is
|
|
304
|
+
loaded when the walk starts. Raises HelperError when it cannot be
|
|
305
|
+
loaded, lacks a function the walk calls or fails in one, and lets
|
|
306
|
+
through a HelperError `keep` raises."""
|
|
307
|
+
iter_files, is_binary_path, file_kind, entry_points = _detector(source_root)
|
|
308
|
+
try:
|
|
309
|
+
for entry in iter_files(source_root):
|
|
310
|
+
if is_binary_path(entry):
|
|
311
|
+
continue
|
|
312
|
+
rel = entry.relative_to(source_root).as_posix()
|
|
313
|
+
if keep is not None and not keep(rel):
|
|
314
|
+
continue
|
|
315
|
+
if _matches_doc_prefix(rel):
|
|
316
|
+
yield rel, "doc"
|
|
317
|
+
continue
|
|
318
|
+
found = file_kind(entry, source_root, entry_points)
|
|
319
|
+
if found is not None:
|
|
320
|
+
yield rel, found[0]
|
|
321
|
+
except HelperError:
|
|
322
|
+
raise
|
|
323
|
+
except Exception as exc: # the detector failing on a file
|
|
324
|
+
raise _helper_error(DETECTOR, exc) from exc
|
|
325
|
+
|
|
326
|
+
|
|
201
327
|
def candidate_source_files(source_root: Path) -> Iterable[str]:
|
|
202
328
|
"""Yield POSIX relative paths under source_root that could be tracked
|
|
203
|
-
as script / asset / doc file_entries.
|
|
329
|
+
as script / asset / doc file_entries (see _walk).
|
|
204
330
|
|
|
205
331
|
Selection rules:
|
|
206
|
-
-
|
|
207
|
-
|
|
332
|
+
- A script or an asset by skf-detect-scripts-assets.py's file_kind:
|
|
333
|
+
a file under a script or asset folder at any depth, a shebang file
|
|
334
|
+
or a `package.json` `bin` entry point in any folder, a pattern
|
|
335
|
+
asset (`*.schema.json`, `openapi.*`, `*.example`, ...) anywhere;
|
|
336
|
+
a module of a Python package named like a script folder is code,
|
|
337
|
+
not a script, unless an asset folder holds it or its name is an
|
|
338
|
+
asset pattern.
|
|
339
|
+
- Or a file under a DOC_DIR_PREFIXES synthetic namespace.
|
|
208
340
|
- Binary extensions are excluded.
|
|
209
341
|
- Generated/vendored directories are pruned from the walk.
|
|
210
342
|
|
|
211
|
-
This is deliberately narrower than "every file"
|
|
343
|
+
This is deliberately narrower than "every file": audit-skill §4b only
|
|
212
344
|
tracks files matching script/asset patterns. Treating every random
|
|
213
345
|
source file as a candidate `added` row would drown real drift in noise.
|
|
346
|
+
Raises HelperError when the detector cannot be loaded.
|
|
214
347
|
"""
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
current = stack.pop()
|
|
218
|
-
try:
|
|
219
|
-
entries = list(current.iterdir())
|
|
220
|
-
except (PermissionError, FileNotFoundError):
|
|
221
|
-
continue
|
|
222
|
-
for entry in entries:
|
|
223
|
-
if entry.is_symlink():
|
|
224
|
-
continue
|
|
225
|
-
if entry.is_dir():
|
|
226
|
-
if entry.name in EXCLUDED_DIR_NAMES:
|
|
227
|
-
continue
|
|
228
|
-
stack.append(entry)
|
|
229
|
-
continue
|
|
230
|
-
if not entry.is_file():
|
|
231
|
-
continue
|
|
232
|
-
if entry.suffix.lower() in BINARY_EXTS:
|
|
233
|
-
continue
|
|
234
|
-
try:
|
|
235
|
-
rel = entry.relative_to(source_root)
|
|
236
|
-
except ValueError:
|
|
237
|
-
continue
|
|
238
|
-
rel_parts = rel.parts[:-1]
|
|
239
|
-
rel_posix = rel.as_posix()
|
|
240
|
-
if _segment_in_tracked_dir(rel_parts) or _matches_doc_prefix(rel_posix):
|
|
241
|
-
yield rel_posix
|
|
348
|
+
for rel, _kind_ in _walk(source_root):
|
|
349
|
+
yield rel
|
|
242
350
|
|
|
243
351
|
|
|
244
352
|
# --------------------------------------------------------------------------
|
|
@@ -246,14 +354,19 @@ def candidate_source_files(source_root: Path) -> Iterable[str]:
|
|
|
246
354
|
# --------------------------------------------------------------------------
|
|
247
355
|
|
|
248
356
|
|
|
249
|
-
def compare(source_root: Path, provenance_path: Path) -> dict:
|
|
357
|
+
def compare(source_root: Path, provenance_path: Path, brief_path: Path | None = None) -> dict:
|
|
250
358
|
"""Compute added / removed / changed / unchanged for tracked file_entries.
|
|
251
359
|
|
|
252
360
|
Stored hashes are normalized before comparison; the diff record carries
|
|
253
361
|
the original stored value (writer-form) plus the freshly computed
|
|
254
|
-
current value (always prefixed sha256:) so reviewers see both.
|
|
362
|
+
current value (always prefixed sha256:) so reviewers see both. Added
|
|
363
|
+
files are looked for only with a brief it can read (`brief_path`):
|
|
364
|
+
otherwise `added` is empty and `added_not_checked` says why, as it
|
|
365
|
+
does when a helper the walk needs cannot be loaded. Raises ValueError
|
|
366
|
+
for a map it cannot read.
|
|
255
367
|
"""
|
|
256
|
-
|
|
368
|
+
data = _read_map(provenance_path)
|
|
369
|
+
entries = _file_entries(data, provenance_path)
|
|
257
370
|
# Build the {posix_path -> stored_hash} index from the provenance.
|
|
258
371
|
stored: dict[str, str | None] = {}
|
|
259
372
|
for entry in entries:
|
|
@@ -266,9 +379,6 @@ def compare(source_root: Path, provenance_path: Path) -> dict:
|
|
|
266
379
|
entry.get("content_hash"), str
|
|
267
380
|
) else None
|
|
268
381
|
|
|
269
|
-
# Walk the source tree once; track which provenance entries are matched.
|
|
270
|
-
candidates_on_disk = set(candidate_source_files(source_root))
|
|
271
|
-
|
|
272
382
|
removed: list[str] = []
|
|
273
383
|
changed: list[dict] = []
|
|
274
384
|
unchanged_count = 0
|
|
@@ -288,11 +398,11 @@ def compare(source_root: Path, provenance_path: Path) -> dict:
|
|
|
288
398
|
"current_hash": current,
|
|
289
399
|
})
|
|
290
400
|
|
|
291
|
-
|
|
292
|
-
added = sorted(candidates_on_disk - set(stored.keys()))
|
|
401
|
+
added, not_checked = _added(source_root, data, stored, brief_path)
|
|
293
402
|
|
|
294
403
|
return {
|
|
295
404
|
"added": added,
|
|
405
|
+
"added_not_checked": not_checked,
|
|
296
406
|
"removed": sorted(removed),
|
|
297
407
|
"changed": sorted(changed, key=lambda r: r["path"]),
|
|
298
408
|
"stats": {
|
|
@@ -304,6 +414,50 @@ def compare(source_root: Path, provenance_path: Path) -> dict:
|
|
|
304
414
|
}
|
|
305
415
|
|
|
306
416
|
|
|
417
|
+
def _added(source_root: Path, data: object, stored: dict, brief_path: Path | None) -> tuple[list[str], str | None]:
|
|
418
|
+
"""(added, added_not_checked): the walked files the map does not track
|
|
419
|
+
and the brief takes, or ([], the reason) without a brief it can read or
|
|
420
|
+
a helper the walk needs. removed and changed need neither. The brief's
|
|
421
|
+
scope is loaded before the detector, and the path rules (tracked or
|
|
422
|
+
cited, [MANUAL], the brief's scope) run before the detector reads a
|
|
423
|
+
file."""
|
|
424
|
+
if brief_path is None:
|
|
425
|
+
return [], NO_BRIEF
|
|
426
|
+
cited = _cited(data)
|
|
427
|
+
try:
|
|
428
|
+
try:
|
|
429
|
+
scope = _sibling(CLASSIFIER).load_scope(brief_path, cited)
|
|
430
|
+
except ValueError as exc: # a brief it cannot find or read
|
|
431
|
+
return [], _one_line(str(exc))
|
|
432
|
+
except HelperError:
|
|
433
|
+
raise
|
|
434
|
+
except Exception as exc: # an out-of-date classifier, or a resolver beside it that cannot load
|
|
435
|
+
raise _helper_error(CLASSIFIER, exc) from exc
|
|
436
|
+
known = {_posix(path) for path in stored}
|
|
437
|
+
|
|
438
|
+
def keep(rel: str) -> bool:
|
|
439
|
+
if rel in known or rel in cited: # tracked, or code the map's entries[] cite
|
|
440
|
+
return False
|
|
441
|
+
if _MANUAL_RE.search(rel): # user-authored, as Category D's skipped_manual
|
|
442
|
+
return False
|
|
443
|
+
return _scope_call(scope.takes, rel)
|
|
444
|
+
|
|
445
|
+
added = [rel for rel, kind in _walk(source_root, keep)
|
|
446
|
+
if kind == "doc" or _scope_call(scope.wants, kind)] # no intent filters a document
|
|
447
|
+
except HelperError as exc:
|
|
448
|
+
return [], _one_line(str(exc))
|
|
449
|
+
return sorted(added), None
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _scope_call(method, value) -> bool:
|
|
453
|
+
"""A Scope method's answer. Raises HelperError when it fails (the
|
|
454
|
+
resolver it calls cannot be loaded or is out of date)."""
|
|
455
|
+
try:
|
|
456
|
+
return bool(method(value))
|
|
457
|
+
except Exception as exc:
|
|
458
|
+
raise _helper_error(CLASSIFIER, exc) from exc
|
|
459
|
+
|
|
460
|
+
|
|
307
461
|
# --------------------------------------------------------------------------
|
|
308
462
|
# CLI
|
|
309
463
|
# --------------------------------------------------------------------------
|
|
@@ -318,8 +472,9 @@ def _cmd_compare(args: argparse.Namespace) -> int:
|
|
|
318
472
|
if not source_root.is_dir():
|
|
319
473
|
print(f"error: source root not a directory: {source_root}", file=sys.stderr)
|
|
320
474
|
return 1
|
|
475
|
+
brief = Path(args.brief) if args.brief is not None and args.brief.strip() else None
|
|
321
476
|
try:
|
|
322
|
-
result = compare(source_root, provenance)
|
|
477
|
+
result = compare(source_root, provenance, brief)
|
|
323
478
|
except ValueError as exc:
|
|
324
479
|
print(f"error: {exc}", file=sys.stderr)
|
|
325
480
|
return 1
|
|
@@ -343,6 +498,13 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
343
498
|
)
|
|
344
499
|
p_cmp.add_argument("provenance_map", help="path to provenance-map.json")
|
|
345
500
|
p_cmp.add_argument("source_root", help="path to the source tree root")
|
|
501
|
+
p_cmp.add_argument(
|
|
502
|
+
"--brief",
|
|
503
|
+
default=None,
|
|
504
|
+
metavar="BRIEF",
|
|
505
|
+
help="the skill brief (skill-brief.yaml): look for added files only in its scope "
|
|
506
|
+
"(none, or one it cannot read: no added file is looked for)",
|
|
507
|
+
)
|
|
346
508
|
p_cmp.set_defaults(func=_cmd_compare)
|
|
347
509
|
return parser
|
|
348
510
|
|
|
@@ -46,6 +46,7 @@ Exit codes:
|
|
|
46
46
|
from __future__ import annotations
|
|
47
47
|
|
|
48
48
|
import argparse
|
|
49
|
+
import importlib.util
|
|
49
50
|
import json
|
|
50
51
|
import sys
|
|
51
52
|
from pathlib import Path
|
|
@@ -53,6 +54,8 @@ from typing import Any
|
|
|
53
54
|
|
|
54
55
|
import yaml
|
|
55
56
|
|
|
57
|
+
RESOLVER = Path(__file__).resolve().parent / "skf-resolve-authoritative-files.py"
|
|
58
|
+
|
|
56
59
|
|
|
57
60
|
def derive_assembly_shape(brief: dict[str, Any]) -> dict[str, Any]:
|
|
58
61
|
"""Return the assembly-shape decision for a parsed brief."""
|
|
@@ -73,6 +76,20 @@ def _die(message: str) -> None:
|
|
|
73
76
|
sys.exit(2)
|
|
74
77
|
|
|
75
78
|
|
|
79
|
+
def _parse_brief_yaml(text: str):
|
|
80
|
+
"""The brief's YAML as skf-resolve-authoritative-files.py (from this
|
|
81
|
+
folder) reads it: an SKF 1.x brief's trailing `---` is no second document."""
|
|
82
|
+
spec = importlib.util.spec_from_file_location("skf_resolve_authoritative_files", RESOLVER)
|
|
83
|
+
if spec is None or spec.loader is None:
|
|
84
|
+
_die(f"cannot load {RESOLVER.name} beside {Path(__file__).name}")
|
|
85
|
+
resolver = importlib.util.module_from_spec(spec)
|
|
86
|
+
try:
|
|
87
|
+
spec.loader.exec_module(resolver)
|
|
88
|
+
except (OSError, ImportError, SyntaxError) as exc:
|
|
89
|
+
_die(f"cannot load {RESOLVER.name} beside {Path(__file__).name}: {exc}")
|
|
90
|
+
return resolver.parse_brief_yaml(text)
|
|
91
|
+
|
|
92
|
+
|
|
76
93
|
def main(argv: list[str] | None = None) -> int:
|
|
77
94
|
parser = argparse.ArgumentParser(
|
|
78
95
|
prog="skf-derive-assembly-shape",
|
|
@@ -93,7 +110,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
93
110
|
text = p.read_text(encoding="utf-8")
|
|
94
111
|
|
|
95
112
|
try:
|
|
96
|
-
brief =
|
|
113
|
+
brief = _parse_brief_yaml(text)
|
|
97
114
|
except yaml.YAMLError as exc:
|
|
98
115
|
_die(f"Brief is not valid YAML: {exc}")
|
|
99
116
|
if not isinstance(brief, dict):
|