bmad-module-skill-forge 3.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/docs/_data/pinned.yaml +1 -1
- package/docs/examples.md +1 -1
- package/docs/troubleshooting.md +8 -0
- package/docs/verifying-a-skill.md +7 -3
- package/docs/workflows.md +3 -3
- package/package.json +1 -1
- package/src/knowledge/qmd-registry.md +1 -1
- package/src/knowledge/version-paths.md +1 -1
- package/src/shared/health-check.md +2 -50
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +2 -2
- package/src/shared/scripts/skf-atomic-write.py +212 -59
- package/src/shared/scripts/skf-build-change-manifest.py +593 -47
- package/src/shared/scripts/skf-classify-changed-files.py +78 -0
- package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
- package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
- package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
- package/src/shared/scripts/skf-extract-public-api.py +1309 -157
- package/src/shared/scripts/skf-extraction-inventory.py +580 -46
- package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
- package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
- package/src/shared/scripts/skf-github-probe.py +47 -8
- package/src/shared/scripts/skf-hash-content.py +14 -8
- package/src/shared/scripts/skf-load-provenance.py +3 -1
- package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
- package/src/shared/scripts/skf-preflight.py +15 -1
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
- package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
- package/src/shared/scripts/skf-structural-diff.py +133 -18
- package/src/shared/scripts/skf-update-active-symlink.py +115 -62
- package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
- package/src/shared/scripts/skf-verify-provenance-completeness.py +977 -49
- package/src/shared/scripts/skf-write-skill-brief.py +41 -8
- package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
- package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
- package/src/skf-audit-skill/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +2 -1
- package/src/skf-audit-skill/references/re-index.md +14 -9
- package/src/skf-audit-skill/references/report.md +2 -2
- package/src/skf-audit-skill/references/semantic-diff.md +15 -15
- package/src/skf-audit-skill/references/structural-diff.md +10 -5
- package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
- package/src/skf-brief-skill/SKILL.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +1 -1
- package/src/skf-brief-skill/assets/skill-brief-schema.md +2 -9
- package/src/skf-brief-skill/references/analyze-target.md +6 -6
- package/src/skf-brief-skill/references/confirm-brief.md +1 -1
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
- package/src/skf-brief-skill/references/version-resolution.md +5 -9
- package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
- package/src/skf-create-skill/assets/skill-sections.md +6 -6
- package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
- package/src/skf-create-skill/references/compile.md +31 -60
- package/src/skf-create-skill/references/component-extraction.md +3 -3
- package/src/skf-create-skill/references/enrich.md +2 -2
- package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
- package/src/skf-create-skill/references/extract.md +22 -19
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
- package/src/skf-create-skill/references/extraction-patterns.md +8 -9
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +16 -5
- package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
- package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
- package/src/skf-drop-skill/references/execute.md +5 -1
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +2 -2
- package/src/skf-drop-skill/references/select.md +7 -3
- package/src/skf-test-skill/references/coherence-check.md +1 -1
- package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
- package/src/skf-test-skill/references/coverage-check.md +39 -57
- package/src/skf-test-skill/references/init.md +3 -5
- package/src/skf-test-skill/references/report.md +10 -8
- package/src/skf-test-skill/references/scoring-rules.md +6 -5
- package/src/skf-test-skill/references/source-access-protocol.md +14 -17
- package/src/skf-test-skill/scripts/build-result-context.py +114 -21
- package/src/skf-test-skill/scripts/gap-ledger.py +315 -30
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +534 -96
- package/src/skf-test-skill/scripts/reconcile-coverage.py +276 -2
- package/src/skf-test-skill/scripts/score-signatures.py +36 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
- package/src/skf-update-skill/references/detect-changes.md +32 -13
- package/src/skf-update-skill/references/gap-driven.md +3 -3
- package/src/skf-update-skill/references/init.md +6 -2
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +2 -2
- package/src/skf-update-skill/references/re-extract.md +2 -2
- package/src/skf-update-skill/references/report.md +3 -1
- package/src/skf-update-skill/references/write.md +43 -23
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
|
@@ -1,18 +1,25 @@
|
|
|
1
1
|
# /// script
|
|
2
2
|
# requires-python = ">=3.11"
|
|
3
|
-
# dependencies = []
|
|
3
|
+
# dependencies = ["pyyaml"]
|
|
4
4
|
# ///
|
|
5
5
|
"""SKF Merge Doc URLs — merge corpora-seeded + detected docs for a skill brief.
|
|
6
6
|
|
|
7
7
|
Deterministic merge of an upstream brief's `doc_urls` (registry-seeded
|
|
8
8
|
companion corpora, source: language-registry) with the docs freshly
|
|
9
9
|
discovered by `skf-detect-docs.py` (mapped to the brief `{url, label, source}`
|
|
10
|
-
contract). Replaces the prose dedup
|
|
11
|
-
`skf-brief-skill/references/step-auto-brief.md` §3
|
|
10
|
+
contract). Replaces the prose mapping, dedup and suppression that lived inline
|
|
11
|
+
in `skf-brief-skill/references/step-auto-brief.md` §3. That logic is fiddly
|
|
12
12
|
(normalized-URL collisions, non-corpus path segments, locale duplicates) and
|
|
13
13
|
must behave identically on every run, so it belongs in a tested script.
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
With --detected-file, each skf-detect-docs.py entry {url, detected_via,
|
|
16
|
+
content_type} is first mapped to {url, label, source}: `source` from
|
|
17
|
+
`detected_via` and `label` from `content_type`, by the two tables below (a
|
|
18
|
+
content_type they do not list gives the label ""). An entry whose
|
|
19
|
+
`detected_via` they do not list is never merged: `suppressed` lists it with
|
|
20
|
+
reason unknown-detected-via.
|
|
21
|
+
|
|
22
|
+
Three merge operations, in order:
|
|
16
23
|
|
|
17
24
|
1. Normalized-URL dedup — lowercase host, strip a trailing `/index.html`
|
|
18
25
|
and any trailing `/` before comparing, so a seeded `…/book/` and a
|
|
@@ -41,10 +48,18 @@ DEGRADED case is intentionally out of scope (there is no canonical corpus host
|
|
|
41
48
|
to anchor suppression against).
|
|
42
49
|
|
|
43
50
|
CLI:
|
|
51
|
+
uv run src/shared/scripts/skf-merge-doc-urls.py --scope-type <type> \
|
|
52
|
+
--detected-file <detect-docs.json> --existing-brief <skill-brief.yaml>
|
|
44
53
|
echo '{"scope_type": "...", "existing": [...], "detected": [...]}' \
|
|
45
54
|
| uv run src/shared/scripts/skf-merge-doc-urls.py
|
|
46
55
|
|
|
47
|
-
|
|
56
|
+
Files form (stdin is not read): --detected-file holds skf-detect-docs.py's
|
|
57
|
+
JSON array, mapped as above; --existing-brief is a skill-brief.yaml whose
|
|
58
|
+
`doc_urls` are the existing entries (none when it has no doc_urls), read as
|
|
59
|
+
skf-resolve-authoritative-files.py reads a brief; --scope-type is the scope
|
|
60
|
+
type. Each flag is optional: an absent one gives no entries (or "").
|
|
61
|
+
|
|
62
|
+
Input of the stdin form (a JSON object), whose entries are merged as given:
|
|
48
63
|
{
|
|
49
64
|
"scope_type": "full-library",
|
|
50
65
|
"existing": [{"url": "...", "label": "...", "source": "language-registry"}, ...],
|
|
@@ -54,22 +69,42 @@ Input (JSON object on stdin):
|
|
|
54
69
|
Output (JSON object on stdout):
|
|
55
70
|
{
|
|
56
71
|
"doc_urls": [{"url": "...", "label": "...", "source": "..."}, ...],
|
|
57
|
-
"suppressed": [{"url": "...", "reason":
|
|
72
|
+
"suppressed": [{"url": "...", "reason":
|
|
73
|
+
"unknown-detected-via|non-corpus-segment|non-primary-locale-dup"}, ...]
|
|
58
74
|
}
|
|
59
75
|
|
|
60
76
|
Exit codes:
|
|
61
77
|
0 success (merged list emitted)
|
|
62
|
-
2 error (bad JSON, wrong input shape
|
|
78
|
+
2 error (bad JSON, wrong input shape, a file that cannot be read, a brief
|
|
79
|
+
that is not a YAML mapping)
|
|
63
80
|
"""
|
|
64
81
|
|
|
65
82
|
from __future__ import annotations
|
|
66
83
|
|
|
84
|
+
import argparse
|
|
85
|
+
import importlib.util
|
|
67
86
|
import json
|
|
68
87
|
import re
|
|
69
88
|
import sys
|
|
89
|
+
from pathlib import Path
|
|
70
90
|
from typing import Any
|
|
71
91
|
from urllib.parse import urlsplit
|
|
72
92
|
|
|
93
|
+
# skf-detect-docs.py's `detected_via` -> the brief's coarse provenance, the
|
|
94
|
+
# skill-brief.v1.json doc_urls[].source enum (a test pins it to the schema).
|
|
95
|
+
_SOURCE_BY_DETECTED_VIA = {
|
|
96
|
+
"homepageUrl": "homepage",
|
|
97
|
+
"readme_link": "readme-detection",
|
|
98
|
+
"pages_api": "pages-api",
|
|
99
|
+
"docs_folder": "docs-folder",
|
|
100
|
+
}
|
|
101
|
+
# skf-detect-docs.py's `content_type` -> the entry's display label.
|
|
102
|
+
_LABEL_BY_CONTENT_TYPE = {
|
|
103
|
+
"api-docs": "API Documentation",
|
|
104
|
+
"guide": "Guide",
|
|
105
|
+
"reference": "Reference",
|
|
106
|
+
}
|
|
107
|
+
|
|
73
108
|
# Path components (case-insensitive) that mark a non-corpus page on a doc host:
|
|
74
109
|
# changelogs/news, contribution guides, wikis. Matched as whole path components
|
|
75
110
|
# (or component + a separator), never as a bare substring.
|
|
@@ -153,6 +188,28 @@ def _matches_non_corpus_segment(path: str) -> bool:
|
|
|
153
188
|
return False
|
|
154
189
|
|
|
155
190
|
|
|
191
|
+
def map_detected(raw: list[Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
192
|
+
"""skf-detect-docs.py entries as brief doc_urls entries {url, label, source}.
|
|
193
|
+
|
|
194
|
+
Returns (mapped, suppressed): an entry whose detected_via maps to no
|
|
195
|
+
source is suppressed with reason unknown-detected-via; one with no url is
|
|
196
|
+
skipped, as the merge skips it.
|
|
197
|
+
"""
|
|
198
|
+
mapped: list[dict[str, Any]] = []
|
|
199
|
+
suppressed: list[dict[str, Any]] = []
|
|
200
|
+
for d in raw:
|
|
201
|
+
if not isinstance(d, dict) or not d.get("url"):
|
|
202
|
+
continue
|
|
203
|
+
via, content_type = d.get("detected_via"), d.get("content_type")
|
|
204
|
+
source = _SOURCE_BY_DETECTED_VIA.get(via) if isinstance(via, str) else None
|
|
205
|
+
if source is None:
|
|
206
|
+
suppressed.append({"url": d["url"], "reason": "unknown-detected-via"})
|
|
207
|
+
continue
|
|
208
|
+
label = _LABEL_BY_CONTENT_TYPE.get(content_type, "") if isinstance(content_type, str) else ""
|
|
209
|
+
mapped.append({"url": d["url"], "label": label, "source": source})
|
|
210
|
+
return mapped, suppressed
|
|
211
|
+
|
|
212
|
+
|
|
156
213
|
def merge_doc_urls(
|
|
157
214
|
scope_type: str,
|
|
158
215
|
existing: list[dict[str, Any]],
|
|
@@ -232,30 +289,96 @@ def _force_utf8(*streams) -> None:
|
|
|
232
289
|
stream.reconfigure(encoding="utf-8", errors=errors)
|
|
233
290
|
|
|
234
291
|
|
|
235
|
-
|
|
236
|
-
|
|
292
|
+
class InputError(Exception):
|
|
293
|
+
"""An input the merge cannot read (exit 2)."""
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _read_detected(path: str) -> list[Any]:
|
|
297
|
+
"""The JSON array skf-detect-docs.py wrote to `path`."""
|
|
298
|
+
try:
|
|
299
|
+
data = json.loads(Path(path).read_bytes().decode("utf-8-sig"))
|
|
300
|
+
except OSError as exc:
|
|
301
|
+
raise InputError(f"cannot read --detected-file {path}: {exc.strerror or exc}") from exc
|
|
302
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
303
|
+
raise InputError(f"--detected-file {path} is not JSON: {exc}") from exc
|
|
304
|
+
if not isinstance(data, list):
|
|
305
|
+
raise InputError(f"--detected-file {path} must hold a JSON array (skf-detect-docs.py output)")
|
|
306
|
+
return data
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _brief_doc_urls(path: str) -> list[Any]:
|
|
310
|
+
"""The doc_urls of the skill brief at `path`, read by
|
|
311
|
+
skf-resolve-authoritative-files.py's parse_brief_yaml (beside this file)."""
|
|
312
|
+
resolver_path = Path(__file__).resolve().parent / "skf-resolve-authoritative-files.py"
|
|
313
|
+
spec = importlib.util.spec_from_file_location("skf_resolve_authoritative_files", resolver_path)
|
|
314
|
+
if spec is None or spec.loader is None:
|
|
315
|
+
raise InputError(f"cannot load {resolver_path.name} beside {Path(__file__).name}")
|
|
316
|
+
resolver = importlib.util.module_from_spec(spec)
|
|
317
|
+
try:
|
|
318
|
+
spec.loader.exec_module(resolver)
|
|
319
|
+
except (OSError, ImportError, SyntaxError) as exc:
|
|
320
|
+
raise InputError(f"cannot load {resolver_path.name} beside {Path(__file__).name}: {exc}") from exc
|
|
321
|
+
try:
|
|
322
|
+
brief = resolver.parse_brief_yaml(Path(path).read_bytes().decode("utf-8-sig"))
|
|
323
|
+
except OSError as exc:
|
|
324
|
+
raise InputError(f"cannot read --existing-brief {path}: {exc.strerror or exc}") from exc
|
|
325
|
+
except (UnicodeDecodeError, resolver.yaml.YAMLError) as exc:
|
|
326
|
+
raise InputError(f"--existing-brief {path} is not YAML: {exc}") from exc
|
|
327
|
+
if not isinstance(brief, dict):
|
|
328
|
+
raise InputError(f"--existing-brief {path} holds no brief mapping")
|
|
329
|
+
doc_urls = brief.get("doc_urls") or []
|
|
330
|
+
if not isinstance(doc_urls, list):
|
|
331
|
+
raise InputError(f"the doc_urls of --existing-brief {path} must be a list")
|
|
332
|
+
return doc_urls
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _stdin_payload() -> tuple[str, list[Any], list[Any]]:
|
|
336
|
+
"""(scope_type, existing, detected) of the stdin form."""
|
|
237
337
|
raw = sys.stdin.read()
|
|
238
338
|
if not raw or not raw.strip():
|
|
239
|
-
|
|
240
|
-
return 2
|
|
339
|
+
raise InputError("empty stdin (expected JSON object)")
|
|
241
340
|
try:
|
|
242
341
|
payload = json.loads(raw)
|
|
243
342
|
except json.JSONDecodeError as exc:
|
|
244
|
-
|
|
245
|
-
return 2
|
|
343
|
+
raise InputError(f"invalid JSON on stdin: {exc}") from exc
|
|
246
344
|
if not isinstance(payload, dict):
|
|
247
|
-
|
|
248
|
-
return 2
|
|
249
|
-
|
|
250
|
-
scope_type = payload.get("scope_type") or ""
|
|
345
|
+
raise InputError("input must be a JSON object")
|
|
251
346
|
existing = payload.get("existing") or []
|
|
252
347
|
detected = payload.get("detected") or []
|
|
253
348
|
if not isinstance(existing, list) or not isinstance(detected, list):
|
|
254
|
-
|
|
349
|
+
raise InputError("'existing' and 'detected' must be arrays")
|
|
350
|
+
return payload.get("scope_type") or "", existing, detected
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def main(argv: list[str] | None = None) -> int:
|
|
354
|
+
_force_utf8(sys.stdin, sys.stdout)
|
|
355
|
+
parser = argparse.ArgumentParser(
|
|
356
|
+
prog="skf-merge-doc-urls",
|
|
357
|
+
description="Merge a brief's doc_urls with the docs skf-detect-docs.py found. With no flag, read the "
|
|
358
|
+
"{scope_type, existing, detected} JSON object on stdin.",
|
|
359
|
+
)
|
|
360
|
+
parser.add_argument("--detected-file", metavar="PATH",
|
|
361
|
+
help="skf-detect-docs.py's JSON array, mapped to {url, label, source}")
|
|
362
|
+
parser.add_argument("--existing-brief", metavar="PATH",
|
|
363
|
+
help="the skill-brief.yaml whose doc_urls are the existing entries")
|
|
364
|
+
parser.add_argument("--scope-type", metavar="TYPE", help="the brief's scope.type")
|
|
365
|
+
args = parser.parse_args(argv)
|
|
366
|
+
unknown: list[dict[str, Any]] = []
|
|
367
|
+
try:
|
|
368
|
+
if args.detected_file is None and args.existing_brief is None and args.scope_type is None:
|
|
369
|
+
scope_type, existing, detected = _stdin_payload()
|
|
370
|
+
else:
|
|
371
|
+
scope_type = args.scope_type or ""
|
|
372
|
+
existing = _brief_doc_urls(args.existing_brief) if args.existing_brief is not None else []
|
|
373
|
+
detected = []
|
|
374
|
+
if args.detected_file is not None:
|
|
375
|
+
detected, unknown = map_detected(_read_detected(args.detected_file))
|
|
376
|
+
except InputError as exc:
|
|
377
|
+
sys.stderr.write(f"skf-merge-doc-urls: {exc}\n")
|
|
255
378
|
return 2
|
|
256
379
|
|
|
257
380
|
doc_urls, suppressed = merge_doc_urls(scope_type, existing, detected)
|
|
258
|
-
json.dump({"doc_urls": doc_urls, "suppressed": suppressed}, sys.stdout, ensure_ascii=False)
|
|
381
|
+
json.dump({"doc_urls": doc_urls, "suppressed": unknown + suppressed}, sys.stdout, ensure_ascii=False)
|
|
259
382
|
sys.stdout.write("\n")
|
|
260
383
|
return 0
|
|
261
384
|
|
|
@@ -49,6 +49,7 @@ SIDECAR_MISSING.
|
|
|
49
49
|
|
|
50
50
|
from __future__ import annotations
|
|
51
51
|
|
|
52
|
+
import datetime
|
|
52
53
|
import json
|
|
53
54
|
import re
|
|
54
55
|
import stat
|
|
@@ -82,6 +83,19 @@ PLACEHOLDER_RE = re.compile(r"\{[^{}]*\}")
|
|
|
82
83
|
FOLDER_KEYS = ("output_folder", "skills_output_folder", "forge_data_folder")
|
|
83
84
|
|
|
84
85
|
|
|
86
|
+
def _json_default(value):
|
|
87
|
+
"""Print a date or time as ISO 8601.
|
|
88
|
+
|
|
89
|
+
YAML reads an unquoted timestamp as a date or datetime object. SKF 1.x
|
|
90
|
+
wrote ``tier_detected_at`` and ``ccc_index.last_indexed`` in
|
|
91
|
+
forge-tier.yaml unquoted, so a sidecar from before 2.0.0 holds them; any
|
|
92
|
+
other value json cannot print is still an error.
|
|
93
|
+
"""
|
|
94
|
+
if isinstance(value, (datetime.date, datetime.time)):
|
|
95
|
+
return value.isoformat()
|
|
96
|
+
raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable")
|
|
97
|
+
|
|
98
|
+
|
|
85
99
|
def load_yaml_file(path):
|
|
86
100
|
"""Load a YAML mapping, returning (data, None) or (None, error_string).
|
|
87
101
|
|
|
@@ -304,5 +318,5 @@ if __name__ == "__main__":
|
|
|
304
318
|
cfg_path = sys.argv[idx + 1]
|
|
305
319
|
|
|
306
320
|
result = run_preflight(proj_root, cfg_path, allow_missing_sidecar="--allow-missing-sidecar" in sys.argv)
|
|
307
|
-
print(json.dumps(result, indent=2))
|
|
321
|
+
print(json.dumps(result, indent=2, default=_json_default))
|
|
308
322
|
sys.exit(1 if result["status"] == "hard-halt" else 0)
|
|
@@ -78,6 +78,7 @@ Exit codes:
|
|
|
78
78
|
from __future__ import annotations
|
|
79
79
|
|
|
80
80
|
import argparse
|
|
81
|
+
import importlib.util
|
|
81
82
|
import json
|
|
82
83
|
import re
|
|
83
84
|
import sys
|
|
@@ -256,14 +257,37 @@ _DEFERRED = "deferred-headless"
|
|
|
256
257
|
_LEGACY_HEADLESS_REASON = "headless: no user to prompt"
|
|
257
258
|
|
|
258
259
|
|
|
260
|
+
_RESOLVER = None
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _resolver():
|
|
264
|
+
"""skf-resolve-authoritative-files.py from this folder, loaded once: its
|
|
265
|
+
parse_brief_yaml reads a brief's YAML, an SKF 1.x brief's trailing `---`
|
|
266
|
+
included. Raises ValueError when it cannot be loaded."""
|
|
267
|
+
global _RESOLVER
|
|
268
|
+
if _RESOLVER is None:
|
|
269
|
+
path = Path(__file__).resolve().parent / "skf-resolve-authoritative-files.py"
|
|
270
|
+
spec = importlib.util.spec_from_file_location("skf_resolve_authoritative_files", path)
|
|
271
|
+
if spec is None or spec.loader is None:
|
|
272
|
+
raise ValueError(f"cannot load {path.name} beside {Path(__file__).name}")
|
|
273
|
+
module = importlib.util.module_from_spec(spec)
|
|
274
|
+
try:
|
|
275
|
+
spec.loader.exec_module(module)
|
|
276
|
+
except (OSError, ImportError, SyntaxError) as exc:
|
|
277
|
+
raise ValueError(f"cannot load {path.name} beside {Path(__file__).name}: {exc}") from exc
|
|
278
|
+
_RESOLVER = module
|
|
279
|
+
return _RESOLVER
|
|
280
|
+
|
|
281
|
+
|
|
259
282
|
def load_brief(brief_path: Path) -> dict:
|
|
260
283
|
"""Parse a skill-brief.yaml file. Raises ValueError on load failure."""
|
|
261
284
|
try:
|
|
262
285
|
text = brief_path.read_text(encoding="utf-8")
|
|
263
286
|
except OSError as exc:
|
|
264
287
|
raise ValueError(f"cannot read brief at {brief_path}: {exc}") from exc
|
|
288
|
+
parse_brief_yaml = _resolver().parse_brief_yaml
|
|
265
289
|
try:
|
|
266
|
-
brief =
|
|
290
|
+
brief = parse_brief_yaml(text)
|
|
267
291
|
except yaml.YAMLError as exc:
|
|
268
292
|
raise ValueError(f"brief at {brief_path} is not valid YAML: {exc}") from exc
|
|
269
293
|
if not isinstance(brief, dict):
|
|
@@ -109,8 +109,17 @@ import yaml
|
|
|
109
109
|
|
|
110
110
|
# skf-classify-changed-files.py loads this file for load_brief,
|
|
111
111
|
# extract_scope, glob_match, scope_match, normalize_rel_path and
|
|
112
|
-
# EXCLUDED_DIR_NAMES,
|
|
113
|
-
#
|
|
112
|
+
# EXCLUDED_DIR_NAMES (its load_scope, the brief-scope test that
|
|
113
|
+
# skf-new-file-diff.py, skf-detect-scripts-assets.py --brief and
|
|
114
|
+
# skf-compare-file-hashes.py --brief load from it, reads the brief through
|
|
115
|
+
# them), skf-detect-registry.py for glob_match, load_brief
|
|
116
|
+
# and extract_scope, skf-extract-public-api.py for glob_match and
|
|
117
|
+
# parse_brief_yaml, and every other script that reads a skill brief
|
|
118
|
+
# (skf-validate-brief-schema.py, skf-provenance-gap-dispatch.py,
|
|
119
|
+
# skf-derive-assembly-shape.py, skf-write-skill-brief.py and skf-test-skill's
|
|
120
|
+
# load-coverage-inputs.py) for parse_brief_yaml, load-coverage-inputs.py
|
|
121
|
+
# also for _amendment_action: keep those names and what they take and
|
|
122
|
+
# return.
|
|
114
123
|
|
|
115
124
|
|
|
116
125
|
# --------------------------------------------------------------------------
|
|
@@ -270,13 +279,30 @@ def scope_match(
|
|
|
270
279
|
# --------------------------------------------------------------------------
|
|
271
280
|
|
|
272
281
|
|
|
282
|
+
def parse_brief_yaml(text: str):
|
|
283
|
+
"""The one document a skill brief's YAML text holds, or None when it holds none.
|
|
284
|
+
|
|
285
|
+
SKF 1.x closed every brief with a trailing `---`, which YAML reads as a
|
|
286
|
+
second, empty document, so `yaml.safe_load` refuses such a brief. A
|
|
287
|
+
document that holds no value (empty, or an explicit null such as `~`) is
|
|
288
|
+
skipped wherever it stands; more than one document that holds a value is
|
|
289
|
+
refused with yaml.YAMLError, as is text that is not YAML.
|
|
290
|
+
"""
|
|
291
|
+
documents = [doc for doc in yaml.safe_load_all(text) if doc is not None]
|
|
292
|
+
if len(documents) > 1:
|
|
293
|
+
raise yaml.YAMLError(
|
|
294
|
+
f"a skill brief is a single document; this text holds {len(documents)}"
|
|
295
|
+
)
|
|
296
|
+
return documents[0] if documents else None
|
|
297
|
+
|
|
298
|
+
|
|
273
299
|
def load_brief(brief_path: Path) -> dict:
|
|
274
300
|
try:
|
|
275
301
|
text = brief_path.read_text(encoding="utf-8")
|
|
276
302
|
except OSError as exc:
|
|
277
303
|
raise ValueError(f"cannot read brief at {brief_path}: {exc}") from exc
|
|
278
304
|
try:
|
|
279
|
-
brief =
|
|
305
|
+
brief = parse_brief_yaml(text)
|
|
280
306
|
except yaml.YAMLError as exc:
|
|
281
307
|
raise ValueError(
|
|
282
308
|
f"brief at {brief_path} is not valid YAML: {exc}"
|
|
@@ -63,6 +63,20 @@ Matching:
|
|
|
63
63
|
listed in ambiguous_names[] with the files on each side, for a reviewer
|
|
64
64
|
to judge.
|
|
65
65
|
|
|
66
|
+
Public surface:
|
|
67
|
+
A current entry may carry `public`, true or false: the mark
|
|
68
|
+
skf-extraction-snapshot.py build --scope-type public-api gives each
|
|
69
|
+
export by the public-surface rule a public-api skill's provenance map is
|
|
70
|
+
kept to. After the pairing above, a current entry marked false that
|
|
71
|
+
pairs with none (no baseline entry of its name and file, and no move)
|
|
72
|
+
goes to not_public[], not added[]: create-skill and update-skill never
|
|
73
|
+
document an export off the surface, so it is no drift. One whose name is
|
|
74
|
+
in ambiguous_names[] stays in added[], where a reviewer pairs it. A
|
|
75
|
+
matched or moved entry is compared whatever its mark, so a baseline
|
|
76
|
+
entry off the surface is never reported removed. An entry with no mark
|
|
77
|
+
(any other scope type, a family with no barrel, an older snapshot) is
|
|
78
|
+
added as before. not_public[] never counts toward the exit code.
|
|
79
|
+
|
|
66
80
|
File scope (--files FILE):
|
|
67
81
|
For a caller that re-extracted only the files that changed (update-skill's
|
|
68
82
|
Category B), --files limits the diff to the exports of the files FILE
|
|
@@ -116,12 +130,27 @@ Canonicalization (applied symmetrically to BOTH sides before matching):
|
|
|
116
130
|
` = default`), so a runner's export compares with the map's entry.
|
|
117
131
|
|
|
118
132
|
Change detection:
|
|
119
|
-
The diffed fields are type
|
|
120
|
-
|
|
121
|
-
|
|
133
|
+
The diffed fields are type (compared through export-type, below),
|
|
134
|
+
signature, params, return_type and line. A field is only compared when
|
|
135
|
+
present (non-null) on BOTH sides: a field absent on one side means
|
|
136
|
+
"insufficient data", never an asserted change.
|
|
122
137
|
This avoids false positives when the two inventories carry different
|
|
123
138
|
metadata. The line of a moved export is not compared: a move changes it.
|
|
124
139
|
|
|
140
|
+
export-type: no record is rewritten, but when a matched export's two
|
|
141
|
+
types differ, a semantic kind a by-eye read records is the base kind
|
|
142
|
+
the recipe runner records for the same declaration when the base-kind
|
|
143
|
+
side's canonical signature shows that form: `async_function` is
|
|
144
|
+
`function` for an `async def`, `async function`, `async fn` or
|
|
145
|
+
`async unsafe fn`, or a `const`, `let` or `var` bound to an `async`
|
|
146
|
+
arrow or function (after `export`, `default`, `declare` or `pub`);
|
|
147
|
+
`decorator` is `function` for a Python `def` or `async def`; `enum` is
|
|
148
|
+
`class` for a Python class with a stdlib `Enum`, `IntEnum`, `StrEnum`,
|
|
149
|
+
`ReprEnum`, `Flag` or `IntFlag` base. Either side may hold the by-eye
|
|
150
|
+
kind; with no signature on the base-kind side the types differ. Each
|
|
151
|
+
export whose types it takes as one counts once as `export-type` in
|
|
152
|
+
applied_transforms.
|
|
153
|
+
|
|
125
154
|
A signature has two parts, the parameters and the return type. A side
|
|
126
155
|
carries the parameters as params or in its signature, and the return
|
|
127
156
|
type as return_type or in its signature. When both sides carry a part
|
|
@@ -145,7 +174,7 @@ Output:
|
|
|
145
174
|
JSON object:
|
|
146
175
|
summary: { added, removed, changed, moved, unchanged,
|
|
147
176
|
label_changes, ambiguous_names,
|
|
148
|
-
signature_unverified }
|
|
177
|
+
signature_unverified, not_public }
|
|
149
178
|
added: list of entries present in current but not baseline
|
|
150
179
|
removed: list of entries present in baseline but not current
|
|
151
180
|
changed: list of { name, field, baseline_value, current_value,
|
|
@@ -168,17 +197,26 @@ Output:
|
|
|
168
197
|
params and return_type each side carries.
|
|
169
198
|
Informational: never a change, never in the exit
|
|
170
199
|
code
|
|
200
|
+
not_public: list of current entries marked `public: false`
|
|
201
|
+
that pair with no baseline entry and whose name is
|
|
202
|
+
not ambiguous (see Public surface): exports off a
|
|
203
|
+
public-api skill's public surface, left out of
|
|
204
|
+
added[]. Not drift, never in the exit code
|
|
171
205
|
unchanged_count: number of entries that matched with no field change
|
|
172
206
|
(a label change or an unverified signature alone
|
|
173
207
|
leaves an entry unchanged)
|
|
174
208
|
applied_transforms: list of { transform, count }: which canonicalization
|
|
175
|
-
transforms
|
|
176
|
-
|
|
209
|
+
transforms, and the export-type comparison,
|
|
210
|
+
actually fired and how many values each touched
|
|
211
|
+
(export-type: how many exports; empty when none
|
|
212
|
+
fired). Surfaced by audit
|
|
177
213
|
step 6's Provenance section so a reviewer can tell
|
|
178
214
|
which differences the diff collapsed.
|
|
179
215
|
|
|
180
|
-
added[] and
|
|
181
|
-
params, return_type, file, line, confidence and
|
|
216
|
+
added[], removed[] and not_public[] hold normalized records: name, type,
|
|
217
|
+
signature, params, return_type, file, line, confidence and
|
|
218
|
+
extraction_method, and `public` when the entry carries a true or false
|
|
219
|
+
mark (see Public surface).
|
|
182
220
|
|
|
183
221
|
With --group-by source_library, each side is split by its entries'
|
|
184
222
|
source_library (a stack's libraries; an entry without one falls in a
|
|
@@ -208,8 +246,9 @@ Exit codes:
|
|
|
208
246
|
inventory whose entries all lack the field while the other's have it,
|
|
209
247
|
an output that cannot be written, or a usage error ({status: "error",
|
|
210
248
|
error} on stdout, or argparse's usage on stderr)
|
|
211
|
-
Label differences (label_changes)
|
|
212
|
-
(signature_unverified)
|
|
249
|
+
Label differences (label_changes), unverified signatures
|
|
250
|
+
(signature_unverified) and exports off the public surface (not_public) do
|
|
251
|
+
not count toward the exit code.
|
|
213
252
|
"""
|
|
214
253
|
|
|
215
254
|
from __future__ import annotations
|
|
@@ -275,6 +314,37 @@ _STDLIB_PREFIX_RE = re.compile(
|
|
|
275
314
|
r"(?<![\w.])(?:typing|dataclasses|collections\.abc|collections|enum)\."
|
|
276
315
|
)
|
|
277
316
|
|
|
317
|
+
# export-type: a semantic kind a by-eye read records, against the base kind
|
|
318
|
+
# the recipe runner records for the same declaration, names one kind when
|
|
319
|
+
# the base-kind side's canonical signature shows that form. Any other pair
|
|
320
|
+
# of types, or a base-kind side with no signature, is compared as written.
|
|
321
|
+
# An async form: `async def`, `async function`, `async fn` or `async unsafe
|
|
322
|
+
# fn`, or a `const` / `let` / `var` bound to an `async` arrow or function
|
|
323
|
+
# (`export const add = async (x) => x;`), after any `export`, `default`,
|
|
324
|
+
# `declare` or `pub` modifier.
|
|
325
|
+
_ASYNC_FORM_RE = re.compile(
|
|
326
|
+
r"^\s*(?:(?:export|default|declare|pub(?:\s*\([^()]*\))?)\s+)*"
|
|
327
|
+
r"(?:async\s+(?:unsafe\s+)?(?:def|function|fn)\b"
|
|
328
|
+
r"|(?:const|let|var)\s+[\w$]+\s*(?::(?:[^=]|=>)*)?=\s*async\b)"
|
|
329
|
+
)
|
|
330
|
+
_DEF_FORM_RE = re.compile(r"^\s*(?:async\s+)?def\s")
|
|
331
|
+
_CLASS_BASES_RE = re.compile(r"^\s*class\s+\w+\s*\(([^()]*)\)")
|
|
332
|
+
_ENUM_BASES = frozenset({"Enum", "IntEnum", "StrEnum", "ReprEnum", "Flag", "IntFlag"})
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _enum_class_form(signature: str) -> bool:
|
|
336
|
+
"""True when a Python class declaration lists a stdlib enum base."""
|
|
337
|
+
m = _CLASS_BASES_RE.match(signature)
|
|
338
|
+
return m is not None and any(base.strip() in _ENUM_BASES for base in m.group(1).split(","))
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
# (by-eye kind, base kind) -> the test of the base-kind side's signature.
|
|
342
|
+
EXPORT_TYPE_FORMS = {
|
|
343
|
+
("async_function", "function"): lambda signature: _ASYNC_FORM_RE.match(signature) is not None,
|
|
344
|
+
("decorator", "function"): lambda signature: _DEF_FORM_RE.match(signature) is not None,
|
|
345
|
+
("enum", "class"): _enum_class_form,
|
|
346
|
+
}
|
|
347
|
+
|
|
278
348
|
|
|
279
349
|
# --------------------------------------------------------------------------
|
|
280
350
|
# Loading
|
|
@@ -561,6 +631,10 @@ def _normalize_entries(
|
|
|
561
631
|
"confidence": entry.get("confidence"),
|
|
562
632
|
"extraction_method": entry.get("extraction_method"),
|
|
563
633
|
}
|
|
634
|
+
# The snapshot's public-surface mark, kept only when it is one, so an
|
|
635
|
+
# unmarked inventory's records are as before.
|
|
636
|
+
if isinstance(entry.get("public"), bool):
|
|
637
|
+
record["public"] = entry["public"]
|
|
564
638
|
key = (name, _file_key(record["file"]))
|
|
565
639
|
kept = result.get(key)
|
|
566
640
|
if kept is not None and not (
|
|
@@ -616,6 +690,21 @@ def _label_change(base_rec: dict, curr_rec: dict) -> dict | None:
|
|
|
616
690
|
return None
|
|
617
691
|
|
|
618
692
|
|
|
693
|
+
def _export_type_collapses(base_rec: dict, curr_rec: dict) -> bool:
|
|
694
|
+
"""True when two matched records' types name one kind (EXPORT_TYPE_FORMS):
|
|
695
|
+
one side's by-eye kind against the other's base kind, whose canonical
|
|
696
|
+
signature shows that form. Either side may hold the by-eye kind."""
|
|
697
|
+
for by_eye, base in ((base_rec, curr_rec), (curr_rec, base_rec)):
|
|
698
|
+
kinds = (by_eye.get("type"), base.get("type"))
|
|
699
|
+
signature = base.get("signature")
|
|
700
|
+
if not all(isinstance(kind, str) for kind in kinds) or not isinstance(signature, str):
|
|
701
|
+
continue
|
|
702
|
+
form = EXPORT_TYPE_FORMS.get(kinds)
|
|
703
|
+
if form is not None and form(signature):
|
|
704
|
+
return True
|
|
705
|
+
return False
|
|
706
|
+
|
|
707
|
+
|
|
619
708
|
def _signature_fields(rec: dict) -> list[str]:
|
|
620
709
|
return [field for field in SIGNATURE_FIELDS if rec.get(field) is not None]
|
|
621
710
|
|
|
@@ -641,7 +730,8 @@ def _unverified_signature(base_rec: dict, curr_rec: dict) -> dict | None:
|
|
|
641
730
|
# --------------------------------------------------------------------------
|
|
642
731
|
|
|
643
732
|
|
|
644
|
-
LISTS = ["added", "removed", "changed", "moved", "ambiguous_names", "label_changes", "signature_unverified"
|
|
733
|
+
LISTS = ["added", "removed", "changed", "moved", "ambiguous_names", "label_changes", "signature_unverified",
|
|
734
|
+
"not_public"]
|
|
645
735
|
|
|
646
736
|
|
|
647
737
|
def _key_order(key: tuple) -> tuple:
|
|
@@ -656,14 +746,18 @@ def _diff_records(
|
|
|
656
746
|
baseline: dict[tuple, dict],
|
|
657
747
|
current: dict[tuple, dict],
|
|
658
748
|
unchanged_names: collections.Counter | None = None,
|
|
749
|
+
transform_counts: collections.Counter | None = None,
|
|
659
750
|
) -> dict:
|
|
660
751
|
"""Diff two (name, file)-keyed record sets: the LISTS, plus the counts
|
|
661
752
|
unchanged_count and changed_exports (exports with a changed field).
|
|
662
753
|
|
|
663
754
|
unchanged_names counts the names of exports left out of both sets as
|
|
664
755
|
unchanged (--files): each counts on both sides when a move needs a name
|
|
665
|
-
unique on both sides.
|
|
756
|
+
unique on both sides. transform_counts gains one export-type for each
|
|
757
|
+
matched export whose types name one kind (_export_type_collapses).
|
|
666
758
|
"""
|
|
759
|
+
if transform_counts is None:
|
|
760
|
+
transform_counts = collections.Counter()
|
|
667
761
|
removed_keys = sorted((k for k in baseline if k not in current), key=_key_order)
|
|
668
762
|
added_keys = sorted((k for k in current if k not in baseline), key=_key_order)
|
|
669
763
|
|
|
@@ -697,8 +791,20 @@ def _diff_records(
|
|
|
697
791
|
|
|
698
792
|
# Emit normalized entries (uniform shape) so added[] and removed[] render
|
|
699
793
|
# into the same report table even though they originate from the
|
|
700
|
-
# snapshot and provenance-map shapes.
|
|
701
|
-
|
|
794
|
+
# snapshot and provenance-map shapes. After the pairing, a leftover
|
|
795
|
+
# current entry marked off the public surface is no addition, unless its
|
|
796
|
+
# name is ambiguous: a reviewer may pair it as a move.
|
|
797
|
+
ambiguous = {item["name"] for item in ambiguous_names}
|
|
798
|
+
added: list[dict] = []
|
|
799
|
+
not_public: list[dict] = []
|
|
800
|
+
for key in added_keys:
|
|
801
|
+
if key in paired_curr:
|
|
802
|
+
continue
|
|
803
|
+
rec = current[key]
|
|
804
|
+
if rec.get("public") is False and rec["name"] not in ambiguous:
|
|
805
|
+
not_public.append(rec)
|
|
806
|
+
else:
|
|
807
|
+
added.append(rec)
|
|
702
808
|
removed = [baseline[k] for k in removed_keys if k not in paired_base]
|
|
703
809
|
|
|
704
810
|
changed: list[dict] = []
|
|
@@ -736,6 +842,11 @@ def _diff_records(
|
|
|
736
842
|
if base_val is None or curr_val is None:
|
|
737
843
|
continue
|
|
738
844
|
if base_val != curr_val:
|
|
845
|
+
# A by-eye kind and the runner's base kind of one declaration
|
|
846
|
+
# are one type: compared pairwise, never rewritten.
|
|
847
|
+
if field == "type" and _export_type_collapses(base_rec, curr_rec):
|
|
848
|
+
transform_counts["export-type"] += 1
|
|
849
|
+
continue
|
|
739
850
|
changed.append({
|
|
740
851
|
"name": curr_rec["name"],
|
|
741
852
|
"field": field,
|
|
@@ -771,6 +882,7 @@ def _diff_records(
|
|
|
771
882
|
"ambiguous_names": ambiguous_names,
|
|
772
883
|
"label_changes": label_changes,
|
|
773
884
|
"signature_unverified": signature_unverified,
|
|
885
|
+
"not_public": not_public,
|
|
774
886
|
"unchanged_count": unchanged_count,
|
|
775
887
|
"changed_exports": changed_exports,
|
|
776
888
|
}
|
|
@@ -786,6 +898,7 @@ def _summary(part: dict) -> dict:
|
|
|
786
898
|
"label_changes": len(part["label_changes"]),
|
|
787
899
|
"ambiguous_names": len(part["ambiguous_names"]),
|
|
788
900
|
"signature_unverified": len(part["signature_unverified"]),
|
|
901
|
+
"not_public": len(part["not_public"]),
|
|
789
902
|
}
|
|
790
903
|
|
|
791
904
|
|
|
@@ -813,10 +926,11 @@ def diff_inventories(
|
|
|
813
926
|
"""Compute the structural diff between two export inventories.
|
|
814
927
|
|
|
815
928
|
Returns a dict with keys: summary, added, removed, changed, moved,
|
|
816
|
-
ambiguous_names, label_changes, signature_unverified,
|
|
817
|
-
applied_transforms; with group_by, group_by and
|
|
818
|
-
file_scope (see the module docstring). files
|
|
819
|
-
exports of those files, and every other export
|
|
929
|
+
ambiguous_names, label_changes, signature_unverified, not_public,
|
|
930
|
+
unchanged_count, applied_transforms; with group_by, group_by and
|
|
931
|
+
groups; and with files, file_scope (see the module docstring). files
|
|
932
|
+
limits the diff to the exports of those files, and every other export
|
|
933
|
+
is taken as unchanged.
|
|
820
934
|
|
|
821
935
|
Raises ValueError when group_by is set and one side has entries but
|
|
822
936
|
none with a group_by value while the other side has some; the message
|
|
@@ -869,6 +983,7 @@ def diff_inventories(
|
|
|
869
983
|
_normalize_entries(base_part, reexport_map, transform_counts),
|
|
870
984
|
_normalize_entries(curr_part, reexport_map, transform_counts),
|
|
871
985
|
unchanged_names,
|
|
986
|
+
transform_counts,
|
|
872
987
|
)
|
|
873
988
|
if group_by:
|
|
874
989
|
for name in LISTS:
|