bmad-module-skill-forge 3.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +3 -3
- package/docs/_data/pinned.yaml +19 -13
- package/docs/architecture.md +2 -2
- package/docs/concepts.md +4 -4
- package/docs/examples.md +33 -26
- package/docs/getting-started.md +3 -3
- package/docs/how-it-works.md +2 -2
- package/docs/index.md +1 -1
- package/docs/skill-model.md +41 -39
- package/docs/troubleshooting.md +8 -0
- package/docs/verifying-a-skill.md +46 -21
- package/docs/why-skf.md +1 -1
- package/docs/workflows.md +5 -5
- package/package.json +1 -1
- package/src/knowledge/qmd-registry.md +1 -1
- package/src/knowledge/version-paths.md +1 -1
- package/src/shared/health-check.md +2 -50
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +12 -4
- package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +212 -59
- package/src/shared/scripts/skf-build-change-manifest.py +1437 -178
- package/src/shared/scripts/skf-classify-changed-files.py +78 -0
- package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
- package/src/shared/scripts/skf-count-tokens.py +54 -10
- package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
- package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
- package/src/shared/scripts/skf-detect-workspaces.py +67 -0
- package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
- package/src/shared/scripts/skf-extract-public-api.py +1673 -189
- package/src/shared/scripts/skf-extraction-inventory.py +584 -46
- package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
- package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
- package/src/shared/scripts/skf-github-probe.py +47 -8
- package/src/shared/scripts/skf-hash-content.py +14 -8
- package/src/shared/scripts/skf-load-provenance.py +3 -1
- package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
- package/src/shared/scripts/skf-parse-gaps.py +135 -26
- package/src/shared/scripts/skf-preflight.py +15 -1
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
- package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
- package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
- package/src/shared/scripts/skf-resolve-customization.py +197 -0
- package/src/shared/scripts/skf-shard-body.py +239 -41
- package/src/shared/scripts/skf-structural-diff.py +133 -18
- package/src/shared/scripts/skf-update-active-symlink.py +115 -62
- package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
- package/src/shared/scripts/skf-validate-output.py +12 -3
- package/src/shared/scripts/skf-verify-provenance-completeness.py +1439 -79
- package/src/shared/scripts/skf-write-skill-brief.py +41 -8
- package/src/shared/tool-requirements.yaml +8 -2
- package/src/skf-analyze-source/SKILL.md +2 -2
- package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
- package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
- package/src/skf-audit-skill/SKILL.md +2 -2
- package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
- package/src/skf-audit-skill/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +2 -1
- package/src/skf-audit-skill/references/re-index.md +14 -9
- package/src/skf-audit-skill/references/report.md +2 -2
- package/src/skf-audit-skill/references/semantic-diff.md +15 -15
- package/src/skf-audit-skill/references/structural-diff.md +10 -5
- package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
- package/src/skf-brief-skill/SKILL.md +3 -3
- package/src/skf-brief-skill/assets/scope-templates.md +1 -1
- package/src/skf-brief-skill/assets/skill-brief-schema.md +4 -11
- package/src/skf-brief-skill/references/analyze-target.md +6 -6
- package/src/skf-brief-skill/references/confirm-brief.md +1 -1
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
- package/src/skf-brief-skill/references/version-resolution.md +5 -9
- package/src/skf-campaign/SKILL.md +2 -2
- package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
- package/src/skf-create-skill/SKILL.md +2 -2
- package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
- package/src/skf-create-skill/assets/skill-sections.md +7 -7
- package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
- package/src/skf-create-skill/references/compile.md +31 -60
- package/src/skf-create-skill/references/component-extraction.md +3 -3
- package/src/skf-create-skill/references/enrich.md +2 -2
- package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
- package/src/skf-create-skill/references/extract.md +23 -19
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
- package/src/skf-create-skill/references/extraction-patterns.md +10 -11
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +21 -5
- package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
- package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
- package/src/skf-create-stack-skill/SKILL.md +2 -2
- package/src/skf-create-stack-skill/references/validate.md +1 -1
- package/src/skf-drop-skill/SKILL.md +2 -2
- package/src/skf-drop-skill/references/execute.md +5 -1
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +2 -2
- package/src/skf-drop-skill/references/select.md +7 -3
- package/src/skf-export-skill/SKILL.md +2 -2
- package/src/skf-export-skill/assets/snippet-format.md +5 -5
- package/src/skf-export-skill/references/generate-snippet.md +22 -15
- package/src/skf-export-skill/references/load-skill.md +1 -1
- package/src/skf-export-skill/references/token-report.md +29 -9
- package/src/skf-forger/SKILL.md +2 -2
- package/src/skf-quick-skill/SKILL.md +2 -2
- package/src/skf-refine-architecture/SKILL.md +6 -8
- package/src/skf-rename-skill/SKILL.md +2 -2
- package/src/skf-setup/SKILL.md +2 -2
- package/src/skf-test-skill/SKILL.md +2 -2
- package/src/skf-test-skill/assets/output-section-formats.md +6 -4
- package/src/skf-test-skill/references/coherence-check.md +5 -4
- package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
- package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
- package/src/skf-test-skill/references/coverage-check.md +69 -111
- package/src/skf-test-skill/references/init.md +5 -6
- package/src/skf-test-skill/references/migration-section-rules.md +26 -14
- package/src/skf-test-skill/references/report.md +11 -9
- package/src/skf-test-skill/references/scoring-rules.md +8 -6
- package/src/skf-test-skill/references/source-access-protocol.md +15 -18
- package/src/skf-test-skill/scripts/build-result-context.py +114 -21
- package/src/skf-test-skill/scripts/gap-ledger.py +580 -80
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +646 -107
- package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
- package/src/skf-test-skill/scripts/reconcile-coverage.py +450 -23
- package/src/skf-test-skill/scripts/score-signatures.py +356 -61
- package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
- package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
- package/src/skf-test-skill/templates/test-report-template.md +1 -0
- package/src/skf-update-skill/SKILL.md +2 -2
- package/src/skf-update-skill/references/detect-changes.md +32 -13
- package/src/skf-update-skill/references/gap-driven.md +37 -22
- package/src/skf-update-skill/references/init.md +6 -2
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +17 -12
- package/src/skf-update-skill/references/re-extract.md +4 -2
- package/src/skf-update-skill/references/report.md +7 -4
- package/src/skf-update-skill/references/write.md +77 -36
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
- package/src/skf-verify-stack/SKILL.md +2 -2
- package/tools/cli/lib/tool-check.js +158 -2
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# /// script
|
|
2
|
+
# requires-python = ">=3.11"
|
|
3
|
+
# dependencies = []
|
|
4
|
+
# ///
|
|
5
|
+
"""SKF Resolve Customization: BMAD core's customization resolver, on any core from 6.4.0.
|
|
6
|
+
|
|
7
|
+
Every SKF workflow, and Ferris, reads its `_bmad/custom/` overrides through
|
|
8
|
+
BMAD core's customization resolver, `_bmad/scripts/resolve_customization.py`
|
|
9
|
+
under the project root, which the BMAD Method installer adds. SKF passes it
|
|
10
|
+
`--project-root`, a flag core takes only from 6.11.0: core 6.4.0 to 6.10.0
|
|
11
|
+
stops on it with argparse's exit 2, so SKF runs this helper in its place.
|
|
12
|
+
It takes core's long flags only (core's -s, -p and -k are refused):
|
|
13
|
+
|
|
14
|
+
uv run skf-resolve-customization.py --skill <skill-root> \\
|
|
15
|
+
--project-root <project-root> --key <key> [--key <key>]...
|
|
16
|
+
|
|
17
|
+
1. It runs core's resolver with those arguments, `--project-root` included,
|
|
18
|
+
under its own interpreter, from the project root. `uv run` gives this
|
|
19
|
+
file a Python that meets its requires-python, which core needs for
|
|
20
|
+
`tomllib`; core 6.10.0 and earlier declare none, so uv would not choose
|
|
21
|
+
one for them.
|
|
22
|
+
2. When core rejects `--project-root` (exit 2, nothing on stdout, and
|
|
23
|
+
`error: unrecognized arguments: ... --project-root` on stderr), it works
|
|
24
|
+
out the root a core before 6.11.0 picks without the flag: the nearest
|
|
25
|
+
folder, from the skill folder up and the skill folder itself first, that
|
|
26
|
+
holds `_bmad` or `.git` (a file or a folder), else the same search from
|
|
27
|
+
the project root. When that is the project root, it runs core again
|
|
28
|
+
without the flag, and that run's result is the helper's. Otherwise core
|
|
29
|
+
would read another folder's overrides, so it refuses.
|
|
30
|
+
|
|
31
|
+
Output:
|
|
32
|
+
stdout on exit 0, core's stdout byte for byte (the JSON core prints);
|
|
33
|
+
on any other exit, nothing.
|
|
34
|
+
stderr the stderr of the core run whose result is the helper's: on a
|
|
35
|
+
rerun without the flag, the rerun's only (the rejected call's is
|
|
36
|
+
dropped). A failure of the helper's own prints one line first: the
|
|
37
|
+
reason a workflow's warning quotes.
|
|
38
|
+
|
|
39
|
+
Exit codes:
|
|
40
|
+
0 core resolved the customization
|
|
41
|
+
1 core's resolver is missing (first stderr line: `not found`), could
|
|
42
|
+
not start, did not finish or was stopped by a signal, or core exited 1
|
|
43
|
+
(a customize.toml or an override it cannot read)
|
|
44
|
+
2 a core before 6.11.0 would read the overrides under another folder
|
|
45
|
+
(first stderr line: `BMAD core before 6.11.0: ...`), a usage error of
|
|
46
|
+
the helper's, or core's own exit 2 (a core from 6.11.0), passed through
|
|
47
|
+
3 core found no Python 3.11 or later (`tomllib`)
|
|
48
|
+
Any other exit code is core's own, passed through.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
from __future__ import annotations
|
|
52
|
+
|
|
53
|
+
import argparse
|
|
54
|
+
import os
|
|
55
|
+
import re
|
|
56
|
+
import subprocess
|
|
57
|
+
import sys
|
|
58
|
+
from pathlib import Path
|
|
59
|
+
|
|
60
|
+
CORE_RESOLVER = ("_bmad", "scripts", "resolve_customization.py")
|
|
61
|
+
FIRST_CORE_WITH_FLAG = "6.11.0"
|
|
62
|
+
CORE_TIMEOUT_SEC = 120
|
|
63
|
+
PROJECT_ROOT_FLAG = "--project-root"
|
|
64
|
+
MISSING = "not found"
|
|
65
|
+
ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
|
|
66
|
+
# argparse's line for a flag it does not know, wherever the usage line wraps.
|
|
67
|
+
REJECTED_FLAG_RE = re.compile(r"error: unrecognized arguments: (?:.*\s)?--project-root(?:[=\s]|$)", re.M)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
71
|
+
parser = argparse.ArgumentParser(
|
|
72
|
+
prog="skf-resolve-customization",
|
|
73
|
+
description=(
|
|
74
|
+
"Run BMAD core's customization resolver with --project-root. On a core before "
|
|
75
|
+
f"{FIRST_CORE_WITH_FLAG}, which rejects that flag, run it again without the flag when "
|
|
76
|
+
"core would read this project's overrides anyway, and refuse otherwise."
|
|
77
|
+
),
|
|
78
|
+
)
|
|
79
|
+
parser.add_argument("--skill", required=True, help="the skill folder that holds customize.toml")
|
|
80
|
+
parser.add_argument(
|
|
81
|
+
"--project-root",
|
|
82
|
+
required=True,
|
|
83
|
+
help="the project folder: core's resolver is under its _bmad/scripts/, the overrides under _bmad/custom/",
|
|
84
|
+
)
|
|
85
|
+
parser.add_argument(
|
|
86
|
+
"--key",
|
|
87
|
+
action="append",
|
|
88
|
+
default=[],
|
|
89
|
+
help="a dotted key to resolve, repeatable; with none, core prints the whole merge",
|
|
90
|
+
)
|
|
91
|
+
return parser
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _nearest_marked(start: Path) -> Path | None:
|
|
95
|
+
"""The nearest folder, `start` itself first, holding `_bmad` or `.git` (a file or a folder)."""
|
|
96
|
+
current = start.resolve()
|
|
97
|
+
while True:
|
|
98
|
+
if (current / "_bmad").exists() or (current / ".git").exists():
|
|
99
|
+
return current
|
|
100
|
+
if current.parent == current:
|
|
101
|
+
return None
|
|
102
|
+
current = current.parent
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def legacy_project_root(skill: Path, cwd: Path) -> Path | None:
|
|
106
|
+
"""The project root BMAD core 6.4.0 to 6.10.0 picks without --project-root, run from `cwd`."""
|
|
107
|
+
return _nearest_marked(skill) or _nearest_marked(cwd)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _same_folder(a: Path, b: Path) -> bool:
|
|
111
|
+
try:
|
|
112
|
+
return os.path.samefile(a, b)
|
|
113
|
+
except OSError:
|
|
114
|
+
return a.resolve() == b.resolve()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def rejects_project_root(returncode: int, stdout: bytes, stderr: bytes) -> bool:
|
|
118
|
+
"""True when core stopped on --project-root as a flag it does not know; never on exit 2 alone."""
|
|
119
|
+
if returncode != 2 or stdout:
|
|
120
|
+
return False
|
|
121
|
+
text = ANSI_RE.sub("", stderr.decode("utf-8", "replace"))
|
|
122
|
+
return REJECTED_FLAG_RE.search(text) is not None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _run_core(argv: list[str], cwd: Path) -> subprocess.CompletedProcess:
|
|
126
|
+
env = dict(os.environ)
|
|
127
|
+
# Core 6.11.0 and later import config_utils from their own folder, which a safe path drops.
|
|
128
|
+
env.pop("PYTHONSAFEPATH", None)
|
|
129
|
+
# Core before 6.8.0 writes its JSON in the locale's encoding.
|
|
130
|
+
env["PYTHONIOENCODING"] = "utf-8"
|
|
131
|
+
return subprocess.run(
|
|
132
|
+
argv, cwd=cwd, env=env, stdin=subprocess.DEVNULL, capture_output=True,
|
|
133
|
+
timeout=CORE_TIMEOUT_SEC, check=False,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _write(stream, data: bytes) -> None:
|
|
138
|
+
if not data:
|
|
139
|
+
return
|
|
140
|
+
buffer = getattr(stream, "buffer", None)
|
|
141
|
+
if buffer is None:
|
|
142
|
+
stream.write(data.decode("utf-8", "replace"))
|
|
143
|
+
stream.flush()
|
|
144
|
+
return
|
|
145
|
+
buffer.write(data)
|
|
146
|
+
buffer.flush()
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _say(*lines: str) -> None:
|
|
150
|
+
_write(sys.stderr, "".join(line + "\n" for line in lines).encode("utf-8"))
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _finish(proc: subprocess.CompletedProcess) -> int:
|
|
154
|
+
"""Core's result as the helper's: its stdout only on exit 0, its stderr always."""
|
|
155
|
+
if proc.returncode < 0: # stopped by a signal: core may have printed no reason
|
|
156
|
+
_say(f"BMAD core's customization resolver was stopped by signal {-proc.returncode}")
|
|
157
|
+
_write(sys.stderr, proc.stderr)
|
|
158
|
+
return 1
|
|
159
|
+
if proc.returncode == 0:
|
|
160
|
+
_write(sys.stdout, proc.stdout)
|
|
161
|
+
_write(sys.stderr, proc.stderr)
|
|
162
|
+
return proc.returncode
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def main(argv: list[str] | None = None) -> int:
|
|
166
|
+
args = _build_parser().parse_args(argv)
|
|
167
|
+
root = Path(os.path.abspath(args.project_root))
|
|
168
|
+
skill = Path(os.path.abspath(args.skill))
|
|
169
|
+
core = root.joinpath(*CORE_RESOLVER)
|
|
170
|
+
if not core.is_file():
|
|
171
|
+
_say(MISSING, f"BMAD core's customization resolver is not at {core}: the BMAD Method installer adds it.")
|
|
172
|
+
return 1
|
|
173
|
+
command = [sys.executable, str(core), "--skill", str(skill)]
|
|
174
|
+
keys = [word for key in args.key for word in ("--key", key)]
|
|
175
|
+
try:
|
|
176
|
+
first = _run_core([*command, PROJECT_ROOT_FLAG, str(root), *keys], root)
|
|
177
|
+
if not rejects_project_root(first.returncode, first.stdout, first.stderr):
|
|
178
|
+
return _finish(first)
|
|
179
|
+
legacy = legacy_project_root(skill, root)
|
|
180
|
+
if legacy is not None and _same_folder(legacy, root):
|
|
181
|
+
return _finish(_run_core([*command, *keys], root))
|
|
182
|
+
except subprocess.TimeoutExpired:
|
|
183
|
+
_say(f"BMAD core's customization resolver did not finish within {CORE_TIMEOUT_SEC} seconds")
|
|
184
|
+
return 1
|
|
185
|
+
except OSError as exc:
|
|
186
|
+
_say(f"BMAD core's customization resolver could not start: {exc}")
|
|
187
|
+
return 1
|
|
188
|
+
_say(
|
|
189
|
+
f"BMAD core before {FIRST_CORE_WITH_FLAG}: its customization resolver takes no {PROJECT_ROOT_FLAG} "
|
|
190
|
+
f"and would read the overrides under {legacy}, not under {root}; upgrade BMAD core to "
|
|
191
|
+
f"{FIRST_CORE_WITH_FLAG} or later"
|
|
192
|
+
)
|
|
193
|
+
return 2
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
if __name__ == "__main__":
|
|
197
|
+
sys.exit(main())
|
|
@@ -8,42 +8,70 @@
|
|
|
8
8
|
Owns the line counting, the section boundaries, the size sort, the file
|
|
9
9
|
writes and the blockquote replacements of `skf-create-skill`'s
|
|
10
10
|
`references/step-auto-shard.md` §0 (which runs it on every pass and keeps no
|
|
11
|
-
by-hand fallback) and `references/validate.md` §4
|
|
12
|
-
|
|
11
|
+
by-hand fallback) and `references/validate.md` §4, and of
|
|
12
|
+
`skf-update-skill`'s `references/write.md` §7 when skill-check reports
|
|
13
|
+
`body.max_lines` or `body.max_tokens`: counted by eye, an over-budget body
|
|
14
|
+
ships silently or the step halts for nothing.
|
|
13
15
|
|
|
14
16
|
What it does (single positional invocation):
|
|
15
17
|
|
|
16
18
|
1. Split frontmatter (leading `---` … `---`) from body. Frontmatter is
|
|
17
19
|
NEVER modified.
|
|
18
20
|
2. Count body lines between the frontmatter close and EOF, excluding
|
|
19
|
-
trailing blank lines → `body_lines_before
|
|
20
|
-
|
|
21
|
-
`
|
|
21
|
+
trailing blank lines → `body_lines_before`, and body words: the
|
|
22
|
+
parts left after splitting the body on JavaScript's `\\s` whitespace
|
|
23
|
+
(`/\\s+/`), empty parts dropped → `body_words_before`: the same split
|
|
24
|
+
skill-check's `body.max_tokens` estimate makes
|
|
25
|
+
(`text.split(/\\s+/).filter(Boolean)`), which update-skill's write.md
|
|
26
|
+
§7 holds under a budget below skill-check's limit.
|
|
27
|
+
3. If `body_lines_before` <= `--budget` (default 400) and, when
|
|
28
|
+
`--word-budget` is set (default 0, off), `body_words_before` <=
|
|
29
|
+
`--word-budget`: emit `action: "skip"`, write nothing.
|
|
22
30
|
4. Otherwise enumerate Tier-2 sections — level-2 headings whose text
|
|
23
31
|
starts with `Full` (`## Full API Reference`, `## Full Type
|
|
24
32
|
Definitions`, …). Each section spans its heading through the line
|
|
25
|
-
before the next level-1/level-2 heading (or EOF).
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
33
|
+
before the next level-1/level-2 heading (or EOF). With
|
|
34
|
+
`--skip-manual`, a section that intersects a [MANUAL] block is never
|
|
35
|
+
a candidate: a block is an open marker paired with the earliest
|
|
36
|
+
following close marker of the same name, read over the whole body
|
|
37
|
+
(so a block opened before a section and closed after it counts, and
|
|
38
|
+
a marker split across lines is found, as skf-hash-content.py reads
|
|
39
|
+
them), and a marker left unpaired protects its own lines. update-skill
|
|
40
|
+
inventories [MANUAL] blocks in SKILL.md only, so a moved block would
|
|
41
|
+
leave its next inventory.
|
|
42
|
+
5. Greedily extract one candidate at a time, re-counting after each,
|
|
43
|
+
stopping the moment the body fits: lines <= `--budget` and, when set,
|
|
44
|
+
words <= `--word-budget`. While the lines exceed the budget the next
|
|
45
|
+
candidate is the largest by lines; once they fit, the largest by
|
|
46
|
+
words (ties keep document order). If extracting every candidate
|
|
47
|
+
still cannot fit, extract them all and report `under_budget: false`
|
|
48
|
+
or `under_word_budget: false`. create-skill then trims Tier-1 by
|
|
49
|
+
editing judgment (that fallback stays in the prompt); update-skill
|
|
50
|
+
records a warning and trims nothing. With no candidate, nothing is
|
|
51
|
+
written.
|
|
32
52
|
6. Each extracted section is written to `references/{kebab(heading)}.md`
|
|
33
53
|
(heading preserved) via `skf-atomic-write.py`, and the section in
|
|
34
54
|
SKILL.md is replaced by a `> See [Title](references/…md)` blockquote.
|
|
35
|
-
|
|
55
|
+
A file is never overwritten: when `references/{kebab}.md` already
|
|
56
|
+
holds other content, or an earlier extracted section of this run took
|
|
57
|
+
the name, the next free `{kebab}-2.md`, `{kebab}-3.md`, … is used (as
|
|
58
|
+
skill-check's split-body names them), in document order, and the
|
|
59
|
+
blockquote points at it. The rewritten SKILL.md is itself written via
|
|
60
|
+
`skf-atomic-write.py`.
|
|
36
61
|
7. Verify every Tier-1 heading that was inline before extraction is still
|
|
37
62
|
inline (`tier1_preserved`, with `tier1_missing[]`), and every emitted
|
|
38
63
|
blockquote resolves to a written reference file (`xref_ok`).
|
|
39
64
|
|
|
40
|
-
Deterministic: same input +
|
|
65
|
+
Deterministic: same input + budgets + flags → same JSON and same on-disk
|
|
66
|
+
result.
|
|
41
67
|
|
|
42
68
|
Output JSON (stdout, or `-o`):
|
|
43
69
|
{
|
|
44
70
|
"action": "skip" | "shard",
|
|
45
71
|
"body_lines_before": <int>,
|
|
46
72
|
"body_lines_after": <int>,
|
|
73
|
+
"body_words_before": <int>,
|
|
74
|
+
"body_words_after": <int>,
|
|
47
75
|
"sections_extracted": [
|
|
48
76
|
{"heading": "Full API Reference",
|
|
49
77
|
"file": "references/full-api-reference.md",
|
|
@@ -53,8 +81,10 @@ Output JSON (stdout, or `-o`):
|
|
|
53
81
|
"tier1_missing": ["Overview", ...],
|
|
54
82
|
"xref_ok": <bool>,
|
|
55
83
|
"under_budget": <bool>,
|
|
84
|
+
"under_word_budget": <bool>, (true when --word-budget is 0)
|
|
56
85
|
"dry_run": <bool>,
|
|
57
|
-
"budget": <int
|
|
86
|
+
"budget": <int>,
|
|
87
|
+
"word_budget": <int> (0 when off)
|
|
58
88
|
}
|
|
59
89
|
|
|
60
90
|
Heading parsing reuses `skf-scan-skill-md-structure.py`'s `_HEADING_RE`
|
|
@@ -68,11 +98,14 @@ Exit codes:
|
|
|
68
98
|
gates on `tier1_preserved` / `xref_ok`.
|
|
69
99
|
1 — user error (file not found, unreadable, bad frontmatter).
|
|
70
100
|
2 — operation failure (a reference/SKILL.md write could not complete).
|
|
101
|
+
The stderr line names the file that failed and each reference file
|
|
102
|
+
already written (`already written: none` when there is none).
|
|
71
103
|
"""
|
|
72
104
|
|
|
73
105
|
from __future__ import annotations
|
|
74
106
|
|
|
75
107
|
import argparse
|
|
108
|
+
import bisect
|
|
76
109
|
import importlib.util
|
|
77
110
|
import json
|
|
78
111
|
import re
|
|
@@ -141,13 +174,16 @@ TIER1_HEADINGS: list[str] = [
|
|
|
141
174
|
class Section:
|
|
142
175
|
"""A Tier-2 (`## Full …`) section as a half-open line range [start, end)."""
|
|
143
176
|
|
|
144
|
-
__slots__ = ("start", "end", "heading", "kebab")
|
|
177
|
+
__slots__ = ("start", "end", "heading", "kebab", "name")
|
|
145
178
|
|
|
146
179
|
def __init__(self, start: int, end: int, heading: str) -> None:
|
|
147
180
|
self.start = start
|
|
148
181
|
self.end = end
|
|
149
182
|
self.heading = heading
|
|
150
183
|
self.kebab = kebab_case(heading)
|
|
184
|
+
# The reference file's stem: the kebab name, or a `-2`, `-3` …
|
|
185
|
+
# variant `assign_names` picks so no file is overwritten.
|
|
186
|
+
self.name = self.kebab
|
|
151
187
|
|
|
152
188
|
@property
|
|
153
189
|
def lines(self) -> int:
|
|
@@ -155,11 +191,11 @@ class Section:
|
|
|
155
191
|
|
|
156
192
|
@property
|
|
157
193
|
def blockquote(self) -> str:
|
|
158
|
-
return f"> See [{self.heading}](references/{self.
|
|
194
|
+
return f"> See [{self.heading}](references/{self.name}.md)"
|
|
159
195
|
|
|
160
196
|
@property
|
|
161
197
|
def ref_rel(self) -> str:
|
|
162
|
-
return f"references/{self.
|
|
198
|
+
return f"references/{self.name}.md"
|
|
163
199
|
|
|
164
200
|
|
|
165
201
|
def kebab_case(text: str) -> str:
|
|
@@ -200,6 +236,83 @@ def _headings(body: list[str]) -> list[tuple[int, int, str]]:
|
|
|
200
236
|
return out
|
|
201
237
|
|
|
202
238
|
|
|
239
|
+
# JavaScript's `\s`, what skill-check's `/\s+/` splits on: tab, LF, VT, FF,
|
|
240
|
+
# CR, space, U+00A0, U+1680, U+2000-U+200A, U+2028, U+2029, U+202F, U+205F,
|
|
241
|
+
# U+3000 and U+FEFF. Python's `str.split()` differs: it also splits on
|
|
242
|
+
# U+001C-U+001F and U+0085, and never on U+FEFF.
|
|
243
|
+
_JS_WHITESPACE_RE = re.compile(
|
|
244
|
+
"[\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+"
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def count_words(text: str) -> int:
|
|
249
|
+
"""Words in `text`: the parts a JavaScript `/\\s+/` split leaves, empty parts dropped."""
|
|
250
|
+
return sum(1 for part in _JS_WHITESPACE_RE.split(text) if part)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def count_body_words(body: list[str]) -> int:
|
|
254
|
+
"""Number of body words (see `count_words`)."""
|
|
255
|
+
return count_words("\n".join(body))
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
# [MANUAL] markers as skf-hash-content.py reads them for the inventory:
|
|
259
|
+
# whitespace-tolerant, so a marker may run across lines. `_MANUAL_LOOSE_RE`
|
|
260
|
+
# also catches a marker too malformed for the inventory to read.
|
|
261
|
+
_MANUAL_OPEN_RE = re.compile(r"<!--\s*\[MANUAL:([^\]]+)\]\s*-->")
|
|
262
|
+
_MANUAL_CLOSE_RE = re.compile(r"<!--\s*\[/MANUAL:([^\]]+)\]\s*-->")
|
|
263
|
+
_MANUAL_LOOSE_RE = re.compile(r"<!--\s*\[/?MANUAL:")
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def manual_ranges(body: list[str]) -> list[tuple[int, int]]:
|
|
267
|
+
"""Half-open body line ranges [first, last + 1) the [MANUAL] blocks cover.
|
|
268
|
+
|
|
269
|
+
Read over the whole body, joined, so a block opened before a section and
|
|
270
|
+
closed after it, and a marker split across lines, are both found. A block
|
|
271
|
+
is an open marker paired with the earliest following close marker of the
|
|
272
|
+
same name, as skf-hash-content.py pairs them; an unpaired marker, or a
|
|
273
|
+
marker too malformed to pair, covers its own lines.
|
|
274
|
+
"""
|
|
275
|
+
text = "\n".join(body)
|
|
276
|
+
starts: list[int] = []
|
|
277
|
+
offset = 0
|
|
278
|
+
for line in body:
|
|
279
|
+
starts.append(offset)
|
|
280
|
+
offset += len(line) + 1
|
|
281
|
+
|
|
282
|
+
def line_of(pos: int) -> int:
|
|
283
|
+
return bisect.bisect_right(starts, pos) - 1
|
|
284
|
+
|
|
285
|
+
def span(begin: int, finish: int) -> tuple[int, int]:
|
|
286
|
+
return line_of(begin), line_of(max(begin, finish - 1)) + 1
|
|
287
|
+
|
|
288
|
+
closes = [(m.start(), m.end(), m.group(1).strip()) for m in _MANUAL_CLOSE_RE.finditer(text)]
|
|
289
|
+
ranges: list[tuple[int, int]] = []
|
|
290
|
+
paired: set[int] = set()
|
|
291
|
+
for om in _MANUAL_OPEN_RE.finditer(text):
|
|
292
|
+
name = om.group(1).strip()
|
|
293
|
+
close = next((c for c in closes if c[0] >= om.end() and c[2] == name), None)
|
|
294
|
+
if close is None:
|
|
295
|
+
ranges.append(span(om.start(), om.end()))
|
|
296
|
+
else:
|
|
297
|
+
paired.add(close[0])
|
|
298
|
+
ranges.append(span(om.start(), close[1]))
|
|
299
|
+
for c_start, c_end, _ in closes:
|
|
300
|
+
if c_start not in paired:
|
|
301
|
+
ranges.append(span(c_start, c_end))
|
|
302
|
+
for m in _MANUAL_LOOSE_RE.finditer(text):
|
|
303
|
+
ranges.append(span(m.start(), m.end()))
|
|
304
|
+
return ranges
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def holds_manual(
|
|
308
|
+
body: list[str], sec: "Section", ranges: list[tuple[int, int]] | None = None
|
|
309
|
+
) -> bool:
|
|
310
|
+
"""True when the section intersects a [MANUAL] block (see `manual_ranges`)."""
|
|
311
|
+
if ranges is None:
|
|
312
|
+
ranges = manual_ranges(body)
|
|
313
|
+
return any(sec.start < last and first < sec.end for first, last in ranges)
|
|
314
|
+
|
|
315
|
+
|
|
203
316
|
def find_full_sections(body: list[str]) -> list[Section]:
|
|
204
317
|
"""Enumerate Tier-2 `## Full …` sections in document order.
|
|
205
318
|
|
|
@@ -250,25 +363,69 @@ def rebuild_body(body: list[str], selected: list[Section]) -> list[str]:
|
|
|
250
363
|
return out
|
|
251
364
|
|
|
252
365
|
|
|
253
|
-
def
|
|
254
|
-
body
|
|
255
|
-
|
|
256
|
-
|
|
366
|
+
def fits(lines: int, words: int, budget: int, word_budget: int) -> bool:
|
|
367
|
+
"""True when the body is within the line budget and, when set, the word budget."""
|
|
368
|
+
return lines <= budget and (word_budget <= 0 or words <= word_budget)
|
|
369
|
+
|
|
257
370
|
|
|
258
|
-
|
|
259
|
-
|
|
371
|
+
def select_sections(
|
|
372
|
+
body: list[str], sections: list[Section], budget: int, word_budget: int = 0
|
|
373
|
+
) -> tuple[list[Section], list[str], int, int]:
|
|
374
|
+
"""Greedily extract one section at a time until the body fits, or all are taken.
|
|
375
|
+
|
|
376
|
+
The body fits when its lines are <= `budget` and, when `word_budget` is
|
|
377
|
+
above 0, its words are <= `word_budget`. Whichever budget the body still
|
|
378
|
+
exceeds picks the next section: the largest by lines while the lines
|
|
379
|
+
exceed `budget`, then the largest by words. `max` keeps the first of equal
|
|
380
|
+
sections, so ties go in document order: deterministic. Returns (selected,
|
|
381
|
+
new_body, body_lines_after, body_words_after).
|
|
260
382
|
"""
|
|
261
|
-
|
|
383
|
+
words = {sec.start: count_words("\n".join(body[sec.start : sec.end])) for sec in sections}
|
|
384
|
+
remaining = list(sections)
|
|
262
385
|
selected: list[Section] = []
|
|
263
386
|
new_body = body
|
|
264
387
|
after = count_body_lines(body)
|
|
265
|
-
|
|
266
|
-
|
|
388
|
+
words_after = count_body_words(body)
|
|
389
|
+
while remaining and not fits(after, words_after, budget, word_budget):
|
|
390
|
+
if after > budget:
|
|
391
|
+
pick = max(remaining, key=lambda s: s.lines)
|
|
392
|
+
else:
|
|
393
|
+
pick = max(remaining, key=lambda s: words[s.start])
|
|
394
|
+
remaining.remove(pick)
|
|
395
|
+
selected.append(pick)
|
|
267
396
|
new_body = rebuild_body(body, selected)
|
|
268
397
|
after = count_body_lines(new_body)
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
398
|
+
words_after = count_body_words(new_body)
|
|
399
|
+
return selected, new_body, after, words_after
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _same_content(path: Path, content: str) -> bool:
|
|
403
|
+
try:
|
|
404
|
+
return path.is_file() and path.read_text(encoding="utf-8") == content
|
|
405
|
+
except (OSError, UnicodeDecodeError):
|
|
406
|
+
return False
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def assign_names(body: list[str], selected: list[Section], references_dir: Path) -> None:
|
|
410
|
+
"""Give each selected section a reference file name that overwrites nothing.
|
|
411
|
+
|
|
412
|
+
In document order, each takes `{kebab}.md`, or the first free
|
|
413
|
+
`{kebab}-2.md`, `{kebab}-3.md`, ... when that file already holds other
|
|
414
|
+
content or an earlier section of this run took the name. A file that
|
|
415
|
+
already holds exactly the section's content is reused.
|
|
416
|
+
"""
|
|
417
|
+
taken: set[str] = set()
|
|
418
|
+
for sec in sorted(selected, key=lambda s: s.start):
|
|
419
|
+
content = _section_content(body, sec)
|
|
420
|
+
n = 1
|
|
421
|
+
while True:
|
|
422
|
+
name = sec.kebab if n == 1 else f"{sec.kebab}-{n}"
|
|
423
|
+
path = references_dir / f"{name}.md"
|
|
424
|
+
if name not in taken and (not path.exists() or _same_content(path, content)):
|
|
425
|
+
break
|
|
426
|
+
n += 1
|
|
427
|
+
sec.name = name
|
|
428
|
+
taken.add(name)
|
|
272
429
|
|
|
273
430
|
|
|
274
431
|
# --------------------------------------------------------------------------
|
|
@@ -325,40 +482,57 @@ def _cmd_shard(args: argparse.Namespace) -> int:
|
|
|
325
482
|
return 1
|
|
326
483
|
|
|
327
484
|
budget: int = args.budget
|
|
485
|
+
word_budget: int = args.word_budget
|
|
328
486
|
dry_run: bool = args.dry_run
|
|
329
487
|
|
|
330
488
|
frontmatter, body = split_frontmatter(text)
|
|
331
489
|
body_lines_before = count_body_lines(body)
|
|
490
|
+
body_words_before = count_body_words(body)
|
|
332
491
|
tier1_before = present_tier1(body)
|
|
333
492
|
|
|
334
493
|
if args.verbose:
|
|
335
494
|
print(
|
|
336
|
-
f"skf-shard-body: body_lines_before={body_lines_before} budget={budget}"
|
|
495
|
+
f"skf-shard-body: body_lines_before={body_lines_before} budget={budget} "
|
|
496
|
+
f"body_words_before={body_words_before} word_budget={word_budget}",
|
|
337
497
|
file=sys.stderr,
|
|
338
498
|
)
|
|
339
499
|
|
|
340
500
|
# --- Skip path: already within budget -----------------------------------
|
|
341
|
-
if body_lines_before
|
|
501
|
+
if fits(body_lines_before, body_words_before, budget, word_budget):
|
|
342
502
|
_emit(
|
|
343
503
|
{
|
|
344
504
|
"action": "skip",
|
|
345
505
|
"body_lines_before": body_lines_before,
|
|
346
506
|
"body_lines_after": body_lines_before,
|
|
507
|
+
"body_words_before": body_words_before,
|
|
508
|
+
"body_words_after": body_words_before,
|
|
347
509
|
"sections_extracted": [],
|
|
348
510
|
"tier1_preserved": True,
|
|
349
511
|
"tier1_missing": [],
|
|
350
512
|
"xref_ok": True,
|
|
351
513
|
"under_budget": True,
|
|
514
|
+
"under_word_budget": True,
|
|
352
515
|
"dry_run": dry_run,
|
|
353
516
|
"budget": budget,
|
|
517
|
+
"word_budget": word_budget,
|
|
354
518
|
},
|
|
355
519
|
args.output,
|
|
356
520
|
)
|
|
357
521
|
return 0
|
|
358
522
|
|
|
359
523
|
# --- Shard path ---------------------------------------------------------
|
|
524
|
+
references_dir = skill_md.parent / "references"
|
|
360
525
|
sections = find_full_sections(body)
|
|
361
|
-
|
|
526
|
+
if args.skip_manual:
|
|
527
|
+
ranges = manual_ranges(body)
|
|
528
|
+
sections = [sec for sec in sections if not holds_manual(body, sec, ranges)]
|
|
529
|
+
selected, new_body, body_lines_after, body_words_after = select_sections(
|
|
530
|
+
body, sections, budget, word_budget
|
|
531
|
+
)
|
|
532
|
+
# A reference file's name never changes a count (no whitespace in it), so
|
|
533
|
+
# the names are picked once the selection is final.
|
|
534
|
+
assign_names(body, selected, references_dir)
|
|
535
|
+
new_body = rebuild_body(body, selected)
|
|
362
536
|
|
|
363
537
|
tier1_after = present_tier1(new_body)
|
|
364
538
|
tier1_missing = [
|
|
@@ -366,28 +540,32 @@ def _cmd_shard(args: argparse.Namespace) -> int:
|
|
|
366
540
|
]
|
|
367
541
|
tier1_preserved = not tier1_missing
|
|
368
542
|
under_budget = body_lines_after <= budget
|
|
369
|
-
|
|
370
|
-
references_dir = skill_md.parent / "references"
|
|
543
|
+
under_word_budget = word_budget <= 0 or body_words_after <= word_budget
|
|
371
544
|
|
|
372
545
|
# Compute the plan fully before touching disk. Only write when we have a
|
|
373
546
|
# real extraction AND Tier-1 survived — never leave a half-mutated tree.
|
|
374
547
|
should_write = (not dry_run) and bool(selected) and tier1_preserved
|
|
375
548
|
if should_write:
|
|
549
|
+
written: list[str] = []
|
|
376
550
|
try:
|
|
377
551
|
for sec in selected:
|
|
378
552
|
_atomic_write(
|
|
379
|
-
references_dir / f"{sec.
|
|
553
|
+
references_dir / f"{sec.name}.md", _section_content(body, sec)
|
|
380
554
|
)
|
|
555
|
+
written.append(sec.ref_rel)
|
|
381
556
|
new_text = "\n".join(frontmatter + new_body) + "\n"
|
|
382
557
|
_atomic_write(skill_md, new_text)
|
|
383
558
|
except OSError as exc:
|
|
384
|
-
print(
|
|
559
|
+
print(
|
|
560
|
+
f"error: {exc}; already written: {', '.join(written) or 'none'}",
|
|
561
|
+
file=sys.stderr,
|
|
562
|
+
)
|
|
385
563
|
return 2
|
|
386
564
|
|
|
387
565
|
blockquotes_ok = all(sec.blockquote in new_body for sec in selected)
|
|
388
566
|
if should_write:
|
|
389
567
|
xref_ok = blockquotes_ok and all(
|
|
390
|
-
(references_dir / f"{sec.
|
|
568
|
+
(references_dir / f"{sec.name}.md").is_file() for sec in selected
|
|
391
569
|
)
|
|
392
570
|
else:
|
|
393
571
|
# dry-run, nothing-to-extract, or aborted tier1 violation
|
|
@@ -398,6 +576,8 @@ def _cmd_shard(args: argparse.Namespace) -> int:
|
|
|
398
576
|
"action": "shard",
|
|
399
577
|
"body_lines_before": body_lines_before,
|
|
400
578
|
"body_lines_after": body_lines_after,
|
|
579
|
+
"body_words_before": body_words_before,
|
|
580
|
+
"body_words_after": body_words_after,
|
|
401
581
|
"sections_extracted": [
|
|
402
582
|
{"heading": sec.heading, "file": sec.ref_rel, "lines": sec.lines}
|
|
403
583
|
for sec in selected
|
|
@@ -406,8 +586,10 @@ def _cmd_shard(args: argparse.Namespace) -> int:
|
|
|
406
586
|
"tier1_missing": tier1_missing,
|
|
407
587
|
"xref_ok": xref_ok,
|
|
408
588
|
"under_budget": under_budget,
|
|
589
|
+
"under_word_budget": under_word_budget,
|
|
409
590
|
"dry_run": dry_run,
|
|
410
591
|
"budget": budget,
|
|
592
|
+
"word_budget": word_budget,
|
|
411
593
|
},
|
|
412
594
|
args.output,
|
|
413
595
|
)
|
|
@@ -420,9 +602,10 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
420
602
|
description=(
|
|
421
603
|
"Deterministically shard an oversized SKILL.md body: extract the "
|
|
422
604
|
"largest `## Full …` Tier-2 sections to references/ until the body "
|
|
423
|
-
"fits under --budget, replacing each
|
|
424
|
-
"blockquote, and emit a JSON report the
|
|
425
|
-
"steps consume instead of
|
|
605
|
+
"fits under --budget (and --word-budget, when set), replacing each "
|
|
606
|
+
"with a cross-reference blockquote, and emit a JSON report the "
|
|
607
|
+
"auto-shard / validate / update write steps consume instead of "
|
|
608
|
+
"counting and splitting by hand."
|
|
426
609
|
),
|
|
427
610
|
)
|
|
428
611
|
parser.add_argument("skill_md", help="path to the staging SKILL.md file")
|
|
@@ -432,6 +615,21 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
432
615
|
default=400,
|
|
433
616
|
help="body line budget below which no shard occurs (default 400)",
|
|
434
617
|
)
|
|
618
|
+
parser.add_argument(
|
|
619
|
+
"--word-budget",
|
|
620
|
+
type=int,
|
|
621
|
+
default=0,
|
|
622
|
+
help=(
|
|
623
|
+
"body word budget (words split as JavaScript's /\\s+/ splits them, "
|
|
624
|
+
"as skill-check's body.max_tokens estimate does); 0 turns it off "
|
|
625
|
+
"(default 0)"
|
|
626
|
+
),
|
|
627
|
+
)
|
|
628
|
+
parser.add_argument(
|
|
629
|
+
"--skip-manual",
|
|
630
|
+
action="store_true",
|
|
631
|
+
help="never move a `## Full` section that intersects a [MANUAL] block",
|
|
632
|
+
)
|
|
435
633
|
parser.add_argument(
|
|
436
634
|
"--dry-run",
|
|
437
635
|
action="store_true",
|