bmad-module-skill-forge 3.0.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +3 -3
  3. package/docs/_data/pinned.yaml +19 -13
  4. package/docs/architecture.md +2 -2
  5. package/docs/concepts.md +4 -4
  6. package/docs/examples.md +33 -26
  7. package/docs/getting-started.md +3 -3
  8. package/docs/how-it-works.md +2 -2
  9. package/docs/index.md +1 -1
  10. package/docs/skill-model.md +41 -39
  11. package/docs/troubleshooting.md +8 -0
  12. package/docs/verifying-a-skill.md +46 -21
  13. package/docs/why-skf.md +1 -1
  14. package/docs/workflows.md +5 -5
  15. package/package.json +1 -1
  16. package/src/knowledge/qmd-registry.md +1 -1
  17. package/src/knowledge/version-paths.md +1 -1
  18. package/src/shared/health-check.md +2 -50
  19. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
  20. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +12 -4
  21. package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
  22. package/src/shared/scripts/skf-atomic-write.py +212 -59
  23. package/src/shared/scripts/skf-build-change-manifest.py +1437 -178
  24. package/src/shared/scripts/skf-classify-changed-files.py +78 -0
  25. package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
  26. package/src/shared/scripts/skf-count-tokens.py +54 -10
  27. package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
  28. package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
  29. package/src/shared/scripts/skf-detect-workspaces.py +67 -0
  30. package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
  31. package/src/shared/scripts/skf-extract-public-api.py +1673 -189
  32. package/src/shared/scripts/skf-extraction-inventory.py +584 -46
  33. package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
  34. package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
  35. package/src/shared/scripts/skf-github-probe.py +47 -8
  36. package/src/shared/scripts/skf-hash-content.py +14 -8
  37. package/src/shared/scripts/skf-load-provenance.py +3 -1
  38. package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
  39. package/src/shared/scripts/skf-parse-gaps.py +135 -26
  40. package/src/shared/scripts/skf-preflight.py +15 -1
  41. package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
  42. package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
  43. package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
  44. package/src/shared/scripts/skf-resolve-customization.py +197 -0
  45. package/src/shared/scripts/skf-shard-body.py +239 -41
  46. package/src/shared/scripts/skf-structural-diff.py +133 -18
  47. package/src/shared/scripts/skf-update-active-symlink.py +115 -62
  48. package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
  49. package/src/shared/scripts/skf-validate-output.py +12 -3
  50. package/src/shared/scripts/skf-verify-provenance-completeness.py +1439 -79
  51. package/src/shared/scripts/skf-write-skill-brief.py +41 -8
  52. package/src/shared/tool-requirements.yaml +8 -2
  53. package/src/skf-analyze-source/SKILL.md +2 -2
  54. package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
  55. package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
  56. package/src/skf-audit-skill/SKILL.md +2 -2
  57. package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
  58. package/src/skf-audit-skill/references/headless-contract.md +1 -1
  59. package/src/skf-audit-skill/references/health-check.md +2 -0
  60. package/src/skf-audit-skill/references/init.md +2 -1
  61. package/src/skf-audit-skill/references/re-index.md +14 -9
  62. package/src/skf-audit-skill/references/report.md +2 -2
  63. package/src/skf-audit-skill/references/semantic-diff.md +15 -15
  64. package/src/skf-audit-skill/references/structural-diff.md +10 -5
  65. package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
  66. package/src/skf-brief-skill/SKILL.md +3 -3
  67. package/src/skf-brief-skill/assets/scope-templates.md +1 -1
  68. package/src/skf-brief-skill/assets/skill-brief-schema.md +4 -11
  69. package/src/skf-brief-skill/references/analyze-target.md +6 -6
  70. package/src/skf-brief-skill/references/confirm-brief.md +1 -1
  71. package/src/skf-brief-skill/references/scope-definition.md +4 -4
  72. package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
  73. package/src/skf-brief-skill/references/version-resolution.md +5 -9
  74. package/src/skf-campaign/SKILL.md +2 -2
  75. package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
  76. package/src/skf-create-skill/SKILL.md +2 -2
  77. package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
  78. package/src/skf-create-skill/assets/skill-sections.md +7 -7
  79. package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
  80. package/src/skf-create-skill/references/compile.md +31 -60
  81. package/src/skf-create-skill/references/component-extraction.md +3 -3
  82. package/src/skf-create-skill/references/enrich.md +2 -2
  83. package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
  84. package/src/skf-create-skill/references/extract.md +23 -19
  85. package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
  86. package/src/skf-create-skill/references/extraction-patterns.md +10 -11
  87. package/src/skf-create-skill/references/generate-artifacts.md +1 -1
  88. package/src/skf-create-skill/references/report.md +21 -5
  89. package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
  90. package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
  91. package/src/skf-create-stack-skill/SKILL.md +2 -2
  92. package/src/skf-create-stack-skill/references/validate.md +1 -1
  93. package/src/skf-drop-skill/SKILL.md +2 -2
  94. package/src/skf-drop-skill/references/execute.md +5 -1
  95. package/src/skf-drop-skill/references/health-check.md +2 -0
  96. package/src/skf-drop-skill/references/invocation-contract.md +1 -1
  97. package/src/skf-drop-skill/references/report.md +2 -2
  98. package/src/skf-drop-skill/references/select.md +7 -3
  99. package/src/skf-export-skill/SKILL.md +2 -2
  100. package/src/skf-export-skill/assets/snippet-format.md +5 -5
  101. package/src/skf-export-skill/references/generate-snippet.md +22 -15
  102. package/src/skf-export-skill/references/load-skill.md +1 -1
  103. package/src/skf-export-skill/references/token-report.md +29 -9
  104. package/src/skf-forger/SKILL.md +2 -2
  105. package/src/skf-quick-skill/SKILL.md +2 -2
  106. package/src/skf-refine-architecture/SKILL.md +6 -8
  107. package/src/skf-rename-skill/SKILL.md +2 -2
  108. package/src/skf-setup/SKILL.md +2 -2
  109. package/src/skf-test-skill/SKILL.md +2 -2
  110. package/src/skf-test-skill/assets/output-section-formats.md +6 -4
  111. package/src/skf-test-skill/references/coherence-check.md +5 -4
  112. package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
  113. package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
  114. package/src/skf-test-skill/references/coverage-check.md +69 -111
  115. package/src/skf-test-skill/references/init.md +5 -6
  116. package/src/skf-test-skill/references/migration-section-rules.md +26 -14
  117. package/src/skf-test-skill/references/report.md +11 -9
  118. package/src/skf-test-skill/references/scoring-rules.md +8 -6
  119. package/src/skf-test-skill/references/source-access-protocol.md +15 -18
  120. package/src/skf-test-skill/scripts/build-result-context.py +114 -21
  121. package/src/skf-test-skill/scripts/gap-ledger.py +580 -80
  122. package/src/skf-test-skill/scripts/load-coverage-inputs.py +646 -107
  123. package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
  124. package/src/skf-test-skill/scripts/reconcile-coverage.py +450 -23
  125. package/src/skf-test-skill/scripts/score-signatures.py +356 -61
  126. package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
  127. package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
  128. package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
  129. package/src/skf-test-skill/templates/test-report-template.md +1 -0
  130. package/src/skf-update-skill/SKILL.md +2 -2
  131. package/src/skf-update-skill/references/detect-changes.md +32 -13
  132. package/src/skf-update-skill/references/gap-driven.md +37 -22
  133. package/src/skf-update-skill/references/init.md +6 -2
  134. package/src/skf-update-skill/references/invocation-contract.md +1 -1
  135. package/src/skf-update-skill/references/merge.md +17 -12
  136. package/src/skf-update-skill/references/re-extract.md +4 -2
  137. package/src/skf-update-skill/references/report.md +7 -4
  138. package/src/skf-update-skill/references/write.md +77 -36
  139. package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
  140. package/src/skf-verify-stack/SKILL.md +2 -2
  141. package/tools/cli/lib/tool-check.js +158 -2
@@ -0,0 +1,197 @@
1
+ # /// script
2
+ # requires-python = ">=3.11"
3
+ # dependencies = []
4
+ # ///
5
+ """SKF Resolve Customization: BMAD core's customization resolver, on any core from 6.4.0.
6
+
7
+ Every SKF workflow, and Ferris, reads its `_bmad/custom/` overrides through
8
+ BMAD core's customization resolver, `_bmad/scripts/resolve_customization.py`
9
+ under the project root, which the BMAD Method installer adds. SKF passes it
10
+ `--project-root`, a flag core takes only from 6.11.0: core 6.4.0 to 6.10.0
11
+ stops on it with argparse's exit 2, so SKF runs this helper in its place.
12
+ It takes core's long flags only (core's -s, -p and -k are refused):
13
+
14
+ uv run skf-resolve-customization.py --skill <skill-root> \\
15
+ --project-root <project-root> --key <key> [--key <key>]...
16
+
17
+ 1. It runs core's resolver with those arguments, `--project-root` included,
18
+ under its own interpreter, from the project root. `uv run` gives this
19
+ file a Python that meets its requires-python, which core needs for
20
+ `tomllib`; core 6.10.0 and earlier declare none, so uv would not choose
21
+ one for them.
22
+ 2. When core rejects `--project-root` (exit 2, nothing on stdout, and
23
+ `error: unrecognized arguments: ... --project-root` on stderr), it works
24
+ out the root a core before 6.11.0 picks without the flag: the nearest
25
+ folder, from the skill folder up and the skill folder itself first, that
26
+ holds `_bmad` or `.git` (a file or a folder), else the same search from
27
+ the project root. When that is the project root, it runs core again
28
+ without the flag, and that run's result is the helper's. Otherwise core
29
+ would read another folder's overrides, so it refuses.
30
+
31
+ Output:
32
+ stdout on exit 0, core's stdout byte for byte (the JSON core prints);
33
+ on any other exit, nothing.
34
+ stderr the stderr of the core run whose result is the helper's: on a
35
+ rerun without the flag, the rerun's only (the rejected call's is
36
+ dropped). A failure of the helper's own prints one line first: the
37
+ reason a workflow's warning quotes.
38
+
39
+ Exit codes:
40
+ 0 core resolved the customization
41
+ 1 core's resolver is missing (first stderr line: `not found`), could
42
+ not start, did not finish or was stopped by a signal, or core exited 1
43
+ (a customize.toml or an override it cannot read)
44
+ 2 a core before 6.11.0 would read the overrides under another folder
45
+ (first stderr line: `BMAD core before 6.11.0: ...`), a usage error of
46
+ the helper's, or core's own exit 2 (a core from 6.11.0), passed through
47
+ 3 core found no Python 3.11 or later (`tomllib`)
48
+ Any other exit code is core's own, passed through.
49
+ """
50
+
51
+ from __future__ import annotations
52
+
53
+ import argparse
54
+ import os
55
+ import re
56
+ import subprocess
57
+ import sys
58
+ from pathlib import Path
59
+
60
+ CORE_RESOLVER = ("_bmad", "scripts", "resolve_customization.py")
61
+ FIRST_CORE_WITH_FLAG = "6.11.0"
62
+ CORE_TIMEOUT_SEC = 120
63
+ PROJECT_ROOT_FLAG = "--project-root"
64
+ MISSING = "not found"
65
+ ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
66
+ # argparse's line for a flag it does not know, wherever the usage line wraps.
67
+ REJECTED_FLAG_RE = re.compile(r"error: unrecognized arguments: (?:.*\s)?--project-root(?:[=\s]|$)", re.M)
68
+
69
+
70
+ def _build_parser() -> argparse.ArgumentParser:
71
+ parser = argparse.ArgumentParser(
72
+ prog="skf-resolve-customization",
73
+ description=(
74
+ "Run BMAD core's customization resolver with --project-root. On a core before "
75
+ f"{FIRST_CORE_WITH_FLAG}, which rejects that flag, run it again without the flag when "
76
+ "core would read this project's overrides anyway, and refuse otherwise."
77
+ ),
78
+ )
79
+ parser.add_argument("--skill", required=True, help="the skill folder that holds customize.toml")
80
+ parser.add_argument(
81
+ "--project-root",
82
+ required=True,
83
+ help="the project folder: core's resolver is under its _bmad/scripts/, the overrides under _bmad/custom/",
84
+ )
85
+ parser.add_argument(
86
+ "--key",
87
+ action="append",
88
+ default=[],
89
+ help="a dotted key to resolve, repeatable; with none, core prints the whole merge",
90
+ )
91
+ return parser
92
+
93
+
94
+ def _nearest_marked(start: Path) -> Path | None:
95
+ """The nearest folder, `start` itself first, holding `_bmad` or `.git` (a file or a folder)."""
96
+ current = start.resolve()
97
+ while True:
98
+ if (current / "_bmad").exists() or (current / ".git").exists():
99
+ return current
100
+ if current.parent == current:
101
+ return None
102
+ current = current.parent
103
+
104
+
105
+ def legacy_project_root(skill: Path, cwd: Path) -> Path | None:
106
+ """The project root BMAD core 6.4.0 to 6.10.0 picks without --project-root, run from `cwd`."""
107
+ return _nearest_marked(skill) or _nearest_marked(cwd)
108
+
109
+
110
+ def _same_folder(a: Path, b: Path) -> bool:
111
+ try:
112
+ return os.path.samefile(a, b)
113
+ except OSError:
114
+ return a.resolve() == b.resolve()
115
+
116
+
117
+ def rejects_project_root(returncode: int, stdout: bytes, stderr: bytes) -> bool:
118
+ """True when core stopped on --project-root as a flag it does not know; never on exit 2 alone."""
119
+ if returncode != 2 or stdout:
120
+ return False
121
+ text = ANSI_RE.sub("", stderr.decode("utf-8", "replace"))
122
+ return REJECTED_FLAG_RE.search(text) is not None
123
+
124
+
125
+ def _run_core(argv: list[str], cwd: Path) -> subprocess.CompletedProcess:
126
+ env = dict(os.environ)
127
+ # Core 6.11.0 and later import config_utils from their own folder, which a safe path drops.
128
+ env.pop("PYTHONSAFEPATH", None)
129
+ # Core before 6.8.0 writes its JSON in the locale's encoding.
130
+ env["PYTHONIOENCODING"] = "utf-8"
131
+ return subprocess.run(
132
+ argv, cwd=cwd, env=env, stdin=subprocess.DEVNULL, capture_output=True,
133
+ timeout=CORE_TIMEOUT_SEC, check=False,
134
+ )
135
+
136
+
137
+ def _write(stream, data: bytes) -> None:
138
+ if not data:
139
+ return
140
+ buffer = getattr(stream, "buffer", None)
141
+ if buffer is None:
142
+ stream.write(data.decode("utf-8", "replace"))
143
+ stream.flush()
144
+ return
145
+ buffer.write(data)
146
+ buffer.flush()
147
+
148
+
149
+ def _say(*lines: str) -> None:
150
+ _write(sys.stderr, "".join(line + "\n" for line in lines).encode("utf-8"))
151
+
152
+
153
+ def _finish(proc: subprocess.CompletedProcess) -> int:
154
+ """Core's result as the helper's: its stdout only on exit 0, its stderr always."""
155
+ if proc.returncode < 0: # stopped by a signal: core may have printed no reason
156
+ _say(f"BMAD core's customization resolver was stopped by signal {-proc.returncode}")
157
+ _write(sys.stderr, proc.stderr)
158
+ return 1
159
+ if proc.returncode == 0:
160
+ _write(sys.stdout, proc.stdout)
161
+ _write(sys.stderr, proc.stderr)
162
+ return proc.returncode
163
+
164
+
165
+ def main(argv: list[str] | None = None) -> int:
166
+ args = _build_parser().parse_args(argv)
167
+ root = Path(os.path.abspath(args.project_root))
168
+ skill = Path(os.path.abspath(args.skill))
169
+ core = root.joinpath(*CORE_RESOLVER)
170
+ if not core.is_file():
171
+ _say(MISSING, f"BMAD core's customization resolver is not at {core}: the BMAD Method installer adds it.")
172
+ return 1
173
+ command = [sys.executable, str(core), "--skill", str(skill)]
174
+ keys = [word for key in args.key for word in ("--key", key)]
175
+ try:
176
+ first = _run_core([*command, PROJECT_ROOT_FLAG, str(root), *keys], root)
177
+ if not rejects_project_root(first.returncode, first.stdout, first.stderr):
178
+ return _finish(first)
179
+ legacy = legacy_project_root(skill, root)
180
+ if legacy is not None and _same_folder(legacy, root):
181
+ return _finish(_run_core([*command, *keys], root))
182
+ except subprocess.TimeoutExpired:
183
+ _say(f"BMAD core's customization resolver did not finish within {CORE_TIMEOUT_SEC} seconds")
184
+ return 1
185
+ except OSError as exc:
186
+ _say(f"BMAD core's customization resolver could not start: {exc}")
187
+ return 1
188
+ _say(
189
+ f"BMAD core before {FIRST_CORE_WITH_FLAG}: its customization resolver takes no {PROJECT_ROOT_FLAG} "
190
+ f"and would read the overrides under {legacy}, not under {root}; upgrade BMAD core to "
191
+ f"{FIRST_CORE_WITH_FLAG} or later"
192
+ )
193
+ return 2
194
+
195
+
196
+ if __name__ == "__main__":
197
+ sys.exit(main())
@@ -8,42 +8,70 @@
8
8
  Owns the line counting, the section boundaries, the size sort, the file
9
9
  writes and the blockquote replacements of `skf-create-skill`'s
10
10
  `references/step-auto-shard.md` §0 (which runs it on every pass and keeps no
11
- by-hand fallback) and `references/validate.md` §4: counted by eye, an
12
- over-budget body ships silently or the step halts for nothing.
11
+ by-hand fallback) and `references/validate.md` §4, and of
12
+ `skf-update-skill`'s `references/write.md` §7 when skill-check reports
13
+ `body.max_lines` or `body.max_tokens`: counted by eye, an over-budget body
14
+ ships silently or the step halts for nothing.
13
15
 
14
16
  What it does (single positional invocation):
15
17
 
16
18
  1. Split frontmatter (leading `---` … `---`) from body. Frontmatter is
17
19
  NEVER modified.
18
20
  2. Count body lines between the frontmatter close and EOF, excluding
19
- trailing blank lines → `body_lines_before`.
20
- 3. If `body_lines_before` <= `--budget` (default 400): emit
21
- `action: "skip"`, write nothing.
21
+ trailing blank lines → `body_lines_before`, and body words: the
22
+ parts left after splitting the body on JavaScript's `\\s` whitespace
23
+ (`/\\s+/`), empty parts dropped → `body_words_before`: the same split
24
+ skill-check's `body.max_tokens` estimate makes
25
+ (`text.split(/\\s+/).filter(Boolean)`), which update-skill's write.md
26
+ §7 holds under a budget below skill-check's limit.
27
+ 3. If `body_lines_before` <= `--budget` (default 400) and, when
28
+ `--word-budget` is set (default 0, off), `body_words_before` <=
29
+ `--word-budget`: emit `action: "skip"`, write nothing.
22
30
  4. Otherwise enumerate Tier-2 sections — level-2 headings whose text
23
31
  starts with `Full` (`## Full API Reference`, `## Full Type
24
32
  Definitions`, …). Each section spans its heading through the line
25
- before the next level-1/level-2 heading (or EOF).
26
- 5. Sort candidates by line count descending (largest first, stable) and
27
- greedily extract the largest, re-counting after each, stopping the
28
- moment the body fits under budget. If selective extraction of every
29
- Tier-2 section still cannot fit, extract them all and report
30
- `under_budget: false` (the caller then trims Tier-1 by editing
31
- judgment — that fallback stays in the prompt).
33
+ before the next level-1/level-2 heading (or EOF). With
34
+ `--skip-manual`, a section that intersects a [MANUAL] block is never
35
+ a candidate: a block is an open marker paired with the earliest
36
+ following close marker of the same name, read over the whole body
37
+ (so a block opened before a section and closed after it counts, and
38
+ a marker split across lines is found, as skf-hash-content.py reads
39
+ them), and a marker left unpaired protects its own lines. update-skill
40
+ inventories [MANUAL] blocks in SKILL.md only, so a moved block would
41
+ leave its next inventory.
42
+ 5. Greedily extract one candidate at a time, re-counting after each,
43
+ stopping the moment the body fits: lines <= `--budget` and, when set,
44
+ words <= `--word-budget`. While the lines exceed the budget the next
45
+ candidate is the largest by lines; once they fit, the largest by
46
+ words (ties keep document order). If extracting every candidate
47
+ still cannot fit, extract them all and report `under_budget: false`
48
+ or `under_word_budget: false`. create-skill then trims Tier-1 by
49
+ editing judgment (that fallback stays in the prompt); update-skill
50
+ records a warning and trims nothing. With no candidate, nothing is
51
+ written.
32
52
  6. Each extracted section is written to `references/{kebab(heading)}.md`
33
53
  (heading preserved) via `skf-atomic-write.py`, and the section in
34
54
  SKILL.md is replaced by a `> See [Title](references/…md)` blockquote.
35
- The rewritten SKILL.md is itself written via `skf-atomic-write.py`.
55
+ A file is never overwritten: when `references/{kebab}.md` already
56
+ holds other content, or an earlier extracted section of this run took
57
+ the name, the next free `{kebab}-2.md`, `{kebab}-3.md`, … is used (as
58
+ skill-check's split-body names them), in document order, and the
59
+ blockquote points at it. The rewritten SKILL.md is itself written via
60
+ `skf-atomic-write.py`.
36
61
  7. Verify every Tier-1 heading that was inline before extraction is still
37
62
  inline (`tier1_preserved`, with `tier1_missing[]`), and every emitted
38
63
  blockquote resolves to a written reference file (`xref_ok`).
39
64
 
40
- Deterministic: same input + budget → same JSON and same on-disk result.
65
+ Deterministic: same input + budgets + flags → same JSON and same on-disk
66
+ result.
41
67
 
42
68
  Output JSON (stdout, or `-o`):
43
69
  {
44
70
  "action": "skip" | "shard",
45
71
  "body_lines_before": <int>,
46
72
  "body_lines_after": <int>,
73
+ "body_words_before": <int>,
74
+ "body_words_after": <int>,
47
75
  "sections_extracted": [
48
76
  {"heading": "Full API Reference",
49
77
  "file": "references/full-api-reference.md",
@@ -53,8 +81,10 @@ Output JSON (stdout, or `-o`):
53
81
  "tier1_missing": ["Overview", ...],
54
82
  "xref_ok": <bool>,
55
83
  "under_budget": <bool>,
84
+ "under_word_budget": <bool>, (true when --word-budget is 0)
56
85
  "dry_run": <bool>,
57
- "budget": <int>
86
+ "budget": <int>,
87
+ "word_budget": <int> (0 when off)
58
88
  }
59
89
 
60
90
  Heading parsing reuses `skf-scan-skill-md-structure.py`'s `_HEADING_RE`
@@ -68,11 +98,14 @@ Exit codes:
68
98
  gates on `tier1_preserved` / `xref_ok`.
69
99
  1 — user error (file not found, unreadable, bad frontmatter).
70
100
  2 — operation failure (a reference/SKILL.md write could not complete).
101
+ The stderr line names the file that failed and each reference file
102
+ already written (`already written: none` when there is none).
71
103
  """
72
104
 
73
105
  from __future__ import annotations
74
106
 
75
107
  import argparse
108
+ import bisect
76
109
  import importlib.util
77
110
  import json
78
111
  import re
@@ -141,13 +174,16 @@ TIER1_HEADINGS: list[str] = [
141
174
  class Section:
142
175
  """A Tier-2 (`## Full …`) section as a half-open line range [start, end)."""
143
176
 
144
- __slots__ = ("start", "end", "heading", "kebab")
177
+ __slots__ = ("start", "end", "heading", "kebab", "name")
145
178
 
146
179
  def __init__(self, start: int, end: int, heading: str) -> None:
147
180
  self.start = start
148
181
  self.end = end
149
182
  self.heading = heading
150
183
  self.kebab = kebab_case(heading)
184
+ # The reference file's stem: the kebab name, or a `-2`, `-3` …
185
+ # variant `assign_names` picks so no file is overwritten.
186
+ self.name = self.kebab
151
187
 
152
188
  @property
153
189
  def lines(self) -> int:
@@ -155,11 +191,11 @@ class Section:
155
191
 
156
192
  @property
157
193
  def blockquote(self) -> str:
158
- return f"> See [{self.heading}](references/{self.kebab}.md)"
194
+ return f"> See [{self.heading}](references/{self.name}.md)"
159
195
 
160
196
  @property
161
197
  def ref_rel(self) -> str:
162
- return f"references/{self.kebab}.md"
198
+ return f"references/{self.name}.md"
163
199
 
164
200
 
165
201
  def kebab_case(text: str) -> str:
@@ -200,6 +236,83 @@ def _headings(body: list[str]) -> list[tuple[int, int, str]]:
200
236
  return out
201
237
 
202
238
 
239
+ # JavaScript's `\s`, what skill-check's `/\s+/` splits on: tab, LF, VT, FF,
240
+ # CR, space, U+00A0, U+1680, U+2000-U+200A, U+2028, U+2029, U+202F, U+205F,
241
+ # U+3000 and U+FEFF. Python's `str.split()` differs: it also splits on
242
+ # U+001C-U+001F and U+0085, and never on U+FEFF.
243
+ _JS_WHITESPACE_RE = re.compile(
244
+ "[\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+"
245
+ )
246
+
247
+
248
+ def count_words(text: str) -> int:
249
+ """Words in `text`: the parts a JavaScript `/\\s+/` split leaves, empty parts dropped."""
250
+ return sum(1 for part in _JS_WHITESPACE_RE.split(text) if part)
251
+
252
+
253
+ def count_body_words(body: list[str]) -> int:
254
+ """Number of body words (see `count_words`)."""
255
+ return count_words("\n".join(body))
256
+
257
+
258
+ # [MANUAL] markers as skf-hash-content.py reads them for the inventory:
259
+ # whitespace-tolerant, so a marker may run across lines. `_MANUAL_LOOSE_RE`
260
+ # also catches a marker too malformed for the inventory to read.
261
+ _MANUAL_OPEN_RE = re.compile(r"<!--\s*\[MANUAL:([^\]]+)\]\s*-->")
262
+ _MANUAL_CLOSE_RE = re.compile(r"<!--\s*\[/MANUAL:([^\]]+)\]\s*-->")
263
+ _MANUAL_LOOSE_RE = re.compile(r"<!--\s*\[/?MANUAL:")
264
+
265
+
266
+ def manual_ranges(body: list[str]) -> list[tuple[int, int]]:
267
+ """Half-open body line ranges [first, last + 1) the [MANUAL] blocks cover.
268
+
269
+ Read over the whole body, joined, so a block opened before a section and
270
+ closed after it, and a marker split across lines, are both found. A block
271
+ is an open marker paired with the earliest following close marker of the
272
+ same name, as skf-hash-content.py pairs them; an unpaired marker, or a
273
+ marker too malformed to pair, covers its own lines.
274
+ """
275
+ text = "\n".join(body)
276
+ starts: list[int] = []
277
+ offset = 0
278
+ for line in body:
279
+ starts.append(offset)
280
+ offset += len(line) + 1
281
+
282
+ def line_of(pos: int) -> int:
283
+ return bisect.bisect_right(starts, pos) - 1
284
+
285
+ def span(begin: int, finish: int) -> tuple[int, int]:
286
+ return line_of(begin), line_of(max(begin, finish - 1)) + 1
287
+
288
+ closes = [(m.start(), m.end(), m.group(1).strip()) for m in _MANUAL_CLOSE_RE.finditer(text)]
289
+ ranges: list[tuple[int, int]] = []
290
+ paired: set[int] = set()
291
+ for om in _MANUAL_OPEN_RE.finditer(text):
292
+ name = om.group(1).strip()
293
+ close = next((c for c in closes if c[0] >= om.end() and c[2] == name), None)
294
+ if close is None:
295
+ ranges.append(span(om.start(), om.end()))
296
+ else:
297
+ paired.add(close[0])
298
+ ranges.append(span(om.start(), close[1]))
299
+ for c_start, c_end, _ in closes:
300
+ if c_start not in paired:
301
+ ranges.append(span(c_start, c_end))
302
+ for m in _MANUAL_LOOSE_RE.finditer(text):
303
+ ranges.append(span(m.start(), m.end()))
304
+ return ranges
305
+
306
+
307
+ def holds_manual(
308
+ body: list[str], sec: "Section", ranges: list[tuple[int, int]] | None = None
309
+ ) -> bool:
310
+ """True when the section intersects a [MANUAL] block (see `manual_ranges`)."""
311
+ if ranges is None:
312
+ ranges = manual_ranges(body)
313
+ return any(sec.start < last and first < sec.end for first, last in ranges)
314
+
315
+
203
316
  def find_full_sections(body: list[str]) -> list[Section]:
204
317
  """Enumerate Tier-2 `## Full …` sections in document order.
205
318
 
@@ -250,25 +363,69 @@ def rebuild_body(body: list[str], selected: list[Section]) -> list[str]:
250
363
  return out
251
364
 
252
365
 
253
- def select_sections(
254
- body: list[str], sections: list[Section], budget: int
255
- ) -> tuple[list[Section], list[str], int]:
256
- """Greedily extract largest-first until the body fits, or all are taken.
366
+ def fits(lines: int, words: int, budget: int, word_budget: int) -> bool:
367
+ """True when the body is within the line budget and, when set, the word budget."""
368
+ return lines <= budget and (word_budget <= 0 or words <= word_budget)
369
+
257
370
 
258
- Returns (selected, new_body, body_lines_after). `sorted(reverse=True)` is
259
- stable, so equal-size sections keep document order — deterministic.
371
+ def select_sections(
372
+ body: list[str], sections: list[Section], budget: int, word_budget: int = 0
373
+ ) -> tuple[list[Section], list[str], int, int]:
374
+ """Greedily extract one section at a time until the body fits, or all are taken.
375
+
376
+ The body fits when its lines are <= `budget` and, when `word_budget` is
377
+ above 0, its words are <= `word_budget`. Whichever budget the body still
378
+ exceeds picks the next section: the largest by lines while the lines
379
+ exceed `budget`, then the largest by words. `max` keeps the first of equal
380
+ sections, so ties go in document order: deterministic. Returns (selected,
381
+ new_body, body_lines_after, body_words_after).
260
382
  """
261
- candidates = sorted(sections, key=lambda s: s.lines, reverse=True)
383
+ words = {sec.start: count_words("\n".join(body[sec.start : sec.end])) for sec in sections}
384
+ remaining = list(sections)
262
385
  selected: list[Section] = []
263
386
  new_body = body
264
387
  after = count_body_lines(body)
265
- for sec in candidates:
266
- selected.append(sec)
388
+ words_after = count_body_words(body)
389
+ while remaining and not fits(after, words_after, budget, word_budget):
390
+ if after > budget:
391
+ pick = max(remaining, key=lambda s: s.lines)
392
+ else:
393
+ pick = max(remaining, key=lambda s: words[s.start])
394
+ remaining.remove(pick)
395
+ selected.append(pick)
267
396
  new_body = rebuild_body(body, selected)
268
397
  after = count_body_lines(new_body)
269
- if after <= budget:
270
- break
271
- return selected, new_body, after
398
+ words_after = count_body_words(new_body)
399
+ return selected, new_body, after, words_after
400
+
401
+
402
+ def _same_content(path: Path, content: str) -> bool:
403
+ try:
404
+ return path.is_file() and path.read_text(encoding="utf-8") == content
405
+ except (OSError, UnicodeDecodeError):
406
+ return False
407
+
408
+
409
+ def assign_names(body: list[str], selected: list[Section], references_dir: Path) -> None:
410
+ """Give each selected section a reference file name that overwrites nothing.
411
+
412
+ In document order, each takes `{kebab}.md`, or the first free
413
+ `{kebab}-2.md`, `{kebab}-3.md`, ... when that file already holds other
414
+ content or an earlier section of this run took the name. A file that
415
+ already holds exactly the section's content is reused.
416
+ """
417
+ taken: set[str] = set()
418
+ for sec in sorted(selected, key=lambda s: s.start):
419
+ content = _section_content(body, sec)
420
+ n = 1
421
+ while True:
422
+ name = sec.kebab if n == 1 else f"{sec.kebab}-{n}"
423
+ path = references_dir / f"{name}.md"
424
+ if name not in taken and (not path.exists() or _same_content(path, content)):
425
+ break
426
+ n += 1
427
+ sec.name = name
428
+ taken.add(name)
272
429
 
273
430
 
274
431
  # --------------------------------------------------------------------------
@@ -325,40 +482,57 @@ def _cmd_shard(args: argparse.Namespace) -> int:
325
482
  return 1
326
483
 
327
484
  budget: int = args.budget
485
+ word_budget: int = args.word_budget
328
486
  dry_run: bool = args.dry_run
329
487
 
330
488
  frontmatter, body = split_frontmatter(text)
331
489
  body_lines_before = count_body_lines(body)
490
+ body_words_before = count_body_words(body)
332
491
  tier1_before = present_tier1(body)
333
492
 
334
493
  if args.verbose:
335
494
  print(
336
- f"skf-shard-body: body_lines_before={body_lines_before} budget={budget}",
495
+ f"skf-shard-body: body_lines_before={body_lines_before} budget={budget} "
496
+ f"body_words_before={body_words_before} word_budget={word_budget}",
337
497
  file=sys.stderr,
338
498
  )
339
499
 
340
500
  # --- Skip path: already within budget -----------------------------------
341
- if body_lines_before <= budget:
501
+ if fits(body_lines_before, body_words_before, budget, word_budget):
342
502
  _emit(
343
503
  {
344
504
  "action": "skip",
345
505
  "body_lines_before": body_lines_before,
346
506
  "body_lines_after": body_lines_before,
507
+ "body_words_before": body_words_before,
508
+ "body_words_after": body_words_before,
347
509
  "sections_extracted": [],
348
510
  "tier1_preserved": True,
349
511
  "tier1_missing": [],
350
512
  "xref_ok": True,
351
513
  "under_budget": True,
514
+ "under_word_budget": True,
352
515
  "dry_run": dry_run,
353
516
  "budget": budget,
517
+ "word_budget": word_budget,
354
518
  },
355
519
  args.output,
356
520
  )
357
521
  return 0
358
522
 
359
523
  # --- Shard path ---------------------------------------------------------
524
+ references_dir = skill_md.parent / "references"
360
525
  sections = find_full_sections(body)
361
- selected, new_body, body_lines_after = select_sections(body, sections, budget)
526
+ if args.skip_manual:
527
+ ranges = manual_ranges(body)
528
+ sections = [sec for sec in sections if not holds_manual(body, sec, ranges)]
529
+ selected, new_body, body_lines_after, body_words_after = select_sections(
530
+ body, sections, budget, word_budget
531
+ )
532
+ # A reference file's name never changes a count (no whitespace in it), so
533
+ # the names are picked once the selection is final.
534
+ assign_names(body, selected, references_dir)
535
+ new_body = rebuild_body(body, selected)
362
536
 
363
537
  tier1_after = present_tier1(new_body)
364
538
  tier1_missing = [
@@ -366,28 +540,32 @@ def _cmd_shard(args: argparse.Namespace) -> int:
366
540
  ]
367
541
  tier1_preserved = not tier1_missing
368
542
  under_budget = body_lines_after <= budget
369
-
370
- references_dir = skill_md.parent / "references"
543
+ under_word_budget = word_budget <= 0 or body_words_after <= word_budget
371
544
 
372
545
  # Compute the plan fully before touching disk. Only write when we have a
373
546
  # real extraction AND Tier-1 survived — never leave a half-mutated tree.
374
547
  should_write = (not dry_run) and bool(selected) and tier1_preserved
375
548
  if should_write:
549
+ written: list[str] = []
376
550
  try:
377
551
  for sec in selected:
378
552
  _atomic_write(
379
- references_dir / f"{sec.kebab}.md", _section_content(body, sec)
553
+ references_dir / f"{sec.name}.md", _section_content(body, sec)
380
554
  )
555
+ written.append(sec.ref_rel)
381
556
  new_text = "\n".join(frontmatter + new_body) + "\n"
382
557
  _atomic_write(skill_md, new_text)
383
558
  except OSError as exc:
384
- print(f"error: {exc}", file=sys.stderr)
559
+ print(
560
+ f"error: {exc}; already written: {', '.join(written) or 'none'}",
561
+ file=sys.stderr,
562
+ )
385
563
  return 2
386
564
 
387
565
  blockquotes_ok = all(sec.blockquote in new_body for sec in selected)
388
566
  if should_write:
389
567
  xref_ok = blockquotes_ok and all(
390
- (references_dir / f"{sec.kebab}.md").is_file() for sec in selected
568
+ (references_dir / f"{sec.name}.md").is_file() for sec in selected
391
569
  )
392
570
  else:
393
571
  # dry-run, nothing-to-extract, or aborted tier1 violation
@@ -398,6 +576,8 @@ def _cmd_shard(args: argparse.Namespace) -> int:
398
576
  "action": "shard",
399
577
  "body_lines_before": body_lines_before,
400
578
  "body_lines_after": body_lines_after,
579
+ "body_words_before": body_words_before,
580
+ "body_words_after": body_words_after,
401
581
  "sections_extracted": [
402
582
  {"heading": sec.heading, "file": sec.ref_rel, "lines": sec.lines}
403
583
  for sec in selected
@@ -406,8 +586,10 @@ def _cmd_shard(args: argparse.Namespace) -> int:
406
586
  "tier1_missing": tier1_missing,
407
587
  "xref_ok": xref_ok,
408
588
  "under_budget": under_budget,
589
+ "under_word_budget": under_word_budget,
409
590
  "dry_run": dry_run,
410
591
  "budget": budget,
592
+ "word_budget": word_budget,
411
593
  },
412
594
  args.output,
413
595
  )
@@ -420,9 +602,10 @@ def _build_parser() -> argparse.ArgumentParser:
420
602
  description=(
421
603
  "Deterministically shard an oversized SKILL.md body: extract the "
422
604
  "largest `## Full …` Tier-2 sections to references/ until the body "
423
- "fits under --budget, replacing each with a cross-reference "
424
- "blockquote, and emit a JSON report the auto-shard / validate "
425
- "steps consume instead of counting and splitting by hand."
605
+ "fits under --budget (and --word-budget, when set), replacing each "
606
+ "with a cross-reference blockquote, and emit a JSON report the "
607
+ "auto-shard / validate / update write steps consume instead of "
608
+ "counting and splitting by hand."
426
609
  ),
427
610
  )
428
611
  parser.add_argument("skill_md", help="path to the staging SKILL.md file")
@@ -432,6 +615,21 @@ def _build_parser() -> argparse.ArgumentParser:
432
615
  default=400,
433
616
  help="body line budget below which no shard occurs (default 400)",
434
617
  )
618
+ parser.add_argument(
619
+ "--word-budget",
620
+ type=int,
621
+ default=0,
622
+ help=(
623
+ "body word budget (words split as JavaScript's /\\s+/ splits them, "
624
+ "as skill-check's body.max_tokens estimate does); 0 turns it off "
625
+ "(default 0)"
626
+ ),
627
+ )
628
+ parser.add_argument(
629
+ "--skip-manual",
630
+ action="store_true",
631
+ help="never move a `## Full` section that intersects a [MANUAL] block",
632
+ )
435
633
  parser.add_argument(
436
634
  "--dry-run",
437
635
  action="store_true",