bmad-module-skill-forge 3.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/docs/_data/pinned.yaml +1 -1
- package/docs/examples.md +1 -1
- package/docs/troubleshooting.md +8 -0
- package/docs/verifying-a-skill.md +7 -3
- package/docs/workflows.md +3 -3
- package/package.json +1 -1
- package/src/knowledge/qmd-registry.md +1 -1
- package/src/knowledge/version-paths.md +1 -1
- package/src/shared/health-check.md +2 -50
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +2 -2
- package/src/shared/scripts/skf-atomic-write.py +212 -59
- package/src/shared/scripts/skf-build-change-manifest.py +593 -47
- package/src/shared/scripts/skf-classify-changed-files.py +78 -0
- package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
- package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
- package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
- package/src/shared/scripts/skf-extract-public-api.py +1309 -157
- package/src/shared/scripts/skf-extraction-inventory.py +580 -46
- package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
- package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
- package/src/shared/scripts/skf-github-probe.py +47 -8
- package/src/shared/scripts/skf-hash-content.py +14 -8
- package/src/shared/scripts/skf-load-provenance.py +3 -1
- package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
- package/src/shared/scripts/skf-preflight.py +15 -1
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
- package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
- package/src/shared/scripts/skf-structural-diff.py +133 -18
- package/src/shared/scripts/skf-update-active-symlink.py +115 -62
- package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
- package/src/shared/scripts/skf-verify-provenance-completeness.py +977 -49
- package/src/shared/scripts/skf-write-skill-brief.py +41 -8
- package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
- package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
- package/src/skf-audit-skill/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +2 -1
- package/src/skf-audit-skill/references/re-index.md +14 -9
- package/src/skf-audit-skill/references/report.md +2 -2
- package/src/skf-audit-skill/references/semantic-diff.md +15 -15
- package/src/skf-audit-skill/references/structural-diff.md +10 -5
- package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
- package/src/skf-brief-skill/SKILL.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +1 -1
- package/src/skf-brief-skill/assets/skill-brief-schema.md +2 -9
- package/src/skf-brief-skill/references/analyze-target.md +6 -6
- package/src/skf-brief-skill/references/confirm-brief.md +1 -1
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
- package/src/skf-brief-skill/references/version-resolution.md +5 -9
- package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
- package/src/skf-create-skill/assets/skill-sections.md +6 -6
- package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
- package/src/skf-create-skill/references/compile.md +31 -60
- package/src/skf-create-skill/references/component-extraction.md +3 -3
- package/src/skf-create-skill/references/enrich.md +2 -2
- package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
- package/src/skf-create-skill/references/extract.md +22 -19
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
- package/src/skf-create-skill/references/extraction-patterns.md +8 -9
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +16 -5
- package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
- package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
- package/src/skf-drop-skill/references/execute.md +5 -1
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +2 -2
- package/src/skf-drop-skill/references/select.md +7 -3
- package/src/skf-test-skill/references/coherence-check.md +1 -1
- package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
- package/src/skf-test-skill/references/coverage-check.md +39 -57
- package/src/skf-test-skill/references/init.md +3 -5
- package/src/skf-test-skill/references/report.md +10 -8
- package/src/skf-test-skill/references/scoring-rules.md +6 -5
- package/src/skf-test-skill/references/source-access-protocol.md +14 -17
- package/src/skf-test-skill/scripts/build-result-context.py +114 -21
- package/src/skf-test-skill/scripts/gap-ledger.py +315 -30
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +534 -96
- package/src/skf-test-skill/scripts/reconcile-coverage.py +276 -2
- package/src/skf-test-skill/scripts/score-signatures.py +36 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
- package/src/skf-update-skill/references/detect-changes.md +32 -13
- package/src/skf-update-skill/references/gap-driven.md +3 -3
- package/src/skf-update-skill/references/init.md +6 -2
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +2 -2
- package/src/skf-update-skill/references/re-extract.md +2 -2
- package/src/skf-update-skill/references/report.md +3 -1
- package/src/skf-update-skill/references/write.md +43 -23
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
|
@@ -5,10 +5,12 @@
|
|
|
5
5
|
# ///
|
|
6
6
|
"""SKF test-skill coverage inputs: the counts, name sets and candidates coverage-check reads.
|
|
7
7
|
|
|
8
|
-
coverage-check.md decides
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
8
|
+
coverage-check.md decides whether a bookkeeping variant is a real export,
|
|
9
|
+
and makes the judgments the denominator clause needs where no file holds
|
|
10
|
+
them (--pattern-reference-intent, and --curated-subset and --barrel where
|
|
11
|
+
the extraction does not say); the clause, everything it counts, folds or
|
|
12
|
+
unions on the way runs here, so no count is made by hand. Three subcommands, each
|
|
13
|
+
reading files and writing one JSON object:
|
|
12
14
|
|
|
13
15
|
census docs-only detection. Counts the citations SKILL.md and each .md
|
|
14
16
|
file directly in references/ write, by form: `[EXT:` against
|
|
@@ -22,7 +24,10 @@ subcommands, each reading files and writing one JSON object:
|
|
|
22
24
|
`confidence_distribution`) under the keys
|
|
23
25
|
check-metadata-coherence.py reads; the provenance named exports
|
|
24
26
|
(`::` impl-block methods left out); the inflation signature
|
|
25
|
-
(`stats.exports_documented` equals `stats.effective_denominator
|
|
27
|
+
(`stats.exports_documented` equals `stats.effective_denominator`,
|
|
28
|
+
or, for a map with a named entry and no `::` entry, its distinct
|
|
29
|
+
entry names do: the stored count counts each name once, the map
|
|
30
|
+
holds a name passed on from two files twice)
|
|
26
31
|
with the declared names verify-declared-numerator.py looks up
|
|
27
32
|
(the `exports[]` names, else the provenance named exports); and,
|
|
28
33
|
for a stack, its composition surface: the distinct provenance
|
|
@@ -34,17 +39,55 @@ subcommands, each reading files and writing one JSON object:
|
|
|
34
39
|
surface the enumerated source surface, from every input given (their
|
|
35
40
|
union):
|
|
36
41
|
--extraction skf-extract-public-api.py --mode full output:
|
|
37
|
-
the names in entry_point_diff.public,
|
|
38
|
-
|
|
39
|
-
|
|
42
|
+
the names in entry_point_diff.public, a
|
|
43
|
+
namespace with a `members` list given as its
|
|
44
|
+
members (the names its module passes on, each
|
|
45
|
+
at its file, once with a public name of the
|
|
46
|
+
same name and file), less the outside_scope
|
|
47
|
+
names (defined in a file the brief scopes
|
|
48
|
+
out); exports[] only gives each name's
|
|
40
49
|
kind, line and signature line, looked up by the
|
|
41
50
|
name its defining file gives it (an aliased
|
|
42
51
|
re-export's `local`) and that file.
|
|
43
|
-
extraction_gaps are listed to read by eye
|
|
44
|
-
|
|
52
|
+
extraction_gaps are listed to read by eye, and
|
|
53
|
+
the runner's warnings join `warnings`. A Python
|
|
54
|
+
top __init__.py inside the folder of another top
|
|
55
|
+
that exports a public name (a sub-package
|
|
56
|
+
behind a folder with no __init__.py) is no
|
|
57
|
+
barrel, unless scope.include or tier_a_include
|
|
58
|
+
lists its file: its names leave `all`, `root`
|
|
59
|
+
and the umbrella ratio and are listed in
|
|
60
|
+
`excluded.nestedEntries`, but keep their records
|
|
61
|
+
(so the scope.include and tier_a_include sets)
|
|
62
|
+
and their outside_scope and extraction_gaps
|
|
63
|
+
rows. When the runner lists an
|
|
64
|
+
include glob that matches no file
|
|
65
|
+
(scope.unmatched_include: a brief older than the
|
|
66
|
+
source), each outside_scope name a root entry
|
|
67
|
+
point exports is restored, with the file and
|
|
68
|
+
line of its outside_scope row, when no include,
|
|
69
|
+
exclude or declined scope-expansion amendment
|
|
70
|
+
(latest action skipped or demoted-include, from
|
|
71
|
+
--brief) matches its file. A tier_a_include glob
|
|
72
|
+
the runner lists as matching no file
|
|
73
|
+
(scope.unmatched_tier_a_include, [] when the key
|
|
74
|
+
is absent) counts no name. An extraction the
|
|
75
|
+
per-file scan replaces (extraction.fallback.needed)
|
|
76
|
+
gives no name and no set: only its scope, its
|
|
77
|
+
stale globs and its `extraction` object, so the
|
|
78
|
+
fallback scan passes it beside its results.
|
|
79
|
+
--quick skf-extract-public-api.py --mode quick output,
|
|
80
|
+
each name at its defined_in (the file that
|
|
81
|
+
defines it), else at its source_file
|
|
45
82
|
--per-file a subagent's per-file result (exports_found and
|
|
46
83
|
types_found), schema-checked as
|
|
47
84
|
score-signatures.py checks it
|
|
85
|
+
--fallback-scan
|
|
86
|
+
the --per-file results are the fallback
|
|
87
|
+
per-file scan's, each at the file that defines
|
|
88
|
+
its names (a Quick-tier or State 4 result is at
|
|
89
|
+
its entry file): as with an extraction, no
|
|
90
|
+
tier_a_include set is no tier A list
|
|
48
91
|
--name a name read by eye whose file is not known
|
|
49
92
|
--provenance the provenance entries' named exports (State 2)
|
|
50
93
|
--metadata the metadata.json exports[] names (States 2, 3)
|
|
@@ -58,34 +101,111 @@ subcommands, each reading files and writing one JSON object:
|
|
|
58
101
|
names to their base name (`X_def`, `X_exact`, `a11y_X`, and
|
|
59
102
|
`<prefix>X` for each --fold-prefix when `X` is itself a name)
|
|
60
103
|
unless --keep names the variant as a real export, and
|
|
61
|
-
`canonical` gives the provenance map's fold summary
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
104
|
+
`canonical` gives the provenance map's fold summary; a map
|
|
105
|
+
whose distinct named exports number
|
|
106
|
+
`stats.effective_denominator` (--metadata) is already canonical
|
|
107
|
+
and is not folded (`canonical.skipped`), and `residual` is the
|
|
108
|
+
canonical count less that stored count. --brief gives the
|
|
109
|
+
skill brief's scope, scope type, language and scope-expansion
|
|
110
|
+
amendments; an extraction's recorded scope wins over its scope.
|
|
111
|
+
|
|
112
|
+
Name sets (surface `sets`, each sorted): `all` (the union above, less a
|
|
113
|
+
nested Python top's names);
|
|
66
114
|
`scope.include` and `tier_a_include` when the scope has those globs and a
|
|
67
115
|
record names a file: the names whose file (an extraction's definition file,
|
|
68
|
-
the quick output's source_file, a per-file result's
|
|
69
|
-
entry's source_file) matches a glob of the list and no
|
|
70
|
-
|
|
71
|
-
`
|
|
72
|
-
|
|
73
|
-
|
|
116
|
+
the quick output's defined_in, else its source_file, a per-file result's
|
|
117
|
+
file, a provenance entry's source_file) matches a glob of the list and no
|
|
118
|
+
scope.exclude glob,
|
|
119
|
+
and for `tier_a_include` a scope.include glob too when the scope has one,
|
|
120
|
+
so a name with no file is in `all` only and a restored name is in no
|
|
121
|
+
tier_a_include set (a tier_a_include glob an extraction lists as matching
|
|
122
|
+
no file is left out of the list, and so is one whose folder or file
|
|
123
|
+
differs from every scope.include glob's before any `*` or `?`, such as
|
|
124
|
+
`gen/**` beside `src/**`, and so is one a scope.exclude glob covers
|
|
125
|
+
segment by segment, such as `src/gen/**` or `src/gen/extra.ts` beside an
|
|
126
|
+
exclude of `src/gen/**`; a list that counts no name, each glob left out
|
|
127
|
+
or matching only excluded files, files scope.include leaves out or files
|
|
128
|
+
no record names, gives no tier_a_include set); and, from an
|
|
129
|
+
extraction only, `subpaths` (the names a non-root `exports` map subpath
|
|
130
|
+
without `*` reaches) and `root` (the names the root entry points reach).
|
|
131
|
+
reconcile-coverage.py and score-signatures.py --surface-set pick one.
|
|
74
132
|
|
|
75
133
|
Candidates and guards (with an extraction, --brief or --metadata): each
|
|
76
134
|
set's count, the metadata `stats.effective_denominator` (--metadata), and
|
|
77
135
|
the extractor's own effective_denominator and its basis. Both guards run
|
|
78
136
|
only on a surface read from the source. The deflation guard compares the
|
|
79
137
|
re-derived scope.include set (the `all` set without one) with
|
|
80
|
-
`stats.effective_denominator` and fires above DEFLATION_PCT when
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
138
|
+
`stats.effective_denominator` and fires above DEFLATION_PCT when there is
|
|
139
|
+
no tier A list (with an extraction or --fallback-scan, no tier_a_include
|
|
140
|
+
set; otherwise no tier_a_include glob left in the list, since a name read
|
|
141
|
+
by eye at Quick tier or State 4, and a quick name with no defined_in,
|
|
142
|
+
carries its entry file, not the file that defines it); a stored 0 is
|
|
143
|
+
deflation, not a
|
|
144
|
+
denominator (`pct` null), and fires under that condition when any name is
|
|
145
|
+
re-derived. The inflation guard compares the scope.include set with the
|
|
146
|
+
provenance entry count (--provenance) and fires above INFLATION_PCT under
|
|
147
|
+
the same condition; `guards.tierAInclude` lists the scope's
|
|
148
|
+
tier_a_include globs ([] when it has none) for gap-ledger.py's wording;
|
|
149
|
+
the umbrella ratio (an
|
|
84
150
|
extraction only) is the share of the root entry points' names that are
|
|
85
|
-
re-exports
|
|
151
|
+
re-exports (a namespace with `members` counting as its members there
|
|
152
|
+
too, each a re-export), and above UMBRELLA_RATIO the barrel is an
|
|
153
|
+
umbrella; the
|
|
154
|
+
stale-scope guard (an extraction with scope.include or tier_a_include only)
|
|
155
|
+
fires when the runner lists an include or tier_a_include glob that matches
|
|
156
|
+
no file, and lists those globs (`unmatchedInclude`,
|
|
157
|
+
`unmatchedTierAInclude`) and the names restored for the include globs, each
|
|
158
|
+
with its file and line (`restored`). Globs follow
|
|
86
159
|
skf-extract-public-api.py: `**` spans any number of path segments, none
|
|
87
160
|
included, and `*` and `?` stay inside one.
|
|
88
161
|
|
|
162
|
+
Denominator (surface `denominator`): the Source API Surface Definition's
|
|
163
|
+
clause (source-access-protocol.md), the branch reconcile-coverage.py takes
|
|
164
|
+
and the set it and score-signatures.py --surface-set read: {clause, branch,
|
|
165
|
+
surfaceSet, ladderStep, absentSet, rebuild, barrel}. The inputs: the scope
|
|
166
|
+
type (the brief's, else the extraction's, else metadata.json's
|
|
167
|
+
`scope_type`), metadata.json's `skill_type` and `stats.effective_denominator`,
|
|
168
|
+
the extraction's `arms`, the deflation guard, and:
|
|
169
|
+
monorepo --monorepo (the workspace layout skf-detect-workspaces.py
|
|
170
|
+
found), or the extraction's `arms.monorepo`.
|
|
171
|
+
curated the extraction's `counts.curated_subset` (the rule Create Skill
|
|
172
|
+
stores the count by), else --curated-subset (the workflow's
|
|
173
|
+
judgment that a monorepo's scope.include lists a curated subset).
|
|
174
|
+
barrel (`barrel`: "barrel", "empty-barrel", "no-entry-point" or null)
|
|
175
|
+
a complete extraction's `entry_points.status`; for one the
|
|
176
|
+
per-file scan replaces, "no-entry-point" when it read a file in
|
|
177
|
+
scope and found no entry point (`entry_points.files` empty), as
|
|
178
|
+
a complete run says; a quick output's (an export is a barrel,
|
|
179
|
+
none and no warning an empty one); else --barrel (`barrel`,
|
|
180
|
+
`empty` or `none`, what the workflow read); else null.
|
|
181
|
+
--pattern-reference-intent the workflow's judgment that scope.notes
|
|
182
|
+
flags pattern-reference intent.
|
|
183
|
+
The first clause that applies:
|
|
184
|
+
stack skill_type stack: branch `stack`, no set.
|
|
185
|
+
pattern-reference not a monorepo, no non-empty barrel, a scope type other
|
|
186
|
+
than specific-modules, component-library, docs-only and
|
|
187
|
+
reference-app, not public-api with a multi-subpath
|
|
188
|
+
`exports` map, and either the intent flag or no entry
|
|
189
|
+
point at all with no such map: the tier_a_include set,
|
|
190
|
+
else `all` with `rebuild` `provenance-fold` on a surface
|
|
191
|
+
read from the source (build it again from the provenance
|
|
192
|
+
map alone, folded), `all` on one that is not.
|
|
193
|
+
empty-barrel an empty barrel: the scope.include set.
|
|
194
|
+
stratified-scope --monorepo and --curated-subset, not full-library.
|
|
195
|
+
specific-modules scope type specific-modules, not --monorepo.
|
|
196
|
+
multi-entry arms.multi_subpath_exports, not --monorepo, a
|
|
197
|
+
full-library or public-api scope.
|
|
198
|
+
standard-barrel any other: the `all` set.
|
|
199
|
+
The last three walk a ladder, `ladderStep` naming the rung taken: rung 1,
|
|
200
|
+
`stats.effective_denominator` when metadata.json has one above 0 (a stored
|
|
201
|
+
0 is no denominator), is the `scalar` branch, whose `surfaceSet` (the set score-signatures.py scores) is
|
|
202
|
+
tier_a_include, else scope.include, else all, unless the deflation guard
|
|
203
|
+
fires: then the barrel branch over scope.include. The next rungs are sets,
|
|
204
|
+
each taken when the surface has it, the last taken in any case:
|
|
205
|
+
tier_a_include then scope.include (stratified-scope), scope.include
|
|
206
|
+
(specific-modules), tier_a_include then subpaths (multi-entry). A set the
|
|
207
|
+
surface lacks is counted as `all`, and `absentSet` names it.
|
|
208
|
+
|
|
89
209
|
Fallback verdict: `extraction.fallback.needed` is true when the extractor
|
|
90
210
|
found no ast-grep (`no-ast-grep`), read no file in scope, read files only
|
|
91
211
|
in languages it has no recipe for (`files_without_recipes` and no export),
|
|
@@ -100,9 +220,10 @@ Usage:
|
|
|
100
220
|
uv run load-coverage-inputs.py metadata --metadata <metadata.json> \\
|
|
101
221
|
[--provenance <provenance-map.json>] [--output <file>]
|
|
102
222
|
uv run load-coverage-inputs.py surface [--extraction <file>] [--quick <file>]... \\
|
|
103
|
-
[--per-file <file>]... [--name <name>]... [--provenance <file>]
|
|
104
|
-
[--brief <skill-brief.yaml>] [--fold] [--fold-prefix <prefix>]...
|
|
105
|
-
[--
|
|
223
|
+
[--per-file <file>]... [--fallback-scan] [--name <name>]... [--provenance <file>] \\
|
|
224
|
+
[--metadata <file>] [--brief <skill-brief.yaml>] [--fold] [--fold-prefix <prefix>]... \\
|
|
225
|
+
[--keep <name>]... [--monorepo] [--pattern-reference-intent] [--curated-subset] \\
|
|
226
|
+
[--barrel barrel|empty|none] [--output <file>]
|
|
106
227
|
|
|
107
228
|
Exit codes:
|
|
108
229
|
0 the result was printed (and written to --output)
|
|
@@ -119,6 +240,7 @@ import argparse
|
|
|
119
240
|
import importlib.util
|
|
120
241
|
import json
|
|
121
242
|
import math
|
|
243
|
+
import posixpath
|
|
122
244
|
import re
|
|
123
245
|
import sys
|
|
124
246
|
from pathlib import Path
|
|
@@ -140,6 +262,19 @@ LOCAL_FORMS = ("AST", "SRC", "from skill")
|
|
|
140
262
|
FOLD_SUFFIXES = ("_def", "_exact")
|
|
141
263
|
FOLD_PREFIX = "a11y_"
|
|
142
264
|
ROOT_SUBPATHS = (None, ".")
|
|
265
|
+
# The latest action of a scope-expansion amendment that keeps its path out of scope.
|
|
266
|
+
DECLINED_ACTIONS = ("skipped", "demoted-include")
|
|
267
|
+
# The rungs each laddered clause tries in order, `scalar` being stats.effective_denominator.
|
|
268
|
+
CLAUSE_LADDERS = {
|
|
269
|
+
"stratified-scope": ("scalar", "tier_a_include", "scope.include"),
|
|
270
|
+
"specific-modules": ("scalar", "scope.include"),
|
|
271
|
+
"multi-entry": ("scalar", "tier_a_include", "subpaths"),
|
|
272
|
+
}
|
|
273
|
+
# --barrel: what the workflow read of the entry points, where surface cannot tell -> the extractor's status.
|
|
274
|
+
BARREL_FLAG = {"barrel": "barrel", "empty": "empty-barrel", "none": "no-entry-point"}
|
|
275
|
+
# The scope types the pattern-reference clause never applies to (a reference app's brief carries its own
|
|
276
|
+
# pattern-surface denominator).
|
|
277
|
+
NOT_PATTERN_REFERENCE = frozenset({"specific-modules", "component-library", "docs-only", "reference-app"})
|
|
143
278
|
# The languages skf-extract-public-api.py's recipes read (its LANGUAGE_FAMILIES).
|
|
144
279
|
RECIPE_LANGUAGES = frozenset({"python", "rust", "go", "golang", "typescript", "ts", "tsx", "javascript", "js",
|
|
145
280
|
"jsx", "vue"})
|
|
@@ -158,17 +293,21 @@ class SchemaError(Exception):
|
|
|
158
293
|
|
|
159
294
|
|
|
160
295
|
_SIBLINGS: dict[str, object] = {}
|
|
296
|
+
SHARED_SCRIPTS = Path(__file__).resolve().parent.parent.parent / "shared" / "scripts"
|
|
161
297
|
|
|
162
298
|
|
|
163
|
-
def _sibling(filename: str):
|
|
299
|
+
def _sibling(filename: str, folder: Path | None = None):
|
|
164
300
|
"""A script beside this one, loaded once: validate-inventory.py (the doc
|
|
165
|
-
text) and score-signatures.py (the per-file result schema)
|
|
301
|
+
text) and score-signatures.py (the per-file result schema); or, with
|
|
302
|
+
`folder`, one in that folder: SHARED_SCRIPTS'
|
|
303
|
+
skf-resolve-authoritative-files.py (a skill brief's YAML)."""
|
|
166
304
|
module = _SIBLINGS.get(filename)
|
|
167
305
|
if module is None:
|
|
168
|
-
path = Path(__file__).resolve().parent / filename
|
|
306
|
+
path = (folder or Path(__file__).resolve().parent) / filename
|
|
169
307
|
spec = importlib.util.spec_from_file_location("skf_" + filename[:-3].replace("-", "_"), path)
|
|
170
308
|
if spec is None or spec.loader is None or not path.is_file():
|
|
171
|
-
|
|
309
|
+
where = f"in {folder}" if folder else f"beside {Path(__file__).name}"
|
|
310
|
+
raise InputError(f"{filename} not found {where}")
|
|
172
311
|
module = importlib.util.module_from_spec(spec)
|
|
173
312
|
spec.loader.exec_module(module)
|
|
174
313
|
_SIBLINGS[filename] = module
|
|
@@ -306,7 +445,13 @@ def metadata_inputs(meta: dict, provenance: dict | None) -> dict:
|
|
|
306
445
|
"confidenceDistribution": dist or None,
|
|
307
446
|
"namedExports": {"count": len(named), "names": named} if entry_names is not None else None,
|
|
308
447
|
"effectiveDenominator": effective,
|
|
309
|
-
|
|
448
|
+
# The stored count counts each name once, while the map (so stats.exports_documented) holds an entry for
|
|
449
|
+
# each name and file: a name the entry points pass on from two files is two entries, one name. Only a map
|
|
450
|
+
# with no `::` entry takes that arm: the stored count holds no method, so a map with methods could match it
|
|
451
|
+
# only by chance, and its methods would stand in for an undocumented name.
|
|
452
|
+
"inflationSignature": effective is not None and (
|
|
453
|
+
documented == effective or (bool(entry_names) and not any(METHOD_SEPARATOR in n for n in entry_names)
|
|
454
|
+
and len(set(entry_names)) == effective)),
|
|
310
455
|
"declaredNames": sorted(set(names)) if names else named,
|
|
311
456
|
"stack": stack,
|
|
312
457
|
}
|
|
@@ -359,13 +504,16 @@ def _scope(include, exclude, tier_a_include) -> dict:
|
|
|
359
504
|
|
|
360
505
|
|
|
361
506
|
def read_brief(path: str) -> dict:
|
|
362
|
-
"""A skill brief's scope globs and language, the fields skf-extract-public-api.py reads
|
|
507
|
+
"""A skill brief's scope globs, scope type and language, the fields skf-extract-public-api.py reads, and the
|
|
508
|
+
latest action of each scope-expansion amendment by its path (`amendments`), a legacy headless skip
|
|
509
|
+
read as a deferral as skf-resolve-authoritative-files.py reads it."""
|
|
363
510
|
try:
|
|
364
511
|
import yaml # --brief only; `uv run` installs it from the header
|
|
365
512
|
except ImportError as exc:
|
|
366
513
|
raise InputError("--brief is read with PyYAML: run the script with `uv run`") from exc
|
|
514
|
+
resolver = _sibling("skf-resolve-authoritative-files.py", SHARED_SCRIPTS)
|
|
367
515
|
try:
|
|
368
|
-
data =
|
|
516
|
+
data = resolver.parse_brief_yaml(Path(path).read_bytes().decode("utf-8-sig"))
|
|
369
517
|
except (OSError, UnicodeDecodeError) as exc:
|
|
370
518
|
raise InputError(f"cannot read --brief {path}: {exc}") from exc
|
|
371
519
|
except yaml.YAMLError as exc:
|
|
@@ -373,8 +521,17 @@ def read_brief(path: str) -> dict:
|
|
|
373
521
|
if not isinstance(data, dict):
|
|
374
522
|
raise InputError(f"--brief {path} is not a YAML mapping")
|
|
375
523
|
scope = data.get("scope") if isinstance(data.get("scope"), dict) else {}
|
|
524
|
+
amendments: dict[str, str] = {}
|
|
525
|
+
for amend in scope.get("amendments") if isinstance(scope.get("amendments"), list) else []:
|
|
526
|
+
if isinstance(amend, dict) and amend.get("category") == "scope-expansion":
|
|
527
|
+
amended = _norm(amend["path"]) if isinstance(amend.get("path"), str) else ""
|
|
528
|
+
action = resolver._amendment_action(amend)
|
|
529
|
+
if amended and action is not None:
|
|
530
|
+
amendments[amended] = action
|
|
376
531
|
return {"scope": _scope(scope.get("include"), scope.get("exclude"), scope.get("tier_a_include")),
|
|
377
|
-
"
|
|
532
|
+
"type": scope.get("type") if isinstance(scope.get("type"), str) else None,
|
|
533
|
+
"language": data.get("language") if isinstance(data.get("language"), str) else None,
|
|
534
|
+
"amendments": amendments}
|
|
378
535
|
|
|
379
536
|
|
|
380
537
|
def matching(records: list[dict], globs: list[str], excludes: list[str]) -> list[str]:
|
|
@@ -384,37 +541,196 @@ def matching(records: list[dict], globs: list[str], excludes: list[str]) -> list
|
|
|
384
541
|
and not any(glob_match(_norm(r["file"]), g) for g in excludes)})
|
|
385
542
|
|
|
386
543
|
|
|
387
|
-
def
|
|
388
|
-
"""
|
|
544
|
+
def _apart(a: str, b: str) -> bool:
|
|
545
|
+
"""True when, before either has a `*` or `?`, the two globs spell a different segment at the same
|
|
546
|
+
place, so no file can match both. Any other pair counts as overlapping, even one no file can match
|
|
547
|
+
both of (`src` beside `src/index.ts`), so this never parts two that can share a file."""
|
|
548
|
+
for x, y in zip(a.split("/"), b.split("/")):
|
|
549
|
+
if any(c in x or c in y for c in "*?"):
|
|
550
|
+
return False
|
|
551
|
+
if x != y:
|
|
552
|
+
return True
|
|
553
|
+
return False
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _inside(a: str, b: str) -> bool:
|
|
557
|
+
"""True when glob `b` matches every path glob `a` can match: segment by segment each of `b`'s is
|
|
558
|
+
`**`, the same as `a`'s, or `*` beside one of `a`'s that is not `**`, and `a` has as many segments
|
|
559
|
+
as `b`, or more when `b` ends in `**`. Any other pair counts as not inside, even one where `b`
|
|
560
|
+
does cover `a` (`gen/**` in `**/gen/**`), so this never drops a glob that can count a name."""
|
|
561
|
+
xs, ys = a.split("/"), b.split("/")
|
|
562
|
+
if len(xs) < len(ys) or (len(xs) > len(ys) and ys[-1] != "**"):
|
|
563
|
+
return False
|
|
564
|
+
return all(y == "**" or x == y or (y == "*" and x != "**") for x, y in zip(xs, ys))
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def _nested_tops(entry_files: list[dict], named: set[str], exporting: set[str]) -> dict[str, str]:
|
|
568
|
+
"""{top: the outermost top whose folder holds it} for each Python top
|
|
569
|
+
__init__.py inside the folder of another top that `exporting` lists (one
|
|
570
|
+
that exports at least one public name), a sub-package behind a folder
|
|
571
|
+
with no __init__.py, unless `named` lists its file. A top that exports
|
|
572
|
+
nothing (a stray __init__.py at the source root, say) nests no other."""
|
|
573
|
+
tops = sorted({e["file"] for e in entry_files
|
|
574
|
+
if e.get("language") == "python" and isinstance(e.get("file"), str) and e["file"]})
|
|
575
|
+
nested = {}
|
|
576
|
+
for top in tops:
|
|
577
|
+
folder = posixpath.dirname(top)
|
|
578
|
+
outer = [o for o in tops if o != top and o in exporting
|
|
579
|
+
and (not posixpath.dirname(o) or folder.startswith(posixpath.dirname(o) + "/"))]
|
|
580
|
+
if outer and _norm(top) not in named:
|
|
581
|
+
nested[top] = min(outer, key=lambda o: (o.count("/"), o))
|
|
582
|
+
return nested
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def _members_in_place(public: list[dict]) -> list[dict]:
|
|
586
|
+
"""The public names, a namespace item with a `members` list (the names
|
|
587
|
+
its module passes on, `pkg.sub.name`) given as its members, each with
|
|
588
|
+
the namespace's entry and language and `via: namespace`. A member whose
|
|
589
|
+
name and file another public name, or an earlier member, has is left
|
|
590
|
+
out; a namespace whose members the runner could not read (null), or one
|
|
591
|
+
from a runner that lists none, stays itself."""
|
|
592
|
+
expanded = [p for p in public if not (p.get("via") == "namespace" and isinstance(p.get("members"), list))]
|
|
593
|
+
seen = {(p["name"], p.get("file")) for p in expanded}
|
|
594
|
+
for item in public:
|
|
595
|
+
if item.get("via") != "namespace" or not isinstance(item.get("members"), list):
|
|
596
|
+
continue
|
|
597
|
+
for member in item["members"]:
|
|
598
|
+
if not isinstance(member, dict) or not isinstance(member.get("name"), str) or not member["name"]:
|
|
599
|
+
continue
|
|
600
|
+
key = (member["name"], member.get("file"))
|
|
601
|
+
if key in seen:
|
|
602
|
+
continue
|
|
603
|
+
seen.add(key)
|
|
604
|
+
expanded.append({"name": member["name"], "language": item.get("language"), "entry": item.get("entry"),
|
|
605
|
+
"via": "namespace", "local": member.get("local"), "file": member.get("file"),
|
|
606
|
+
"line": member.get("line")})
|
|
607
|
+
return expanded
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _extraction_summary(data: dict, language: str | None) -> dict:
|
|
611
|
+
"""The surface's `extraction` object: the runner's status and counts,
|
|
612
|
+
the files to read by eye and the fallback verdict."""
|
|
613
|
+
counts = data.get("counts") if isinstance(data.get("counts"), dict) else {}
|
|
614
|
+
arms = data.get("arms") if isinstance(data.get("arms"), dict) else {}
|
|
615
|
+
entry_points = data.get("entry_points") if isinstance(data.get("entry_points"), dict) else {}
|
|
616
|
+
without = data.get("files_without_recipes") if isinstance(data.get("files_without_recipes"), dict) else {}
|
|
617
|
+
status = data.get("status")
|
|
618
|
+
reason = None
|
|
619
|
+
if status == "no-ast-grep":
|
|
620
|
+
reason = "no ast-grep the runner can run"
|
|
621
|
+
elif not data.get("files_in_scope"):
|
|
622
|
+
reason = "no file in scope"
|
|
623
|
+
elif without and not data.get("exports"):
|
|
624
|
+
reason = f"no recipe reads the files in scope ({', '.join(sorted(without))})"
|
|
625
|
+
elif without and language and language.strip().lower() not in RECIPE_LANGUAGES:
|
|
626
|
+
reason = (f"no recipe reads the skill's language, {language} "
|
|
627
|
+
f"({', '.join(sorted(without))} in scope)")
|
|
628
|
+
errors = [e for e in data.get("errors") or [] if isinstance(e, dict)]
|
|
629
|
+
read_by_eye = sorted({str(i["file"]) for i in data.get("file_issues") or []
|
|
630
|
+
if isinstance(i, dict) and i.get("file") and i.get("issue") != "no-recipes"}
|
|
631
|
+
| {str(e["first_file"]) for e in errors if e.get("first_file")}
|
|
632
|
+
| {str(f) for e in errors if isinstance(e.get("unread"), list) for f in e["unread"] if f})
|
|
633
|
+
return {
|
|
634
|
+
"status": status,
|
|
635
|
+
"filesInScope": data.get("files_in_scope"),
|
|
636
|
+
"truncated": data.get("truncated"),
|
|
637
|
+
"effectiveDenominator": _count(counts.get("effective_denominator")),
|
|
638
|
+
"effectiveDenominatorBasis": counts.get("effective_denominator_basis"),
|
|
639
|
+
"exportsPublicApi": _count(counts.get("exports_public_api")),
|
|
640
|
+
"arms": arms,
|
|
641
|
+
"curatedSubset": counts.get("curated_subset") if isinstance(counts.get("curated_subset"), bool) else None,
|
|
642
|
+
"entryPointStatus": entry_points.get("status"),
|
|
643
|
+
# A run that read a file in scope and found no entry point says so whether or not ast-grep ran.
|
|
644
|
+
"entryPointsFound": (bool(entry_points["files"]) if isinstance(entry_points.get("files"), list)
|
|
645
|
+
and data.get("files_in_scope") else None),
|
|
646
|
+
"readByEye": read_by_eye,
|
|
647
|
+
"fallback": {"needed": reason is not None, "reason": reason},
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
NO_UMBRELLA = {"entries": [], "names": 0, "reexported": 0, "ratio": None, "umbrella": False}
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def from_extraction(data: dict, language: str | None = None, brief: dict | None = None) -> dict:
|
|
655
|
+
"""The public surface of an extraction, less what lies outside scope,
|
|
656
|
+
plus the root exports a stale scope.include glob left out. A nested
|
|
657
|
+
Python top's records come apart (`nestedRecords`): they count in the
|
|
658
|
+
scope sets, not in `all`. An extraction the per-file scan replaces
|
|
659
|
+
(`extraction.fallback.needed`) gives only its scope, its stale globs and
|
|
660
|
+
its `extraction` object: no name and no set."""
|
|
389
661
|
if data.get("mode") != "full":
|
|
390
662
|
raise InputError("--extraction is not skf-extract-public-api.py --mode full output")
|
|
663
|
+
|
|
664
|
+
# The scope: the one the extractor recorded, else the brief's.
|
|
665
|
+
recorded = data.get("scope") if isinstance(data.get("scope"), dict) else {}
|
|
666
|
+
scope = _scope(recorded.get("include"), recorded.get("exclude"), recorded.get("tier_a_include"))
|
|
667
|
+
if not any(scope.values()) and brief is not None:
|
|
668
|
+
scope = brief["scope"]
|
|
669
|
+
unmatched = _globs(recorded.get("unmatched_include"))
|
|
670
|
+
# A tier A glob that matches no file: the runner's list, never re-matched here.
|
|
671
|
+
unmatched_tier_a = _globs(recorded.get("unmatched_tier_a_include"))
|
|
672
|
+
extraction = _extraction_summary(data, language)
|
|
673
|
+
if extraction["fallback"]["needed"]:
|
|
674
|
+
return {"records": [], "nestedRecords": [], "sets": {}, "outside": [], "nested": [],
|
|
675
|
+
"staleScope": {"unmatchedInclude": unmatched, "unmatchedTierAInclude": unmatched_tier_a,
|
|
676
|
+
"restored": []},
|
|
677
|
+
"gaps": [], "scope": scope, "scopeType": recorded.get("type"), "umbrella": dict(NO_UMBRELLA),
|
|
678
|
+
"extraction": extraction}
|
|
679
|
+
|
|
391
680
|
diff = data.get("entry_point_diff") if isinstance(data.get("entry_point_diff"), dict) else {}
|
|
392
|
-
public = [p for p in diff.get("public") or []
|
|
681
|
+
public = _members_in_place([p for p in diff.get("public") or []
|
|
682
|
+
if isinstance(p, dict) and isinstance(p.get("name"), str)])
|
|
393
683
|
outside = [o for o in diff.get("outside_scope") or [] if isinstance(o, dict)]
|
|
394
684
|
gaps = [g for g in diff.get("extraction_gaps") or [] if isinstance(g, dict)]
|
|
685
|
+
|
|
686
|
+
entry_points = data.get("entry_points") if isinstance(data.get("entry_points"), dict) else {}
|
|
687
|
+
entry_files = [e for e in entry_points.get("files") or [] if isinstance(e, dict)]
|
|
688
|
+
# A Python top nested in another that exports a name is no barrel: its
|
|
689
|
+
# names leave `all`, `root` and the umbrella ratio, and keep their
|
|
690
|
+
# records (so the scope sets a specific-modules or stratified brief
|
|
691
|
+
# counts) and their outside_scope and extraction_gaps rows.
|
|
692
|
+
nested = _nested_tops(entry_files, set(scope["include"]) | set(scope["tier_a_include"]),
|
|
693
|
+
{p.get("entry") for p in public})
|
|
694
|
+
nested_names = {top: sorted({p["name"] for p in public if p.get("entry") == top}) for top in nested}
|
|
695
|
+
|
|
696
|
+
subpath_files = {e.get("file") for e in entry_files
|
|
697
|
+
if e.get("subpath") not in ROOT_SUBPATHS and "*" not in str(e.get("subpath"))}
|
|
698
|
+
root_files = {e.get("file") for e in entry_files if e.get("subpath") in ROOT_SUBPATHS} - set(nested)
|
|
699
|
+
|
|
700
|
+
# A scope.include glob that matches no file: the brief predates the
|
|
701
|
+
# source, so a root export defined in a file no glob covers comes back,
|
|
702
|
+
# unless the brief excludes or declined that file.
|
|
703
|
+
declined = [path for path, action in ((brief or {}).get("amendments") or {}).items()
|
|
704
|
+
if action in DECLINED_ACTIONS]
|
|
705
|
+
restored = [o for o in outside if unmatched and o.get("entry") in root_files
|
|
706
|
+
and isinstance(o.get("name"), str) and isinstance(o.get("file"), str) and o["file"]
|
|
707
|
+
and not any(glob_match(_norm(o["file"]), g)
|
|
708
|
+
for g in scope["include"] + scope["exclude"] + declined)]
|
|
709
|
+
restored_at = {(o["name"], o["file"]): o.get("line") for o in restored}
|
|
710
|
+
outside = [o for o in outside if (o.get("name"), o.get("file")) not in restored_at]
|
|
711
|
+
|
|
395
712
|
outside_keys = {(o.get("name"), o.get("file")) for o in outside}
|
|
396
713
|
index = {(e.get("export_name"), e.get("source_file")): e
|
|
397
714
|
for e in data.get("exports") or [] if isinstance(e, dict)}
|
|
398
|
-
records = []
|
|
715
|
+
records, nested_records = [], []
|
|
399
716
|
for info in public:
|
|
400
717
|
name, file = info["name"], info.get("file")
|
|
401
718
|
if (name, file) in outside_keys:
|
|
402
719
|
continue
|
|
720
|
+
if (name, file) in restored_at:
|
|
721
|
+
# No recipe reads a file out of scope, so its kind is unknown: the
|
|
722
|
+
# name carries the file and line of its outside_scope row.
|
|
723
|
+
records.append(_record(name, None, file, restored_at[(name, file)], None, "restored"))
|
|
724
|
+
continue
|
|
403
725
|
# An aliased re-export (`export { Options as Config }`) is recorded
|
|
404
726
|
# under the name its defining file gives it, `local`.
|
|
405
727
|
found = index.get((info.get("local") or name, file)) or index.get((name, file))
|
|
406
728
|
kind = found.get("export_type") if found else None
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
scope = data.get("scope") if isinstance(data.get("scope"), dict) else {}
|
|
729
|
+
(nested_records if info.get("entry") in nested else records).append(
|
|
730
|
+
_record(name, None if kind == "re-export" else kind, file,
|
|
731
|
+
found.get("source_line") if found else info.get("line"),
|
|
732
|
+
found.get("signature_line") if found else None, "extraction"))
|
|
412
733
|
|
|
413
|
-
entry_points = data.get("entry_points") if isinstance(data.get("entry_points"), dict) else {}
|
|
414
|
-
entry_files = [e for e in entry_points.get("files") or [] if isinstance(e, dict)]
|
|
415
|
-
subpath_files = {e.get("file") for e in entry_files
|
|
416
|
-
if e.get("subpath") not in ROOT_SUBPATHS and "*" not in str(e.get("subpath"))}
|
|
417
|
-
root_files = {e.get("file") for e in entry_files if e.get("subpath") in ROOT_SUBPATHS}
|
|
418
734
|
root_names = [p for p in public if p.get("entry") in root_files]
|
|
419
735
|
reexported = [p for p in root_names if p.get("via") not in (None, "declaration")]
|
|
420
736
|
|
|
@@ -424,30 +740,18 @@ def from_extraction(data: dict, language: str | None = None) -> dict:
|
|
|
424
740
|
if root_files:
|
|
425
741
|
sets["root"] = sorted({p["name"] for p in root_names})
|
|
426
742
|
|
|
427
|
-
counts = data.get("counts") if isinstance(data.get("counts"), dict) else {}
|
|
428
|
-
arms = data.get("arms") if isinstance(data.get("arms"), dict) else {}
|
|
429
|
-
without = data.get("files_without_recipes") if isinstance(data.get("files_without_recipes"), dict) else {}
|
|
430
|
-
status = data.get("status")
|
|
431
|
-
reason = None
|
|
432
|
-
if status == "no-ast-grep":
|
|
433
|
-
reason = "no ast-grep the runner can run"
|
|
434
|
-
elif not data.get("files_in_scope"):
|
|
435
|
-
reason = "no file in scope"
|
|
436
|
-
elif without and not data.get("exports"):
|
|
437
|
-
reason = f"no recipe reads the files in scope ({', '.join(sorted(without))})"
|
|
438
|
-
elif without and language and language.strip().lower() not in RECIPE_LANGUAGES:
|
|
439
|
-
reason = (f"no recipe reads the skill's language, {language} "
|
|
440
|
-
f"({', '.join(sorted(without))} in scope)")
|
|
441
|
-
read_by_eye = sorted({str(i["file"]) for i in data.get("file_issues") or []
|
|
442
|
-
if isinstance(i, dict) and i.get("file") and i.get("issue") != "no-recipes"}
|
|
443
|
-
| {str(e["first_file"]) for e in data.get("errors") or []
|
|
444
|
-
if isinstance(e, dict) and e.get("first_file")})
|
|
445
743
|
return {
|
|
446
744
|
"records": records,
|
|
745
|
+
"nestedRecords": nested_records,
|
|
447
746
|
"sets": sets,
|
|
448
747
|
"outside": [{"name": o.get("name"), "file": o.get("file")} for o in outside],
|
|
748
|
+
"nested": [{"entry": top, "within": nested[top], "names": nested_names[top]} for top in sorted(nested)],
|
|
749
|
+
"staleScope": {"unmatchedInclude": unmatched, "unmatchedTierAInclude": unmatched_tier_a,
|
|
750
|
+
"restored": [{"name": o["name"], "file": o["file"], "line": o.get("line")}
|
|
751
|
+
for o in sorted(restored, key=lambda o: (o["name"], o["file"]))]},
|
|
449
752
|
"gaps": [{"name": g.get("name"), "file": g.get("file"), "line": g.get("line")} for g in gaps],
|
|
450
|
-
"scope":
|
|
753
|
+
"scope": scope,
|
|
754
|
+
"scopeType": recorded.get("type") if isinstance(recorded.get("type"), str) else None,
|
|
451
755
|
"umbrella": {
|
|
452
756
|
"entries": sorted(f for f in root_files if f),
|
|
453
757
|
"names": len(root_names),
|
|
@@ -455,17 +759,7 @@ def from_extraction(data: dict, language: str | None = None) -> dict:
|
|
|
455
759
|
"ratio": round2(len(reexported) / len(root_names)) if root_names else None,
|
|
456
760
|
"umbrella": bool(root_names) and len(reexported) / len(root_names) > UMBRELLA_RATIO,
|
|
457
761
|
},
|
|
458
|
-
"extraction":
|
|
459
|
-
"status": status,
|
|
460
|
-
"filesInScope": data.get("files_in_scope"),
|
|
461
|
-
"truncated": data.get("truncated"),
|
|
462
|
-
"effectiveDenominator": _count(counts.get("effective_denominator")),
|
|
463
|
-
"effectiveDenominatorBasis": counts.get("effective_denominator_basis"),
|
|
464
|
-
"exportsPublicApi": _count(counts.get("exports_public_api")),
|
|
465
|
-
"arms": arms,
|
|
466
|
-
"readByEye": read_by_eye,
|
|
467
|
-
"fallback": {"needed": reason is not None, "reason": reason},
|
|
468
|
-
},
|
|
762
|
+
"extraction": extraction,
|
|
469
763
|
}
|
|
470
764
|
|
|
471
765
|
|
|
@@ -473,7 +767,7 @@ def from_quick(data: dict) -> list[dict]:
|
|
|
473
767
|
exports = data.get("exports")
|
|
474
768
|
if not isinstance(exports, list):
|
|
475
769
|
raise InputError("--quick is not skf-extract-public-api.py --mode quick output (no exports list)")
|
|
476
|
-
return [_record(e["name"], e.get("type"), e.get("source_file"), None, None, "quick")
|
|
770
|
+
return [_record(e["name"], e.get("type"), e.get("defined_in") or e.get("source_file"), None, None, "quick")
|
|
477
771
|
for e in exports if isinstance(e, dict) and isinstance(e.get("name"), str) and e["name"]]
|
|
478
772
|
|
|
479
773
|
|
|
@@ -525,15 +819,68 @@ def fold_names(names: list[str], prefixes: list[str], keep: set[str],
|
|
|
525
819
|
return variants, tally
|
|
526
820
|
|
|
527
821
|
|
|
528
|
-
|
|
822
|
+
def _over(clause: str, sets: dict, name: str, step: int | None, rebuild: str | None = None) -> dict:
|
|
823
|
+
"""The barrel branch over set `name`, the `all` set when the surface lacks it."""
|
|
824
|
+
present = name in sets
|
|
825
|
+
return {"clause": clause, "branch": "barrel", "surfaceSet": name if present else "all", "ladderStep": step,
|
|
826
|
+
"absentSet": None if present else name, "rebuild": rebuild}
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
def pick_denominator(sets: dict, stats_eff: int | None, deflation_fires: bool, barrel: str | None, arms: dict,
|
|
830
|
+
scope_type: str | None, skill_type: str | None, monorepo: bool, pattern_reference: bool,
|
|
831
|
+
curated: bool, source_read: bool) -> dict:
|
|
832
|
+
"""The Source API Surface Definition's clause for a skill, with the branch
|
|
833
|
+
and the name set reconcile-coverage.py and score-signatures.py read, and
|
|
834
|
+
the entry points' state it was picked by (see "Denominator" above)."""
|
|
835
|
+
return {**_pick(sets, stats_eff, deflation_fires, barrel, arms, scope_type, skill_type, monorepo,
|
|
836
|
+
pattern_reference, curated, source_read), "barrel": barrel}
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
def _pick(sets: dict, stats_eff: int | None, deflation_fires: bool, barrel: str | None, arms: dict,
|
|
840
|
+
scope_type: str | None, skill_type: str | None, monorepo: bool, pattern_reference: bool,
|
|
841
|
+
curated: bool, source_read: bool) -> dict:
|
|
842
|
+
if skill_type == "stack":
|
|
843
|
+
return {"clause": "stack", "branch": "stack", "surfaceSet": None, "ladderStep": None, "absentSet": None,
|
|
844
|
+
"rebuild": None}
|
|
845
|
+
multi = bool(arms.get("multi_subpath_exports"))
|
|
846
|
+
if (not monorepo and barrel != "barrel" and scope_type not in NOT_PATTERN_REFERENCE
|
|
847
|
+
and not (multi and scope_type == "public-api")
|
|
848
|
+
and (pattern_reference or (barrel == "no-entry-point" and not multi))):
|
|
849
|
+
if "tier_a_include" in sets:
|
|
850
|
+
return _over("pattern-reference", sets, "tier_a_include", None)
|
|
851
|
+
# The canonicalized provenance map: a surface read from the source is built again from the map alone.
|
|
852
|
+
return _over("pattern-reference", sets, "all", None, "provenance-fold" if source_read else None)
|
|
853
|
+
if barrel == "empty-barrel":
|
|
854
|
+
return _over("empty-barrel", sets, "scope.include", None)
|
|
855
|
+
if monorepo and curated and scope_type != "full-library":
|
|
856
|
+
clause = "stratified-scope"
|
|
857
|
+
elif scope_type == "specific-modules" and not monorepo:
|
|
858
|
+
clause = "specific-modules"
|
|
859
|
+
elif multi and not monorepo and scope_type in ("full-library", "public-api"):
|
|
860
|
+
clause = "multi-entry"
|
|
861
|
+
else:
|
|
862
|
+
return _over("standard-barrel", sets, "all", None)
|
|
863
|
+
ladder = CLAUSE_LADDERS[clause]
|
|
864
|
+
for step, rung in enumerate(ladder, 1):
|
|
865
|
+
if rung != "scalar":
|
|
866
|
+
if rung in sets or step == len(ladder):
|
|
867
|
+
return _over(clause, sets, rung, step)
|
|
868
|
+
elif stats_eff:
|
|
869
|
+
# A stored 0 is no denominator: the next rung.
|
|
870
|
+
if deflation_fires:
|
|
871
|
+
return _over(clause, sets, "scope.include", step)
|
|
872
|
+
scored = next((name for name in ("tier_a_include", "scope.include") if name in sets), "all")
|
|
873
|
+
return {"clause": clause, "branch": "scalar", "surfaceSet": scored, "ladderStep": step,
|
|
874
|
+
"absentSet": None, "rebuild": None}
|
|
875
|
+
raise AssertionError(f"the {clause} ladder ends on a set")
|
|
529
876
|
|
|
530
877
|
|
|
531
878
|
def surface(args: argparse.Namespace) -> dict:
|
|
532
879
|
records: list[dict] = []
|
|
533
880
|
sets: dict[str, list[str]] = {}
|
|
534
881
|
warnings: list[str] = []
|
|
535
|
-
out: dict = {"candidates": None, "guards": None, "state2": None, "canonical": None,
|
|
536
|
-
"extraction": None, "extractionGaps": [], "excluded": {"outsideScope": []}}
|
|
882
|
+
out: dict = {"candidates": None, "guards": None, "denominator": None, "state2": None, "canonical": None,
|
|
883
|
+
"extraction": None, "extractionGaps": [], "excluded": {"outsideScope": [], "nestedEntries": []}}
|
|
537
884
|
|
|
538
885
|
meta = _object(args.metadata, "--metadata") if args.metadata else None
|
|
539
886
|
provenance = _object(args.provenance, "--provenance") if args.provenance else None
|
|
@@ -541,17 +888,23 @@ def surface(args: argparse.Namespace) -> dict:
|
|
|
541
888
|
language = (brief or {}).get("language")
|
|
542
889
|
if not language and meta is not None and isinstance(meta.get("language"), str):
|
|
543
890
|
language = meta["language"]
|
|
891
|
+
stats = meta.get("stats") if isinstance(meta, dict) and isinstance(meta.get("stats"), dict) else {}
|
|
892
|
+
stats_eff = _count(stats.get("effective_denominator"))
|
|
544
893
|
|
|
545
894
|
extracted = None
|
|
546
895
|
if args.extraction:
|
|
547
|
-
|
|
896
|
+
data = _object(args.extraction, "--extraction")
|
|
897
|
+
extracted = from_extraction(data, language, brief)
|
|
548
898
|
records += extracted["records"]
|
|
549
899
|
sets.update(extracted["sets"])
|
|
550
900
|
out["extraction"] = extracted["extraction"]
|
|
551
901
|
out["extractionGaps"] = extracted["gaps"]
|
|
552
|
-
out["excluded"]
|
|
902
|
+
out["excluded"] = {"outsideScope": extracted["outside"], "nestedEntries": extracted["nested"]}
|
|
903
|
+
warnings += [f"{Path(args.extraction).name}: {w}" for w in data.get("warnings") or [] if isinstance(w, str)]
|
|
904
|
+
quick_outputs = []
|
|
553
905
|
for path in args.quick or []:
|
|
554
906
|
data = _object(path, "--quick")
|
|
907
|
+
quick_outputs.append(data)
|
|
555
908
|
records += from_quick(data)
|
|
556
909
|
warnings += [f"{Path(path).name}: {w}" for w in data.get("warnings") or [] if isinstance(w, str)]
|
|
557
910
|
violations: list[str] = []
|
|
@@ -574,11 +927,15 @@ def surface(args: argparse.Namespace) -> dict:
|
|
|
574
927
|
exports = meta.get("exports") if meta is not None else None
|
|
575
928
|
meta_names = sorted({n for n in (_export_name(e) for e in exports) if n}) if isinstance(exports, list) else []
|
|
576
929
|
base_of: dict[str, str] = {}
|
|
930
|
+
# A map whose raw count is the stored effective_denominator is already canonical: no fold.
|
|
931
|
+
skipped = bool(args.fold) and provenance is not None and stats_eff == len(prov_names)
|
|
577
932
|
if args.fold:
|
|
578
933
|
# One fold over both lists, so a variant folded out of the map
|
|
579
934
|
# never comes back through metadata.json.
|
|
580
935
|
base_of, tally = fold_names(sorted(set(prov_names) | set(meta_names)), args.fold_prefix or [],
|
|
581
936
|
set(args.keep or []), tallied=set(prov_names))
|
|
937
|
+
if skipped:
|
|
938
|
+
base_of, tally = {}, dict.fromkeys(tally, 0)
|
|
582
939
|
if provenance is not None:
|
|
583
940
|
canonical = sorted({base_of.get(n, n) for n in prov_names})
|
|
584
941
|
out["canonical"] = {
|
|
@@ -586,10 +943,14 @@ def surface(args: argparse.Namespace) -> dict:
|
|
|
586
943
|
"canonical": len(canonical),
|
|
587
944
|
"folded": tally,
|
|
588
945
|
"variants": {n: base_of[n] for n in prov_names if n in base_of},
|
|
946
|
+
"skipped": skipped,
|
|
947
|
+
"residual": None if stats_eff is None else len(canonical) - stats_eff,
|
|
589
948
|
"summary": (f"Provenance-map canonicalization: {len(prov_names)} raw entries → "
|
|
590
949
|
f"{len(canonical)} canonical bases ({len(prov_names) - len(canonical)} "
|
|
591
950
|
f"bookkeeping variants folded: _def×{tally['_def']}, _exact×{tally['_exact']}, "
|
|
592
|
-
f"a11y_×{tally[FOLD_PREFIX]}, other×{tally['other']})"
|
|
951
|
+
f"a11y_×{tally[FOLD_PREFIX]}, other×{tally['other']})"
|
|
952
|
+
+ (": the raw count equals `stats.effective_denominator`, so the map is already "
|
|
953
|
+
"canonical" if skipped else "")),
|
|
593
954
|
}
|
|
594
955
|
seen = set()
|
|
595
956
|
for e in named:
|
|
@@ -609,22 +970,39 @@ def surface(args: argparse.Namespace) -> dict:
|
|
|
609
970
|
"metadataOnly": sorted(set(meta_bases) - seen),
|
|
610
971
|
}
|
|
611
972
|
|
|
612
|
-
records.sort(key=lambda r: (r["name"], str(r["file"] or ""), r["origin"]))
|
|
613
973
|
sets["all"] = sorted({r["name"] for r in records})
|
|
974
|
+
# A nested Python top's records count in the scope sets, never in `all`.
|
|
975
|
+
records += extracted["nestedRecords"] if extracted is not None else []
|
|
976
|
+
records.sort(key=lambda r: (r["name"], str(r["file"] or ""), r["origin"]))
|
|
614
977
|
|
|
615
978
|
# The scope: the one the extractor recorded, else the brief's.
|
|
616
979
|
scope = extracted["scope"] if extracted is not None and any(extracted["scope"].values()) else None
|
|
617
980
|
if scope is None:
|
|
618
981
|
scope = brief["scope"] if brief is not None else _scope(None, None, None)
|
|
982
|
+
# A tier A glob the runner lists as matching no file counts no name, and
|
|
983
|
+
# so does one whose folder or file differs from every scope.include
|
|
984
|
+
# glob's before any `*` or `?` (`gen/**` beside `src/**`) or one a
|
|
985
|
+
# scope.exclude glob covers segment by segment (`src/gen/**` beside an
|
|
986
|
+
# exclude of `src/gen/**`), whatever the tree holds: the set comes from
|
|
987
|
+
# the other globs, over the names whose file a scope.include glob matches
|
|
988
|
+
# too (any file with none), so a restored name is in no set. A tier A
|
|
989
|
+
# list that counts no name (every glob stale, or matching only excluded
|
|
990
|
+
# files, files scope.include leaves out or files that define no name)
|
|
991
|
+
# narrows nothing: there is no set.
|
|
992
|
+
stale_tier_a = set(extracted["staleScope"]["unmatchedTierAInclude"]) if extracted is not None else set()
|
|
993
|
+
tier_a = [g for g in scope["tier_a_include"] if g not in stale_tier_a
|
|
994
|
+
and not (scope["include"] and all(_apart(g, i) for i in scope["include"]))
|
|
995
|
+
and not any(_inside(g, e) for e in scope["exclude"])]
|
|
619
996
|
filed = [r for r in records if r["file"]]
|
|
620
997
|
if filed and scope["include"]:
|
|
621
998
|
sets["scope.include"] = matching(filed, scope["include"], scope["exclude"])
|
|
622
|
-
|
|
623
|
-
|
|
999
|
+
in_include = [r for r in filed if not scope["include"]
|
|
1000
|
+
or any(glob_match(_norm(r["file"]), g) for g in scope["include"])]
|
|
1001
|
+
tier_a_names = matching(in_include, tier_a, scope["exclude"]) if tier_a else []
|
|
1002
|
+
if tier_a_names:
|
|
1003
|
+
sets["tier_a_include"] = tier_a_names
|
|
624
1004
|
|
|
625
1005
|
if extracted is not None or brief is not None or meta is not None:
|
|
626
|
-
stats = meta.get("stats") if isinstance(meta, dict) and isinstance(meta.get("stats"), dict) else {}
|
|
627
|
-
stats_eff = _count(stats.get("effective_denominator"))
|
|
628
1006
|
include_union = len(sets["scope.include"]) if "scope.include" in sets else len(sets["all"])
|
|
629
1007
|
extraction = extracted["extraction"] if extracted is not None else {}
|
|
630
1008
|
out["candidates"] = {
|
|
@@ -636,12 +1014,21 @@ def surface(args: argparse.Namespace) -> dict:
|
|
|
636
1014
|
"extractorEffectiveDenominator": extraction.get("effectiveDenominator"),
|
|
637
1015
|
"extractorBasis": extraction.get("effectiveDenominatorBasis"),
|
|
638
1016
|
}
|
|
639
|
-
|
|
640
|
-
|
|
1017
|
+
# With an extraction or the fallback scan's results (each at the file
|
|
1018
|
+
# that defines its names), no tier A set is no tier A list; otherwise
|
|
1019
|
+
# the globs left in the list decide (a name read by eye at Quick tier
|
|
1020
|
+
# or State 4, and a quick name with no defined_in, carries its entry
|
|
1021
|
+
# file, not the file that defines it).
|
|
1022
|
+
no_tier_a = ("tier_a_include" not in sets if extracted is not None or args.fallback_scan
|
|
1023
|
+
else not tier_a)
|
|
1024
|
+
deflation = {"applicable": source_read and stats_eff is not None, "effectiveDenominator": stats_eff,
|
|
641
1025
|
"rederived": include_union if source_read else None, "pct": None, "fires": False}
|
|
642
|
-
if deflation["applicable"]:
|
|
1026
|
+
if deflation["applicable"] and stats_eff:
|
|
643
1027
|
deflation["pct"] = round2((include_union - stats_eff) / stats_eff * 100)
|
|
644
1028
|
deflation["fires"] = no_tier_a and deflation["pct"] > DEFLATION_PCT
|
|
1029
|
+
elif deflation["applicable"]:
|
|
1030
|
+
# A stored 0 is deflation, not a denominator, once a name is re-derived.
|
|
1031
|
+
deflation["fires"] = no_tier_a and include_union > 0
|
|
645
1032
|
entry_count = len(_entries(provenance)) if provenance is not None else 0
|
|
646
1033
|
inflation = {"applicable": source_read and no_tier_a and bool(scope["include"]) and entry_count > 0,
|
|
647
1034
|
"scopeIncludeUnion": include_union if scope["include"] else None,
|
|
@@ -650,14 +1037,47 @@ def surface(args: argparse.Namespace) -> dict:
|
|
|
650
1037
|
if inflation["applicable"]:
|
|
651
1038
|
inflation["pct"] = round2((include_union - entry_count) / entry_count * 100)
|
|
652
1039
|
inflation["fires"] = inflation["pct"] > INFLATION_PCT
|
|
1040
|
+
stale_scope = {"applicable": extracted is not None and bool(scope["include"] or scope["tier_a_include"]),
|
|
1041
|
+
"fires": False, "unmatchedInclude": [], "unmatchedTierAInclude": [], "restored": []}
|
|
1042
|
+
if stale_scope["applicable"]:
|
|
1043
|
+
stale_scope.update(extracted["staleScope"])
|
|
1044
|
+
stale_scope["fires"] = bool(stale_scope["unmatchedInclude"] or stale_scope["unmatchedTierAInclude"])
|
|
653
1045
|
out["guards"] = {
|
|
654
1046
|
"deflation": deflation,
|
|
655
1047
|
"inflation": inflation,
|
|
1048
|
+
"staleScope": stale_scope,
|
|
1049
|
+
"tierAInclude": list(scope["tier_a_include"]),
|
|
656
1050
|
"umbrella": extracted["umbrella"] if extracted is not None else dict(NO_UMBRELLA),
|
|
657
1051
|
"thresholds": {"deflationPct": DEFLATION_PCT, "inflationPct": INFLATION_PCT,
|
|
658
1052
|
"umbrellaRatio": UMBRELLA_RATIO},
|
|
659
1053
|
}
|
|
660
1054
|
|
|
1055
|
+
# The clause: an extraction's entry-point status (one the per-file scan replaces says only that it found no
|
|
1056
|
+
# entry point), else a quick output's (an export is a barrel; none, and no warning, an empty one), else what
|
|
1057
|
+
# the workflow read (--barrel).
|
|
1058
|
+
barrel = None
|
|
1059
|
+
summary = extracted["extraction"] if extracted is not None else None
|
|
1060
|
+
if summary is not None and not summary["fallback"]["needed"]:
|
|
1061
|
+
barrel = summary["entryPointStatus"]
|
|
1062
|
+
elif summary is not None and summary["entryPointsFound"] is False:
|
|
1063
|
+
barrel = "no-entry-point"
|
|
1064
|
+
elif quick_outputs:
|
|
1065
|
+
barrel = "barrel" if any(q["exports"] for q in quick_outputs) else (
|
|
1066
|
+
None if any(q.get("warnings") for q in quick_outputs) else "empty-barrel")
|
|
1067
|
+
if barrel is None and args.barrel:
|
|
1068
|
+
barrel = BARREL_FLAG[args.barrel]
|
|
1069
|
+
recorded_type = extracted["scopeType"] if extracted is not None else None
|
|
1070
|
+
meta_type = meta.get("scope_type") if isinstance(meta, dict) and isinstance(meta.get("scope_type"), str) else None
|
|
1071
|
+
arms = summary["arms"] if summary is not None else {}
|
|
1072
|
+
# The extraction's verdict, the rule Create Skill stores the count by, else the workflow's judgment.
|
|
1073
|
+
curated = summary["curatedSubset"] if summary is not None and summary["curatedSubset"] is not None \
|
|
1074
|
+
else bool(args.curated_subset)
|
|
1075
|
+
out["denominator"] = pick_denominator(
|
|
1076
|
+
sets, stats_eff, bool((out["guards"] or {}).get("deflation", {}).get("fires")), barrel, arms,
|
|
1077
|
+
(brief or {}).get("type") or recorded_type or meta_type,
|
|
1078
|
+
meta.get("skill_type") if isinstance(meta, dict) else None, bool(args.monorepo or arms.get("monorepo")),
|
|
1079
|
+
bool(args.pattern_reference_intent), curated, source_read)
|
|
1080
|
+
|
|
661
1081
|
return {
|
|
662
1082
|
"inputs": {
|
|
663
1083
|
"extraction": args.extraction, "quick": args.quick or [], "perFile": args.per_file or [],
|
|
@@ -708,6 +1128,13 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
708
1128
|
p.add_argument("--extraction", help="skf-extract-public-api.py --mode full output")
|
|
709
1129
|
p.add_argument("--quick", action="append", help="skf-extract-public-api.py --mode quick output (repeatable)")
|
|
710
1130
|
p.add_argument("--per-file", action="append", help="a subagent per-file result (repeatable)")
|
|
1131
|
+
p.add_argument("--fallback-scan", action="store_true",
|
|
1132
|
+
help="the --per-file results are the fallback scan's, each at the file that defines its "
|
|
1133
|
+
"names: as with --extraction, no tier_a_include set is no tier A list, so the deflation "
|
|
1134
|
+
"and inflation guards run; without either, a tier_a_include glob keeps them off unless "
|
|
1135
|
+
"its folder or file differs from every scope.include glob's before any wildcard "
|
|
1136
|
+
"(gen/** beside src/**) or a scope.exclude glob covers it segment by segment "
|
|
1137
|
+
"(src/gen/** beside an exclude of src/gen/**)")
|
|
711
1138
|
p.add_argument("--name", action="append", help="a name read by eye whose file is not known (repeatable)")
|
|
712
1139
|
p.add_argument("--provenance", help="the provenance map: its named exports")
|
|
713
1140
|
p.add_argument("--metadata", help="metadata.json: its exports[] names, stats and language")
|
|
@@ -715,6 +1142,17 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
715
1142
|
p.add_argument("--fold", action="store_true", help="fold the provenance and metadata bookkeeping variants")
|
|
716
1143
|
p.add_argument("--fold-prefix", action="append", help="a renderer prefix to fold (repeatable)")
|
|
717
1144
|
p.add_argument("--keep", action="append", help="a variant that is a real export (repeatable)")
|
|
1145
|
+
p.add_argument("--monorepo", action="store_true",
|
|
1146
|
+
help="the source is a monorepo (skf-detect-workspaces.py's is_monorepo)")
|
|
1147
|
+
p.add_argument("--pattern-reference-intent", action="store_true",
|
|
1148
|
+
help="the brief's scope.notes flags pattern-reference intent (your judgment)")
|
|
1149
|
+
p.add_argument("--curated-subset", action="store_true",
|
|
1150
|
+
help="with --monorepo: the brief's scope.include lists a curated subset rather than the full "
|
|
1151
|
+
"workspace (your judgment; an extraction's counts.curated_subset wins)")
|
|
1152
|
+
p.add_argument("--barrel", choices=tuple(BARREL_FLAG),
|
|
1153
|
+
help="the entry points as you read them, where no extraction or quick output tells: none (no "
|
|
1154
|
+
"barrel file at any recognized entry-point path), empty (the primary one exports nothing) "
|
|
1155
|
+
"or barrel")
|
|
718
1156
|
p.add_argument("--output", help="also write the result to this file")
|
|
719
1157
|
p.set_defaults(func=surface)
|
|
720
1158
|
return parser
|