bmad-module-skill-forge 3.0.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +3 -3
  3. package/docs/_data/pinned.yaml +19 -13
  4. package/docs/architecture.md +2 -2
  5. package/docs/concepts.md +4 -4
  6. package/docs/examples.md +33 -26
  7. package/docs/getting-started.md +3 -3
  8. package/docs/how-it-works.md +2 -2
  9. package/docs/index.md +1 -1
  10. package/docs/skill-model.md +41 -39
  11. package/docs/troubleshooting.md +8 -0
  12. package/docs/verifying-a-skill.md +46 -21
  13. package/docs/why-skf.md +1 -1
  14. package/docs/workflows.md +5 -5
  15. package/package.json +1 -1
  16. package/src/knowledge/qmd-registry.md +1 -1
  17. package/src/knowledge/version-paths.md +1 -1
  18. package/src/shared/health-check.md +2 -50
  19. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
  20. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +12 -4
  21. package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
  22. package/src/shared/scripts/skf-atomic-write.py +212 -59
  23. package/src/shared/scripts/skf-build-change-manifest.py +1437 -178
  24. package/src/shared/scripts/skf-classify-changed-files.py +78 -0
  25. package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
  26. package/src/shared/scripts/skf-count-tokens.py +54 -10
  27. package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
  28. package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
  29. package/src/shared/scripts/skf-detect-workspaces.py +67 -0
  30. package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
  31. package/src/shared/scripts/skf-extract-public-api.py +1673 -189
  32. package/src/shared/scripts/skf-extraction-inventory.py +584 -46
  33. package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
  34. package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
  35. package/src/shared/scripts/skf-github-probe.py +47 -8
  36. package/src/shared/scripts/skf-hash-content.py +14 -8
  37. package/src/shared/scripts/skf-load-provenance.py +3 -1
  38. package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
  39. package/src/shared/scripts/skf-parse-gaps.py +135 -26
  40. package/src/shared/scripts/skf-preflight.py +15 -1
  41. package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
  42. package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
  43. package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
  44. package/src/shared/scripts/skf-resolve-customization.py +197 -0
  45. package/src/shared/scripts/skf-shard-body.py +239 -41
  46. package/src/shared/scripts/skf-structural-diff.py +133 -18
  47. package/src/shared/scripts/skf-update-active-symlink.py +115 -62
  48. package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
  49. package/src/shared/scripts/skf-validate-output.py +12 -3
  50. package/src/shared/scripts/skf-verify-provenance-completeness.py +1439 -79
  51. package/src/shared/scripts/skf-write-skill-brief.py +41 -8
  52. package/src/shared/tool-requirements.yaml +8 -2
  53. package/src/skf-analyze-source/SKILL.md +2 -2
  54. package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
  55. package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
  56. package/src/skf-audit-skill/SKILL.md +2 -2
  57. package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
  58. package/src/skf-audit-skill/references/headless-contract.md +1 -1
  59. package/src/skf-audit-skill/references/health-check.md +2 -0
  60. package/src/skf-audit-skill/references/init.md +2 -1
  61. package/src/skf-audit-skill/references/re-index.md +14 -9
  62. package/src/skf-audit-skill/references/report.md +2 -2
  63. package/src/skf-audit-skill/references/semantic-diff.md +15 -15
  64. package/src/skf-audit-skill/references/structural-diff.md +10 -5
  65. package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
  66. package/src/skf-brief-skill/SKILL.md +3 -3
  67. package/src/skf-brief-skill/assets/scope-templates.md +1 -1
  68. package/src/skf-brief-skill/assets/skill-brief-schema.md +4 -11
  69. package/src/skf-brief-skill/references/analyze-target.md +6 -6
  70. package/src/skf-brief-skill/references/confirm-brief.md +1 -1
  71. package/src/skf-brief-skill/references/scope-definition.md +4 -4
  72. package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
  73. package/src/skf-brief-skill/references/version-resolution.md +5 -9
  74. package/src/skf-campaign/SKILL.md +2 -2
  75. package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
  76. package/src/skf-create-skill/SKILL.md +2 -2
  77. package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
  78. package/src/skf-create-skill/assets/skill-sections.md +7 -7
  79. package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
  80. package/src/skf-create-skill/references/compile.md +31 -60
  81. package/src/skf-create-skill/references/component-extraction.md +3 -3
  82. package/src/skf-create-skill/references/enrich.md +2 -2
  83. package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
  84. package/src/skf-create-skill/references/extract.md +23 -19
  85. package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
  86. package/src/skf-create-skill/references/extraction-patterns.md +10 -11
  87. package/src/skf-create-skill/references/generate-artifacts.md +1 -1
  88. package/src/skf-create-skill/references/report.md +21 -5
  89. package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
  90. package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
  91. package/src/skf-create-stack-skill/SKILL.md +2 -2
  92. package/src/skf-create-stack-skill/references/validate.md +1 -1
  93. package/src/skf-drop-skill/SKILL.md +2 -2
  94. package/src/skf-drop-skill/references/execute.md +5 -1
  95. package/src/skf-drop-skill/references/health-check.md +2 -0
  96. package/src/skf-drop-skill/references/invocation-contract.md +1 -1
  97. package/src/skf-drop-skill/references/report.md +2 -2
  98. package/src/skf-drop-skill/references/select.md +7 -3
  99. package/src/skf-export-skill/SKILL.md +2 -2
  100. package/src/skf-export-skill/assets/snippet-format.md +5 -5
  101. package/src/skf-export-skill/references/generate-snippet.md +22 -15
  102. package/src/skf-export-skill/references/load-skill.md +1 -1
  103. package/src/skf-export-skill/references/token-report.md +29 -9
  104. package/src/skf-forger/SKILL.md +2 -2
  105. package/src/skf-quick-skill/SKILL.md +2 -2
  106. package/src/skf-refine-architecture/SKILL.md +6 -8
  107. package/src/skf-rename-skill/SKILL.md +2 -2
  108. package/src/skf-setup/SKILL.md +2 -2
  109. package/src/skf-test-skill/SKILL.md +2 -2
  110. package/src/skf-test-skill/assets/output-section-formats.md +6 -4
  111. package/src/skf-test-skill/references/coherence-check.md +5 -4
  112. package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
  113. package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
  114. package/src/skf-test-skill/references/coverage-check.md +69 -111
  115. package/src/skf-test-skill/references/init.md +5 -6
  116. package/src/skf-test-skill/references/migration-section-rules.md +26 -14
  117. package/src/skf-test-skill/references/report.md +11 -9
  118. package/src/skf-test-skill/references/scoring-rules.md +8 -6
  119. package/src/skf-test-skill/references/source-access-protocol.md +15 -18
  120. package/src/skf-test-skill/scripts/build-result-context.py +114 -21
  121. package/src/skf-test-skill/scripts/gap-ledger.py +580 -80
  122. package/src/skf-test-skill/scripts/load-coverage-inputs.py +646 -107
  123. package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
  124. package/src/skf-test-skill/scripts/reconcile-coverage.py +450 -23
  125. package/src/skf-test-skill/scripts/score-signatures.py +356 -61
  126. package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
  127. package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
  128. package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
  129. package/src/skf-test-skill/templates/test-report-template.md +1 -0
  130. package/src/skf-update-skill/SKILL.md +2 -2
  131. package/src/skf-update-skill/references/detect-changes.md +32 -13
  132. package/src/skf-update-skill/references/gap-driven.md +37 -22
  133. package/src/skf-update-skill/references/init.md +6 -2
  134. package/src/skf-update-skill/references/invocation-contract.md +1 -1
  135. package/src/skf-update-skill/references/merge.md +17 -12
  136. package/src/skf-update-skill/references/re-extract.md +4 -2
  137. package/src/skf-update-skill/references/report.md +7 -4
  138. package/src/skf-update-skill/references/write.md +77 -36
  139. package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
  140. package/src/skf-verify-stack/SKILL.md +2 -2
  141. package/tools/cli/lib/tool-check.js +158 -2
@@ -4,7 +4,7 @@
4
4
  # ///
5
5
  """SKF Extract Public API: one package's public surface, by parser or by recipe.
6
6
 
7
- Three modes:
7
+ Four modes:
8
8
 
9
9
  --mode quick (the default) A pure parser: takes one logical package (one
10
10
  manifest, one or more entry-point files) and emits its
@@ -16,9 +16,10 @@ Three modes:
16
16
  raw fetches into a folder, or a local checkout), so a file's
17
17
  text never passes through the model or a shell string.
18
18
  --follow also reads the modules an `export *` chain passes
19
- names on from, fetching them first with --repo. Multi-module
20
- monorepos are caller-orchestrated: invoke once per module,
21
- aggregate.
19
+ names on from, fetching them first with --repo, and
20
+ --resolve-version looks up a version the manifest does
21
+ not state. Multi-module monorepos are caller-orchestrated:
22
+ invoke once per module, aggregate.
22
23
  --mode full The ast-grep recipe runner (skf-create-skill and the
23
24
  workflows that extract as it does): runs the recipes in
24
25
  src/shared/data/ast-grep-recipes.yaml over a source tree
@@ -30,6 +31,11 @@ Three modes:
30
31
  rules, so no caller derives an entry file from a
31
32
  distribution name or reads a manifest by eye. See "Entries
32
33
  mode" below.
34
+ --mode denominator
35
+ Full mode's counts and effective_denominator decision for
36
+ public names counted without its diff (by quick mode or by
37
+ eye), and its scope facts with no recipe run. See
38
+ "Denominator mode" below.
33
39
 
34
40
  Quick mode
35
41
  ----------
@@ -120,7 +126,7 @@ or the files, read from disk (stdin is then never read):
120
126
  --mode quick --language <language> [--source-root <dir>]
121
127
  [--manifest-file <path>] [--entry-file <path>]...
122
128
  [--follow-file <path>]... [--tree-file <listing>] [--fetch-list <file>]
123
- [--follow [--repo <owner/repo> [--ref <ref>]]]
129
+ [--follow] [--resolve-version] [--repo <owner/repo> [--ref <ref>]]
124
130
 
125
131
  An entry with `followed` true (a --follow-file) is a module an `unlisted[]`
126
132
  record named, read as an entry is, less a JS/TS `export default`, which
@@ -138,6 +144,41 @@ A module that cannot be fetched or read stays in `unlisted[]`, and a
138
144
  warning names it and why. The output then also lists `followed`, the
139
145
  modules read this way, in the order they were read.
140
146
 
147
+ --resolve-version (with --source-root) looks up a version the manifest does
148
+ not state, in this order, and names where it came from in `version_source`:
149
+
150
+ python the `__version__` literal of an entry `__init__.py`, then of the
151
+ `_version.py` beside it, then the `version=` literal of the
152
+ setup.py beside the manifest (a dynamic version, `attr:` or
153
+ `file:`, or a version setup.py computes)
154
+ rust the `[workspace.package] version` of the workspace root, the
155
+ nearest Cargo.toml at or above the manifest's folder that holds
156
+ a `[workspace]` table (`version.workspace = true`)
157
+ any the release tag: with --repo, skf-github-probe.py's tag listing
158
+ at --ref (the newest tag whose name is a release version, X.Y.Z,
159
+ at the default branch; the pinned tag itself at a tag; none at a
160
+ branch or a commit); without, the tag `git describe --tags
161
+ --abbrev=0` names in --source-root, when its name is a version
162
+
163
+ A file the lookups read is read under --source-root; with --repo and
164
+ --tree-file, one the listing holds is fetched there first, at --ref. A
165
+ version a file gives must start with a digit and be no placeholder, and
166
+ one a tag gives drops a leading `v`. For JavaScript and TypeScript read
167
+ with --repo at the default branch, a manifest pre-release (`-alpha`,
168
+ `-beta`, `-rc`, `-next`, `-canary`) gives way to the newest release, with
169
+ a warning. When nothing gives a version, a warning says so.
170
+
171
+ With --source-root, each Python, JavaScript and TypeScript export also
172
+ carries `defined_in`: the file that defines it, traced as full mode traces
173
+ an entry point (through each import, re-export, `export *` and star import
174
+ to the end of its chain, a cycle cut), so a name an entry file only passes
175
+ on names its own module. It reads only files under --source-root (the
176
+ files full mode lists there, never the --tree-file listing) and needs no
177
+ ast-grep: Python with full mode's module reader, JavaScript and TypeScript
178
+ from each module's text as quick mode reads an entry file. It is null when
179
+ the trace finds no file, as for a name passed on from another package or a
180
+ module --source-root does not hold. `source_file` stays the file read.
181
+
141
182
  Each path is relative to --source-root, the folder the files were staged in
142
183
  laid out like the repository (or a local checkout), else to the current
143
184
  folder, and is the `path` the output names (`source_file`), written with
@@ -155,9 +196,20 @@ Output JSON shape (stdout):
155
196
  {
156
197
  "language": "python",
157
198
  "package_name": "foo",
158
- "version": "1.2.3",
199
+ "version": "1.2.3", (the manifest's literal
200
+ version, else null: a
201
+ go.mod, a dynamic or
202
+ workspace-inherited one,
203
+ a placeholder; with
204
+ --resolve-version, the
205
+ one it looked up)
206
+ "version_source": "manifest" | "file:<path>" | "tag:<tag>" | null,
207
+ (--resolve-version only)
159
208
  "description": "...",
160
- "exports": [{"name": "Bar", "type": "class", "source_file": "..."}, ...],
209
+ "exports": [{"name": "Bar", "type": "class", "source_file": "...",
210
+ "defined_in": "..."}, ...], (defined_in: Python,
211
+ JS and TS, with
212
+ --source-root)
161
213
  "dependencies": ["requests", "pydantic", ...],
162
214
  "modules": ["server", "client"], (Maven/Gradle only)
163
215
  "module_folders": ["server", "client"], (each module's folder in
@@ -193,10 +245,12 @@ Exit codes (quick):
193
245
  or given twice with the file inputs, a --manifest-file, --entry-file
194
246
  or --follow-file that cannot be read or leads outside --source-root,
195
247
  a --tree-file that cannot be read or a --fetch-list that cannot be
196
- written, --follow without --tree-file and --source-root, --repo or
197
- --ref without --follow (--ref without --repo), or a --repo or --ref
198
- skf-github-fetch.py does not accept. A module --follow could not
199
- read changes no exit code.
248
+ written, --follow without --tree-file and --source-root,
249
+ --resolve-version without --source-root, --repo or --ref without
250
+ --follow or --resolve-version (--ref without --repo), or a --repo or
251
+ --ref skf-github-fetch.py does not accept. A module --follow could
252
+ not read, and a version --resolve-version could not find, change no
253
+ exit code.
200
254
 
201
255
  Entries mode
202
256
  ------------
@@ -276,7 +330,15 @@ is one the recipes read (python, typescript, tsx, javascript, rust, go and
276
330
  .vue files) and --language allows it. Globs follow
277
331
  skf-resolve-authoritative-files.py's rules: `**` spans any number of path
278
332
  segments, none included, and `*` and `?` stay inside one, so `src/**/*.ts`
279
- matches `src/index.ts` and `**/test_*` matches a top-level `test_x.py`.
333
+ matches `src/index.ts` and `**/test_*` matches a top-level `test_x.py`. An
334
+ include glob that matches no file (a brief written for an older layout,
335
+ say) is named in `warnings` and listed in `scope.unmatched_include`. A
336
+ scope.tier_a_include glob is matched against the whole tree under
337
+ --source-root, regardless of --files-from and before the excludes: one
338
+ that matches no file is named in `warnings` too and listed in
339
+ `scope.unmatched_tier_a_include`, and, when scope.include has a glob, one
340
+ that matches only files no scope.include glob matches is named in
341
+ `warnings`.
280
342
 
281
343
  --brief supplies scope.include, scope.exclude, scope.tier_a_include,
282
344
  scope.type and language; --include, --exclude, --tier-a-include,
@@ -288,7 +350,11 @@ scope.type and language; --include, --exclude, --tier-a-include,
288
350
  Which recipes. --recipe-set picks the recipes whose metadata.sets lists it:
289
351
  `component-library` when the scope type is component-library, else
290
352
  `standard`. Each recipe runs in the languages its metadata.languages
291
- lists. A match whose name starts with `_` is never an export. --head-cap
353
+ lists. A Python or Rust match whose name starts with `_` is never an
354
+ export, nor public anywhere below: Python's by the Source API Surface
355
+ Definition's rule, Rust's because a `_` item is hidden or unnamed (a
356
+ `#[doc(hidden)] pub mod __private`, which the runner cannot read, or a
357
+ `pub const _`); a TS/JS `_` name is an export like any other. --head-cap
292
358
  keeps at most N matches of each recipe, in file and line order (0: no
293
359
  cap); without it the cap is 200, or at Forge+ and Deep tiers 500 for a
294
360
  full-library scope and 300 for a component-library one. `truncated` is
@@ -337,34 +403,125 @@ JS/TS one that is not). Chains are followed to the end, a cycle cut: an
337
403
  `export *`, a star import or a glob passes on what its module exports,
338
404
  that module's own chains included; a re-exported name is traced through
339
405
  each re-export, import and glob between to the file that defines it (a
340
- default export to the name it declares). A namespace (`export * as ns`, a
341
- Python submodule, a Rust `pub mod`) counts once, and what it makes
342
- reachable is not internal: the names its module passes on, the modules
343
- among them in turn, and for Rust every recipe match in the crate root and
344
- in each module file a `pub mod` or `pub use` of a reachable module names.
406
+ default export to the name it declares). A Python package's explicit
407
+ binding of a name (an import, or a def or class in its __init__.py)
408
+ comes before its submodule: `from .x import y` imports the y
409
+ x/__init__.py binds, and the submodule x/y.py only when x/__init__.py
410
+ binds no y or binds the submodule itself (`from . import y`, `from
411
+ pkg.x import y`); a y x/__init__.py binds only through a star import
412
+ (`from .impl import *`) is still read as the submodule x/y.py. A
413
+ namespace (`export * as ns`, a Python submodule, a Rust `pub mod`)
414
+ lists in `members` the names its module passes on, which users reach as
415
+ `pkg.ns.name`, each traced to the file that defines it as a public name
416
+ is, and in place of a namespace among them its members (each module once,
417
+ so a cycle is cut); `members` is null when the trace cannot read the
418
+ module (a JavaScript target no file here holds, a Rust inline or
419
+ `#[path]` module), and a nested one the trace cannot read is a member
420
+ itself. A member re-exported from another package has no file, is no
421
+ gap when a recipe found its name, marks its name public, and, unlike the
422
+ same form in an entry point, adds no `unresolved` item. What a namespace
423
+ makes reachable is not internal: the names its module passes on, the
424
+ modules among them in turn, and for Rust every recipe match in the crate
425
+ root and in each module file a `pub mod` or `pub use` of a reachable
426
+ module names.
345
427
  A chain the trace cannot follow (a module no file here holds, a `#[path]`
346
428
  module) is listed in `unresolved`. The diff against the recipe inventory,
347
429
  one language family at a time (JavaScript/TypeScript, Python, Rust, Go):
348
- `public` (the entry points' names), `internal` (names the recipes found
349
- that no entry point exports or makes reachable), `extraction_gaps` (names
350
- an entry point exports that no recipe found, defined in a file in scope or
351
- in none the trace reached: read them by eye) and `outside_scope` (such
352
- names defined in a file out of scope: widen scope.include to document
353
- them). In a family with no entry point, or only empty ones, every name the
354
- recipes found is public.
430
+ `public` (the entry points' names, each at the file and line the first
431
+ entry point naming it gives, in path order, a JS/TS entry point's own and
432
+ named exports all coming before any `export *`, unless that file is out
433
+ of scope, or it gives none, and another entry point gives a file in
434
+ scope, other than through a star or glob its own export of the name
435
+ shadows),
436
+ `other_definitions` (each other definition of a public name in a file in
437
+ scope, one row per file and `local` name: the file and line an entry
438
+ point gives, never a namespace nor one beside a public row that is a
439
+ namespace, nor a star or glob through which an entry point passes on a
440
+ name it already exports (its own export shadows it), while two exports
441
+ of the name by one entry point, such as Rust `pub use` items a `cfg`
442
+ gates, are both kept; in every language family, so a name each
443
+ framework package of a monorepo defines keeps
444
+ each package's file, while a gap and an outside_scope row still go by
445
+ name, from `public` alone),
446
+ `internal` (names the recipes found that no entry point exports or makes
447
+ reachable), `extraction_gaps` (names an entry point exports that no
448
+ recipe found, defined in a file in scope or in none the trace reached:
449
+ read them by eye) and `outside_scope` (names an entry point exports that
450
+ a file out of scope defines, even when a recipe found the name in another
451
+ file, which is another definition: widen scope.include to document
452
+ them). A namespace with `members` is diffed as its members: each is found
453
+ when a recipe found its name, or its `local` one, in its file (one with
454
+ no file by its name alone, as a public name is), and has no row of its
455
+ own when a public name (or another namespace's member) has its name and
456
+ file; one with null `members`, and a member that is such a module, is no
457
+ gap. In a family with no entry point, or only empty ones, every name the
458
+ recipes found is public. A JS/TS module exports, besides the names the
459
+ recipes match, those its column-0 `export` statements declare in a form
460
+ no recipe matches, read from its text: `export let`, `export declare
461
+ const`, `export namespace`, and each name a destructuring pattern that
462
+ opens an `export const`, `let` or `var` statement binds (`export const
463
+ { a, b: c } = obj` exports a and c), at the line the name is on. Such a
464
+ name that an entry point exports, or passes on through `export *`, is
465
+ public and, no recipe having found it, an `extraction_gaps` row
466
+ (`outside_scope`, in a file out of scope) at that line. A pattern after
467
+ the statement's first declarator (`export const a = 1, { b } = o`), or
468
+ one the reader cannot read (one that does not close, or a default
469
+ holding a template literal over several lines), binds nothing here and
470
+ no warning names it: read it by eye.
355
471
 
356
472
  Counts. `exports_public_api` counts the public names (the package's whole
357
- public API, `outside_scope` names included) and `exports_internal` the
358
- internal ones. `effective_denominator` counts the public names whose
359
- definition file matches scope.tier_a_include (else scope.include) and no
360
- scope.exclude glob: each name once, so a type counts once whatever its
361
- methods, and a barrel that only re-exports adds nothing of its own. `arms` gives compile's effective_denominator arms:
362
- (a) `monorepo` from skf-detect-workspaces.py, (b) `specific_modules`
473
+ public API, `outside_scope` names included), each distinct name and file
474
+ `public` gives once (`other_definitions` adds none): a namespace with
475
+ `members` counts as its members (none for an empty list, and a member
476
+ that is also a public name, or another namespace's member, once), one
477
+ with null `members` as one. `exports_internal` counts the internal names.
478
+ `effective_denominator` counts those public names with a definition file
479
+ (the `public` row's, or one of its `other_definitions`') that matches
480
+ scope.include (any file when it has no glob), scope.tier_a_include when
481
+ given, and no scope.exclude glob: each name once, however many in-scope
482
+ files define it, so a type counts once whatever its methods, a barrel
483
+ that only re-exports adds nothing of its own, and a name the entry points
484
+ pass on from two files (two packages' entry points that each define it,
485
+ or a namespace member named like another public name, or like another
486
+ namespace's member, at another file) or in two languages counts once,
487
+ when any one of its files matches, as Test Skill's sets count it. A tier
488
+ A list that counts no public name (each glob matches no file, or only
489
+ files scope.exclude drops, that scope.include leaves out or that define
490
+ no public name) narrows nothing on a run that counted a public name and
491
+ read every file (no `errors` item): the count falls back to scope.include
492
+ (else every file), with a warning.
493
+ `arms` gives compile's effective_denominator arms:
494
+ (a) `monorepo` from skf-detect-workspaces.py over the tree the brief
495
+ keeps (its scoped_tree: a member counts only through the files no
496
+ scope.exclude glob matches, so excluding its sources drops it, excluding
497
+ only its manifest does not, root manifests always stay, and a workspace
498
+ with no member left is none), (b) `specific_modules`
363
499
  (scope type specific-modules on a repository that is not a monorepo) and
364
500
  (c) `multi_subpath_exports` (an in-scope package.json whose `exports` map
365
501
  has more than one non-root subpath without a `*`, on a repository that is
366
502
  not a monorepo).
367
503
 
504
+ Store or not. `curated_subset` is true when the files the count's basis
505
+ globs list (scope.include's, else every file; a tier A count's, those
506
+ among them a tier A glob matches too; under arm (b), less those a
507
+ scope.exclude glob matches) leave out a file that defines a counted public
508
+ name (an `outside_scope` name's file and an `other_definitions` file
509
+ included) or every source file of a
510
+ workspace skf-detect-workspaces.py finds over the whole tree
511
+ (scope.exclude aside, unlike arm (a): a member the brief drops is still
512
+ part of the repository it chose a subset of); a basis of `all-files` lists
513
+ none. `effective_denominator_decision` says whether create-skill and
514
+ update-skill store `effective_denominator` as stats.effective_denominator,
515
+ the brief's scope.notes aside (documenting the stratification is the one
516
+ judgment left to them): `store` is true only when none of these holds, and
517
+ `reason` names the first that does, else `curated-subset`: `reference-app`
518
+ (scope type reference-app: pattern_surfaces_documented is its basis),
519
+ `incomplete-run` (an `errors` item: a file a failed call left unread can
520
+ hide a public name, so the count can be 0 or partial), `no-arm` (no arm is
521
+ true), `full-library` (scope type full-library), `all-files` (the basis),
522
+ `not-curated` (`curated_subset` false) and `zero-count` (a stored 0 is no
523
+ denominator: Test Skill reads it as deflation).
524
+
368
525
  Output JSON (stdout, or -o):
369
526
 
370
527
  {
@@ -376,7 +533,9 @@ Output JSON (stdout, or -o):
376
533
  "ast_grep": {"path": "<exe>" | null, "version": "<x.y.z>" | null},
377
534
  "scope": {"include": [...], "exclude": [...],
378
535
  "tier_a_include": [...] | null, "type": "<type>" | null,
379
- "languages": [...], "files_from": "<file>" | null},
536
+ "languages": [...], "files_from": "<file>" | null,
537
+ "unmatched_include": [...], # include globs no file matches
538
+ "unmatched_tier_a_include": [...]}, # tier A globs no file matches
380
539
  "files_in_scope": N,
381
540
  "files_by_language": {"<language>": N, ...},
382
541
  "files_without_recipes": {".java": N, ...}, # in scope, no recipe reads them
@@ -409,35 +568,46 @@ Output JSON (stdout, or -o):
409
568
  | {"entry", "name", "from", "file"}, ...]},
410
569
  "entry_point_diff": {
411
570
  "public": [{"name", "language", "entry", "via", "from", "local",
412
- "file", "line"}, ...],
571
+ "file", "line",
572
+ "members": [{"name", "local", "file", "line"}, ...]
573
+ | null}, ...], # members: via namespace only
574
+ "other_definitions": [{"name", "language", "entry", "via", "from",
575
+ "local", "file", "line"}, ...], # by name, then file
413
576
  "internal": [{"name", "language", "source_file", "source_line"}, ...],
414
577
  "extraction_gaps": [{"name", "language", "entry", "file", "line"}, ...],
415
578
  "outside_scope": [{"name", "language", "entry", "file", "line"}, ...]},
416
579
  "reexport_targets": [{"name", "language", "local", "from", "entry",
417
580
  "file", "line", "via"}, ...],
418
- "counts": {"exports_public_api": N, "exports_internal": N,
419
- "effective_denominator": N,
581
+ "counts": {"exports_public_api": N, # (family, name, file) keys
582
+ "exports_internal": N,
583
+ "effective_denominator": N, # names, each once
420
584
  "effective_denominator_basis": "tier_a_include" | "scope.include"
421
- | "all-files",
422
- "denominator_files": N},
585
+ | "all-files", # not tier_a_include when it counts no scope.include name
586
+ "denominator_files": N,
587
+ "curated_subset": bool,
588
+ "effective_denominator_decision": {"store": bool, "reason": "<reason>"}},
423
589
  "arms": {"monorepo": bool, "monorepo_kind": "<kind>" | null,
424
590
  "specific_modules": bool, "multi_subpath_exports": bool},
425
591
  "errors": [{"reason": "ast-grep-error" | "ast-grep-timeout" | "time-limit",
426
592
  "files": N, "first_file", "detail",
427
- "unread": [...]}, ...], # unread: time-limit only
428
- "warnings": ["..."]
593
+ "unread": [...]}, ...], # unread: the files its call(s) may have left unread
594
+ "warnings": ["..."],
595
+ "files_in_scope_list": ["<file>", ...], # the files files_in_scope counts, in tree order
596
+ "files_without_recipes_list": ["<file>", ...] # the files files_without_recipes counts
429
597
  }
430
598
 
431
599
  `via` says how an entry point exports a name: `declaration` (defined in the
432
600
  entry point), `re-export` (from another file, which `file` names, `local`
433
601
  giving the name it has there when it differs), `star` (through `export *`,
434
602
  `from .x import *` or `pub use x::*`) or `namespace` (a module: `export *
435
- as ns`, a Python submodule, a Rust module). `line` is null where no line
436
- defines the name (a module, a name from another package). An `unresolved`
437
- item names a subpath whose targets name no file, or a chain met tracing
438
- `entry` that stops at `file`, the module `from` naming no file here.
439
- Without ast-grep the JSON still has the entry points' `files`, the
440
- `exports_maps`, the unresolved subpaths and the `arms`.
603
+ as ns`, a Python submodule, a Rust module, its names in `members`). `line`
604
+ is null where no line defines the name (a module, a name from another
605
+ package), and a member's `file` is null for a name from another package.
606
+ An `unresolved` item names a subpath whose targets name no file, or a
607
+ chain met tracing `entry` that stops at `file`, the module `from` naming
608
+ no file here.
609
+ Without ast-grep the JSON still has the files in scope, the entry points'
610
+ `files`, the `exports_maps`, the unresolved subpaths and the `arms`.
441
611
  `file_issues` lists the files to read by eye: a file ast-grep skips (not
442
612
  UTF-8), and a file where the parser met code it could not read (an ERROR
443
613
  node), where a recipe can miss an export. `t1_low` is 0: every export here
@@ -449,14 +619,18 @@ under the 10-minute limit an agent's shell call often has) have passed
449
619
  since the run started: the run going on is stopped, no other starts, and
450
620
  an `errors[]` item with the reason `time-limit` lists in `unread` the
451
621
  files left unread (each ast-grep call also stops after 600 seconds,
452
- `ast-grep-timeout`). The JSON is still written, status "incomplete".
622
+ `ast-grep-timeout`). A call that times out keeps no match; one that exited
623
+ keeps the matches it printed before any line that is not JSON, but a kill
624
+ can cut a file's matches anywhere, so the item of every failed call lists
625
+ all its files in `unread`. The JSON is still written, status "incomplete".
453
626
 
454
627
  Exit codes (full):
455
628
 
456
629
  0 the recipes ran on every file in scope ("ok")
457
- 1 an ast-grep run failed (exited non-zero) or timed out, or the
458
- --timeout ran out: its files may be unread and the result is
459
- incomplete ("incomplete"); `errors` lists them
630
+ 1 an ast-grep call could not start, exited non-zero, printed other
631
+ than JSON lines or timed out, or the --timeout ran out: the result
632
+ is incomplete ("incomplete"), and each `errors` item lists in
633
+ `unread` the files it may have left unread
460
634
  2 input error (no source root, an unreadable brief, file list or
461
635
  recipe file, a recipe without its metadata, no PyYAML, an -o file
462
636
  that cannot be written, a --timeout not above 0), or a failure the
@@ -466,6 +640,72 @@ Exit codes (full):
466
640
  beside it (cmd.exe would read a repository path's `&` as syntax);
467
641
  the JSON has the scope and no exports; extract by source reading
468
642
 
643
+ Denominator mode
644
+ ----------------
645
+
646
+ --mode denominator [--source-root <dir> | --tree-file <listing>]
647
+ [--brief <skill-brief.yaml>] [--include GLOB]... [--exclude GLOB]...
648
+ [--tier-a-include GLOB]... [--scope-type TYPE] [--language LANG]...
649
+ [--quick <quick.json>]... [--extraction <full.json>] [-o <out.json>]
650
+ < {payload_json}
651
+
652
+ The counts full mode writes, for public names no runner diff gave: those
653
+ a stdin payload lists,
654
+
655
+ {"public": [{"name", "file", "language",
656
+ "members": [{"name", "file"}, ...]}, ...],
657
+ "exports": [{"name", "file", "language"}, ...]}
658
+
659
+ (`file` relative to --source-root, with `/`; `members` for a submodule
660
+ the entry points export by name, which counts as the names its module
661
+ passes on), each export of a --quick output at its `defined_in` (else its
662
+ `source_file`), and the --extraction runner's `entry_point_diff.public`;
663
+ a payload name replaces a --quick or runner one of its family and name.
664
+ The runner's `other_definitions` count in `effective_denominator` and
665
+ `curated_subset` as full mode counts them, never in `exports_public_api`,
666
+ each only while the record kept for its name names the file of the
667
+ runner's `public` row (a row whose name has no runner row, or one a
668
+ payload or --quick name moves to another file, adds nothing), and none
669
+ beside a namespace with members, as in full mode.
670
+ `exports_internal` counts the names of the payload's `exports` (the
671
+ exports extraction found; Python, Rust and Dart `_` names left out, those
672
+ of every other language counted) and of the runner's `internal` that no
673
+ public name or member has. `effective_denominator`,
674
+ its basis, `curated_subset` and `effective_denominator_decision` follow
675
+ full mode's rules, over the tree under --source-root, with the scope and
676
+ arms full mode works out there; an --extraction that is `incomplete` is
677
+ never stored. A name whose `file` is not in that tree counts in
678
+ `exports_public_api` alone, with a warning. With no --source-root (a
679
+ remote source read by eye) the tree is the --tree-file listing (a file,
680
+ or `-` for stdin, which then holds no payload: JSON with a `tree` list, a
681
+ list, or one path per line; one that lists no file is an input error),
682
+ else the names' files; the arms are all false, there is no entry point,
683
+ and the decision is never to store (`no-source-tree`). An empty payload
684
+ (`{}`, or no stdin) gives the scope facts alone, with no recipe run:
685
+ `scope`, the two file lists (by full mode's Which files rule, null with
686
+ neither a tree nor a listing), `entry_points` (`files`, `exports_maps`)
687
+ and `arms`.
688
+
689
+ `entry_point_diff` replaces a compare by eye, each item {"name",
690
+ "language", "file"}: `public`, the names `exports_public_api` counts (a
691
+ namespace as its `members`); `other_definitions`, the runner's other
692
+ definitions the counts read, by name then file; `internal`, the exports
693
+ `exports_internal` counts; `extraction_gaps`, the public names the
694
+ payload and --quick give that no export found has (the payload's
695
+ `exports`, the --extraction runner's names), to trace (the runner lists
696
+ its own gaps).
697
+
698
+ Output JSON: `mode` "denominator", `source_root`, `scope` (as full mode
699
+ writes it), `files_in_scope`, `files_without_recipes`, `entry_points`,
700
+ `arms`, `runner_status` (the --extraction's `status`, else null),
701
+ `counts` (as full mode writes it), `entry_point_diff`, `warnings`, and
702
+ last `files_in_scope_list` and `files_without_recipes_list` (as full
703
+ mode writes them), to -o when given (its folder created). Exit 0, or 2
704
+ on an input error (a --source-root that is no folder, a brief, --quick,
705
+ --extraction or --tree-file that cannot be read, --tree-file beside
706
+ --source-root, a payload that is not this shape, a flag of another
707
+ mode).
708
+
469
709
  CLI examples:
470
710
  uv run skf-extract-public-api.py --mode quick < {payload_json}
471
711
  uv run skf-extract-public-api.py --mode entries --language python \\
@@ -722,7 +962,7 @@ def parse_package_swift(content: str) -> dict:
722
962
  """Best-effort parse of a SwiftPM Package.swift manifest.
723
963
 
724
964
  SwiftPM has no version field in the manifest (versions come from git tags),
725
- so `version` is always None; the brief falls back to target_version / default.
965
+ so `version` is always None; a caller reads it from a git tag.
726
966
  """
727
967
  name_m = re.search(r"\bPackage\s*\(\s*name:\s*['\"]([^'\"]+)['\"]", content, re.DOTALL)
728
968
  deps: list[str] = []
@@ -1332,6 +1572,35 @@ def _detect_placeholder_version(version: str | None) -> str | None:
1332
1572
  return None
1333
1573
 
1334
1574
 
1575
+ def _literal_version(version: object, warnings: list[str]) -> str | None:
1576
+ """The version the manifest writes as a literal string, else None.
1577
+
1578
+ A placeholder and a version that is not a string (Cargo's
1579
+ `version = { workspace = true }`, which the workspace root's Cargo.toml
1580
+ holds) give None with a warning, so the output's `version` is a string
1581
+ or null, which --resolve-version looks up."""
1582
+ if version is None:
1583
+ return None
1584
+ if not isinstance(version, str):
1585
+ if isinstance(version, dict) and version.get("workspace") is True:
1586
+ warnings.append(
1587
+ "manifest version is inherited from the workspace (version.workspace = true), "
1588
+ "which the workspace root's Cargo.toml holds, so the manifest states none"
1589
+ )
1590
+ else:
1591
+ warnings.append(f"manifest version {version!r} is not a string, so the manifest states none")
1592
+ return None
1593
+ placeholder = _detect_placeholder_version(version)
1594
+ if placeholder is not None:
1595
+ warnings.append(
1596
+ f"manifest version {placeholder!r} is a release-time placeholder "
1597
+ f"(workspace protocol or semantic-release sentinel) and is not a real version, "
1598
+ f"so the manifest states none"
1599
+ )
1600
+ return None
1601
+ return version
1602
+
1603
+
1335
1604
  def extract(payload: dict) -> dict:
1336
1605
  """Orchestrate manifest parse + export scan for one logical package."""
1337
1606
  warnings: list[str] = []
@@ -1353,14 +1622,7 @@ def extract(payload: dict) -> dict:
1353
1622
  warnings.append(parsed["_parse_error"])
1354
1623
  parsed = {}
1355
1624
 
1356
- placeholder = _detect_placeholder_version(parsed.get("version"))
1357
- if placeholder is not None:
1358
- warnings.append(
1359
- f"manifest version {placeholder!r} is a release-time placeholder "
1360
- f"(workspace protocol or semantic-release sentinel) and is not a real version; "
1361
- f"the brief will fall back to user-supplied target_version or the default"
1362
- )
1363
- parsed["version"] = None
1625
+ parsed["version"] = _literal_version(parsed.get("version"), warnings)
1364
1626
 
1365
1627
  _, scanner = LANGUAGE_DISPATCH[language]
1366
1628
  exports: list[dict] = []
@@ -1536,6 +1798,8 @@ JS_SOURCE_OF = {
1536
1798
  JS_TARGET_SUFFIXES = (".d.mts", ".d.cts", ".d.ts", ".mjs", ".cjs", ".jsx", ".js", ".mts", ".cts", ".tsx", ".ts")
1537
1799
  PACKAGE_ENTRY_FIELDS = ("types", "typings", "module", "main", "browser")
1538
1800
  WORKSPACE_MANIFESTS = ("package.json", "pnpm-workspace.yaml", "lerna.json", "Cargo.toml", "rush.json", "nx.json")
1801
+ # The arms under which a curated count can be stored as stats.effective_denominator.
1802
+ DENOMINATOR_ARMS = ("monorepo", "specific_modules", "multi_subpath_exports")
1539
1803
 
1540
1804
 
1541
1805
  class RunnerError(Exception):
@@ -1560,8 +1824,9 @@ def _sibling(filename: str):
1560
1824
  """A helper installed beside this script, loaded once as a module:
1561
1825
  skf-source-tree.py (git without the caller's hooks and git variables,
1562
1826
  and a tool lookup that never takes a shim planted in the current
1563
- folder), skf-resolve-authoritative-files.py (the brief's glob rules) and
1564
- skf-detect-workspaces.py (monorepo detection). Raises RunnerError."""
1827
+ folder), skf-resolve-authoritative-files.py (the brief's YAML and glob
1828
+ rules) and skf-detect-workspaces.py (monorepo detection). Raises
1829
+ RunnerError."""
1565
1830
  module = _SIBLINGS.get(filename)
1566
1831
  if module is None:
1567
1832
  path = Path(__file__).resolve().parent / filename
@@ -1856,8 +2121,9 @@ def _read_file_list(path: Path) -> list[str]:
1856
2121
  def _read_brief(path: Path) -> dict:
1857
2122
  """The scope fields and language of a skill brief."""
1858
2123
  yaml = _yaml()
2124
+ parse_brief_yaml = _sibling("skf-resolve-authoritative-files.py").parse_brief_yaml
1859
2125
  try:
1860
- data = yaml.safe_load(path.read_text(encoding="utf-8"))
2126
+ data = parse_brief_yaml(path.read_text(encoding="utf-8"))
1861
2127
  except (OSError, UnicodeDecodeError) as exc:
1862
2128
  raise RunnerError(f"cannot read brief {path}: {exc}") from exc
1863
2129
  except yaml.YAMLError as exc:
@@ -2003,12 +2269,17 @@ def _scan(exe: str, root: Path, folder: Path, rules: list[dict], files: list[str
2003
2269
  errors: list[dict], clock: _Clock | None = None) -> list[dict]:
2004
2270
  """ast-grep's matches of `rules` in `files` (relative to `root`), in
2005
2271
  batches. The rules go in a file as JSON documents (YAML reads JSON),
2006
- never on the command line. A call that times out, exits non-zero (its
2007
- rules carry no severity, so a scan that read every file exits 0, matches
2008
- or not) or prints other than JSON lines adds to `errors`: its batch's
2009
- files may be unread. The matches it printed are kept. When `clock` (the
2010
- run's --timeout) runs out, the call running is stopped and no other
2011
- starts: one `time-limit` item names every file left unread."""
2272
+ never on the command line. A call that cannot start, times out, exits
2273
+ non-zero (its rules carry no severity, so a scan that read every file
2274
+ exits 0, matches or not) or prints other than JSON lines adds to
2275
+ `errors` an item whose `unread` lists every file of its batch: ast-grep
2276
+ prints a file's matches as one block per rule, with other files' blocks
2277
+ between them, and a kill can stop it anywhere, so the output cannot show
2278
+ which files it printed in full. A call that exited keeps the matches it
2279
+ printed before any line that is not JSON; a call that timed out keeps
2280
+ none. When `clock` (the run's --timeout) runs out, a call running is
2281
+ stopped (its `time-limit` item lists its batch) and no other starts: a
2282
+ `time-limit` item lists the files no call read, if any."""
2012
2283
  if not rules or not files:
2013
2284
  return []
2014
2285
  rule_file = folder / "rules.yml"
@@ -2055,10 +2326,8 @@ def _scan(exe: str, root: Path, folder: Path, rules: list[dict], files: list[str
2055
2326
  if match is not None:
2056
2327
  found.append(match)
2057
2328
  if failure is not None:
2058
- error = {"reason": failure[0], "files": len(batch), "first_file": batch[0], "detail": failure[1]}
2059
- if failure[0] == "time-limit":
2060
- error["unread"] = list(batch)
2061
- errors.append(error)
2329
+ errors.append({"reason": failure[0], "files": len(batch), "first_file": batch[0], "detail": failure[1],
2330
+ "unread": list(batch)})
2062
2331
  return found
2063
2332
 
2064
2333
 
@@ -2410,11 +2679,22 @@ def _preferred(at_line: list[dict], forms: dict, order: dict[str, int]) -> dict:
2410
2679
  return min(left, key=lambda m: (order[m["rule"]], m["language"]))
2411
2680
 
2412
2681
 
2682
+ def _private(family: str | None, name: str) -> bool:
2683
+ """A name full mode and denominator mode never count: one that starts
2684
+ with `_` in Python (the Source API Surface Definition leaves it out),
2685
+ Rust (a `#[doc(hidden)] pub mod __private`, which the runner cannot
2686
+ read, or a `pub const _`) or Dart (private to its library). A TS/JS `_`
2687
+ name is exported like any other, and Go's recipe reads only a
2688
+ capitalized name. Quick mode drops only Python's `_` names."""
2689
+ return family in ("python", "rust", "dart") and name.startswith("_")
2690
+
2691
+
2413
2692
  def merge_exports(matches: list[dict], recipes: list[dict], recipe_set: str,
2414
2693
  head_cap: int | None) -> tuple[list[dict], list[dict]]:
2415
2694
  """(exports, per-recipe stats) from the matches of one recipe set: at
2416
2695
  most `head_cap` matches of each recipe, in file and line order, merged
2417
- by (file, name) at the first line that names it."""
2696
+ by (file, name) at the first line that names it. A Python or Rust `_`
2697
+ name is no export."""
2418
2698
  forms = _forms(recipes)
2419
2699
  order: dict[str, int] = {}
2420
2700
  languages: dict[str, list[str]] = {}
@@ -2425,7 +2705,8 @@ def merge_exports(matches: list[dict], recipes: list[dict], recipe_set: str,
2425
2705
  langs.extend(lang for lang in recipe["metadata"]["languages"] if lang not in langs)
2426
2706
  per_recipe: dict[str, list[dict]] = {rid: [] for rid in order}
2427
2707
  for m in matches:
2428
- if m["rule"] in per_recipe and m["name"] and not m["name"].startswith("_") and (m["rule"], m["language"]) in forms:
2708
+ if (m["rule"] in per_recipe and m["name"] and not _private(FAMILY_OF.get(m["language"]), m["name"])
2709
+ and (m["rule"], m["language"]) in forms):
2429
2710
  per_recipe[m["rule"]].append(m)
2430
2711
  stats, kept = [], []
2431
2712
  for rid, found in per_recipe.items():
@@ -2622,14 +2903,190 @@ _JS_DEFAULT_NAME_RE = re.compile(r"^export\s+default\s+([A-Za-z_$][\w$]*)\s*;?\s
2622
2903
  _JS_DEFAULT_EXPORT_RE = re.compile(r"export\s+default\b")
2623
2904
 
2624
2905
 
2906
+ # Keep identical to _js_pattern_bindings in skf-verify-provenance-completeness.py.
2907
+ def _js_pattern_bindings(
2908
+ text: str, start: int
2909
+ ) -> tuple[list[tuple[str, int]], int | None]:
2910
+ """The names the destructuring pattern whose `{` or `[` is at
2911
+ `text[start]` binds, each with its offset in `text`, and the offset
2912
+ just past the pattern's closing bracket.
2913
+
2914
+ An object pattern's property is `...target`, a shorthand key or
2915
+ `key: target` (a key is a name, a string, a number or a computed
2916
+ `[...]`, and only a shorthand key binds itself); an array pattern's
2917
+ element is `...target`, a target or a hole; a target is a name or a
2918
+ nested pattern. A `= default` is skipped to the next comma at the
2919
+ pattern's depth, and so is a property or element of any other shape,
2920
+ which binds nothing. '...', "..." and `...` strings are skipped
2921
+ whole, so no bracket or comma inside one counts. Whitespace, newlines
2922
+ included, may sit anywhere between the parts.
2923
+
2924
+ Returns ([], None) when `text[start]` opens no pattern, the pattern
2925
+ does not close in `text`, a string in it does not end, or a bracket in
2926
+ it closes one it did not open. A pure function of its arguments (no
2927
+ module state), so another script can carry a copy of it.
2928
+ """
2929
+ n = len(text)
2930
+ names: list[tuple[str, int]] = []
2931
+
2932
+ def space(i: int) -> int:
2933
+ while i < n and text[i].isspace():
2934
+ i += 1
2935
+ return i
2936
+
2937
+ def name_end(i: int) -> int:
2938
+ """The end of the identifier at `i`, `i` itself when none starts there."""
2939
+ if i < n and (text[i].isalpha() or text[i] in "_$"):
2940
+ i += 1
2941
+ while i < n and (text[i].isalnum() or text[i] in "_$"):
2942
+ i += 1
2943
+ return i
2944
+
2945
+ def string_end(i: int) -> int | None:
2946
+ """The offset past the string whose quote is at `i`."""
2947
+ quote = text[i]
2948
+ i += 1
2949
+ while i < n:
2950
+ if text[i] == "\\":
2951
+ i += 2
2952
+ elif text[i] == quote:
2953
+ return i + 1
2954
+ else:
2955
+ i += 1
2956
+ return None
2957
+
2958
+ def skip(i: int, to_comma: bool = True) -> int | None:
2959
+ """From `i`, the offset of the next comma (unless not `to_comma`)
2960
+ or closing bracket at depth 0; with `to_comma` False, `text[i]`
2961
+ opens a bracket and the offset past the one closing it."""
2962
+ depth = 0
2963
+ while i < n:
2964
+ ch = text[i]
2965
+ if ch in "'\"`":
2966
+ end = string_end(i)
2967
+ if end is None:
2968
+ return None
2969
+ i = end
2970
+ continue
2971
+ if ch in "([{":
2972
+ depth += 1
2973
+ elif ch in ")]}":
2974
+ if depth == 0:
2975
+ return i
2976
+ depth -= 1
2977
+ if depth == 0 and not to_comma:
2978
+ return i + 1
2979
+ elif ch == "," and depth == 0 and to_comma:
2980
+ return i
2981
+ i += 1
2982
+ return None
2983
+
2984
+ def default(i: int) -> int | None:
2985
+ """After a target: past its `= default`, when it has one."""
2986
+ i = space(i)
2987
+ if i < n and text[i] == "=" and text[i + 1 : i + 2] not in ("=", ">"):
2988
+ return skip(i + 1)
2989
+ return i
2990
+
2991
+ def bind(i: int, end: int, close: str) -> int | None:
2992
+ """The name `text[i:end]` is bound when its property or element
2993
+ ends after it or takes a default (any other text: another shape)."""
2994
+ after = space(end)
2995
+ if after < n and (
2996
+ text[after] in (",", close)
2997
+ or text[after] == "=" and text[after + 1 : after + 2] not in ("=", ">")
2998
+ ):
2999
+ names.append((text[i:end], i))
3000
+ return default(after)
3001
+ return after
3002
+
3003
+ def target(i: int, close: str) -> int | None:
3004
+ if i < n and text[i] in "{[":
3005
+ end = pattern(i)
3006
+ return None if end is None else default(end)
3007
+ end = name_end(i)
3008
+ return i if end == i else bind(i, end, close)
3009
+
3010
+ def element(i: int, close: str) -> int | None:
3011
+ """One property or element at `i`: the offset after it."""
3012
+ if text.startswith("...", i):
3013
+ return target(space(i + 3), close)
3014
+ if close == "]":
3015
+ return target(i, close)
3016
+ if text[i] in "'\"":
3017
+ end = string_end(i)
3018
+ elif text[i] == "[":
3019
+ end = skip(i, to_comma=False) # a computed key
3020
+ else:
3021
+ end = name_end(i)
3022
+ if end > i:
3023
+ after = space(end)
3024
+ if after < n and text[after] == ":":
3025
+ return target(space(after + 1), close)
3026
+ return bind(i, end, close) # a shorthand key
3027
+ while end < n and (text[end].isalnum() or text[end] in "._"):
3028
+ end += 1 # a number key
3029
+ if end is None:
3030
+ return None
3031
+ after = space(end)
3032
+ if end > i and after < n and text[after] == ":":
3033
+ return target(space(after + 1), close)
3034
+ return after
3035
+
3036
+ def pattern(i: int) -> int | None:
3037
+ """The pattern opening at `i`: the offset past its closing bracket."""
3038
+ close = "}" if text[i] == "{" else "]"
3039
+ i += 1
3040
+ while True:
3041
+ i = space(i)
3042
+ if i >= n:
3043
+ return None
3044
+ if text[i] == close:
3045
+ return i + 1
3046
+ if text[i] in ")]}":
3047
+ return None
3048
+ if text[i] == ",":
3049
+ i += 1 # a hole, or a stray comma
3050
+ continue
3051
+ end = element(i, close)
3052
+ if end is None:
3053
+ return None
3054
+ end = space(end)
3055
+ if end < n and text[end] not in (",", close):
3056
+ end = skip(end) # any other shape binds nothing
3057
+ if end is None:
3058
+ return None
3059
+ i = end + 1 if end < n and text[end] == "," else end
3060
+
3061
+ if not (0 <= start < n and text[start] in "{["):
3062
+ return [], None
3063
+ end = pattern(start)
3064
+ return (names, end) if end is not None else ([], None)
3065
+
3066
+
2625
3067
  def _js_text_names(text: str) -> tuple[list[tuple[str, int, bool]], list[tuple[str, int]]]:
2626
3068
  """The names a JS/TS file's column-0 `export` declarations bind, read
2627
3069
  from its text for the forms no recipe matches (`export let`, `export
2628
- declare const`, `export namespace`), as (name, line, is the default
2629
- export), and its `export default <name>;` statements, as (name, line)."""
3070
+ declare const`, `export namespace`, and each name a destructuring
3071
+ pattern that opens an `export const`, `let` or `var` statement binds:
3072
+ `export const { a, b: c, ...d } = obj` binds a, c and d), as (name,
3073
+ line, is the default export), and its `export default <name>;`
3074
+ statements, as (name, line).
3075
+
3076
+ A pattern is read from its `{` or `[` on (`_JS_DESTRUCTURE_RE`) with
3077
+ `_js_pattern_bindings`, so it may span lines, and each name it binds
3078
+ takes the line the name is on. A pattern after the first declarator
3079
+ is not read, and one that does not close, or that nests deeper than
3080
+ the interpreter's recursion limit, binds nothing."""
2630
3081
  code = _blank(text, "js")
2631
3082
  declared = [(m.group(2), _line_of(code, m.start(2)), bool(m.group(1)))
2632
3083
  for rx in (_JS_DECLARATION_RE, _JS_IMPORT_EQUALS_RE) for m in rx.finditer(code)]
3084
+ for m in _JS_DESTRUCTURE_RE.finditer(code):
3085
+ try:
3086
+ names, _ = _js_pattern_bindings(code, m.end() - 1)
3087
+ except RecursionError:
3088
+ continue # patterns nested past the interpreter's limit: none bound
3089
+ declared += [(name, _line_of(code, offset), False) for name, offset in names]
2633
3090
  defaults = [(m.group(1), _line_of(code, m.start(1))) for m in _JS_DEFAULT_NAME_RE.finditer(code)]
2634
3091
  return declared, defaults
2635
3092
 
@@ -2861,26 +3318,96 @@ def _specifier_local(text: str) -> str:
2861
3318
  return re.split(r"\s+as\s+", item, maxsplit=1)[0].strip().strip("'\"")
2862
3319
 
2863
3320
 
2864
- class _Surface:
2865
- """The public names one language family's entry points declare:
2866
- {name: info}, the first entry point naming it wins; the names, and the
2867
- files, a namespace export (a module) makes reachable, which are not
2868
- internal; and the re-exports the trace could not follow."""
3321
+ def _member(name: str, local: str | None, file: str | None, line: int | None) -> dict:
3322
+ """One name a namespace's module passes on, traced to the file that
3323
+ defines it (`local` the name it has there when it differs)."""
3324
+ return {"name": name, "local": local if local and local != name else None, "file": file, "line": line}
3325
+
3326
+
3327
+ def _flatten(start, direct: Callable[[object], list[tuple[str, str, object]]], family: str) -> list[dict]:
3328
+ """The members of the namespace module `start`, of language family
3329
+ `family`: what `direct` says it passes on, (name, "member", member) or
3330
+ (name, "module", the module), each nested namespace module's members
3331
+ spliced in its place (each module once, so a cycle is cut), each
3332
+ (name, file) once, and no Python or Rust name that starts with `_`,
3333
+ which is never public."""
3334
+ out: list[dict] = []
3335
+ seen: set[tuple[str, str | None]] = set()
3336
+ visited: set = set()
3337
+
3338
+ def walk(module) -> None:
3339
+ visited.add(module)
3340
+ for name, kind, value in direct(module):
3341
+ if not name or _private(family, name):
3342
+ continue
3343
+ if kind == "module":
3344
+ if value not in visited:
3345
+ walk(value)
3346
+ elif (value["name"], value["file"]) not in seen:
3347
+ seen.add((value["name"], value["file"]))
3348
+ out.append(value)
3349
+
3350
+ walk(start)
3351
+ return out
3352
+
2869
3353
 
2870
- def __init__(self) -> None:
3354
+ class _Surface:
3355
+ """The public names the entry points of language family `family`
3356
+ declare: {name: info}, the first `add` of a name wins (the JS/TS trace
3357
+ adds every entry point's own and named exports before any `export *`),
3358
+ unless the file it gives is out of scope (or it gives none) and another
3359
+ entry point gives a file in `in_scope`, which then replaces it (one
3360
+ entry point never replaces its own, and a star or glob that passes on a
3361
+ name its entry point already exports replaces none: an explicit export
3362
+ shadows it; with no `in_scope`, the first always wins); `others`,
3363
+ {name: [info]}, the other definitions of a held name in files in
3364
+ `in_scope` (#830), each an add that does not become the held record,
3365
+ is no such shadowed star or glob, is no namespace, beside a held record
3366
+ that is none, and gives a file in scope whose (file, `local` or name)
3367
+ neither the held record nor an earlier one of `others` gives; the
3368
+ names, and the files, a namespace export (a module) makes reachable,
3369
+ which are not internal; and the re-exports the trace could not follow.
3370
+ A namespace item also carries its `members`, the names its module
3371
+ passes on (None when the trace cannot read that module); `opaque` holds
3372
+ the (name, file) of each member that is itself a module the trace
3373
+ cannot read, which, as such a namespace, is no recipe's target. A
3374
+ Python or Rust `_` name is never added."""
3375
+
3376
+ def __init__(self, family: str, in_scope: set[str] | None = None) -> None:
3377
+ self.family = family
3378
+ self.in_scope = in_scope
2871
3379
  self.names: dict[str, dict] = {}
3380
+ self.others: dict[str, list[dict]] = {}
3381
+ self._other_at: dict[str, set[tuple[str, str]]] = {} # name: the (file, local or name) of its others
3382
+ # (entry, name) of each add: a star or glob an entry point passes an exported name on through is shadowed
3383
+ self._added: set[tuple[str, str]] = set()
2872
3384
  self.reachable: set[str] = set()
2873
3385
  self.reachable_files: set[str] = set()
2874
3386
  self.unresolved: list[dict] = []
3387
+ self.opaque: set[tuple[str, str | None]] = set()
2875
3388
 
2876
3389
  def add(self, name: str, entry: str, via: str, file: str | None, line: int | None,
2877
- source: str | None = None, local: str | None = None) -> dict | None:
2878
- if not name or name.startswith("_"):
3390
+ source: str | None = None, local: str | None = None,
3391
+ members: list[dict] | None = None) -> dict | None:
3392
+ if not name or _private(self.family, name):
2879
3393
  return None
2880
- if name not in self.names:
2881
- self.names[name] = {"name": name, "entry": entry, "via": via, "from": source,
2882
- "local": local if local and local != name else None,
2883
- "file": file, "line": line}
3394
+ held = self.names.get(name)
3395
+ shadowed = via == "star" and (entry, name) in self._added
3396
+ self._added.add((entry, name))
3397
+ info = {"name": name, "entry": entry, "via": via, "from": source,
3398
+ "local": local if local and local != name else None, "file": file, "line": line}
3399
+ at = (file, info["local"] or name)
3400
+ if held is None or (not shadowed and self.in_scope is not None and file in self.in_scope
3401
+ and entry != held["entry"]
3402
+ and (held["file"] is None or held["file"] not in self.in_scope)):
3403
+ self.names[name] = info
3404
+ if via == "namespace":
3405
+ self.names[name]["members"] = [dict(m) for m in members] if members is not None else None
3406
+ elif (not shadowed and via != "namespace" and held["via"] != "namespace" and self.in_scope is not None
3407
+ and file in self.in_scope and at != (held["file"], held["local"] or name)
3408
+ and at not in self._other_at.get(name, ())):
3409
+ self.others.setdefault(name, []).append(info)
3410
+ self._other_at.setdefault(name, set()).add(at)
2884
3411
  return self.names[name]
2885
3412
 
2886
3413
  def unresolve(self, entry: str, name: str, source: str | None, file: str) -> None:
@@ -3053,6 +3580,40 @@ def _js_surface(root: Path, entries: list[dict], tree_set: set[str], forms: dict
3053
3580
  if fact is not None and fact["how"] == "namespace":
3054
3581
  reach(target(fact["source"], where))
3055
3582
 
3583
+ passes: dict[str, list[tuple[str, str, object]]] = {}
3584
+
3585
+ def direct(file: str) -> list[tuple[str, str, object]]:
3586
+ """What the namespace module `file` passes on, in order: (name,
3587
+ "member", a member traced as publish traces a name) or (name,
3588
+ "module", the file of a namespace among them); a namespace whose
3589
+ module no file here holds is a member itself, with no file, and so
3590
+ is a re-export from another package, which (unlike the same form in
3591
+ an entry point) is not unresolved."""
3592
+ if file not in passes:
3593
+ out: list[tuple[str, str, object]] = []
3594
+ for name, owner in passed_on(file).items():
3595
+ where, local, line, fact = resolve(owner, name, set())
3596
+ how = fact["how"] if fact is not None else None
3597
+ namespace = target(fact["source"], where) if how == "namespace" else None
3598
+ if namespace is not None:
3599
+ out.append((name, "module", namespace))
3600
+ elif how == "namespace":
3601
+ surface.opaque.add((name, None))
3602
+ out.append((name, "member", _member(name, None, None, None)))
3603
+ elif how == "re-export":
3604
+ out.append((name, "member", _member(name, local, None, None)))
3605
+ else:
3606
+ out.append((name, "member", _member(name, local, where, line)))
3607
+ passes[file] = out
3608
+ return passes[file]
3609
+
3610
+ member_lists: dict[str, list[dict]] = {}
3611
+
3612
+ def members(file: str) -> list[dict]:
3613
+ if file not in member_lists:
3614
+ member_lists[file] = _flatten(file, direct, "javascript")
3615
+ return member_lists[file]
3616
+
3056
3617
  def publish(entry: str, name: str, via: str, module: str, source: str | None) -> None:
3057
3618
  """Add public `name`, which `module` exports, traced to what it is."""
3058
3619
  chain: set[str] = set()
@@ -3061,7 +3622,8 @@ def _js_surface(root: Path, entries: list[dict], tree_set: set[str], forms: dict
3061
3622
  how = fact["how"] if fact is not None else None
3062
3623
  if how == "namespace":
3063
3624
  namespace = target(fact["source"], where)
3064
- surface.add(name, entry, "namespace", namespace, None, source)
3625
+ surface.add(name, entry, "namespace", namespace, None, source,
3626
+ members=members(namespace) if namespace is not None else None)
3065
3627
  if namespace is None:
3066
3628
  surface.unresolve(entry, name, fact["source"], where)
3067
3629
  reach(namespace)
@@ -3371,9 +3933,16 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
3371
3933
  if found is None:
3372
3934
  return None, binding["imported"], None, "import"
3373
3935
  if found.endswith("__init__.py"):
3374
- sub = _py_submodule(posixpath.dirname(found), binding["imported"], tree_set)
3375
- if sub and sub != rel:
3376
- return sub, binding["imported"], None, "submodule"
3936
+ # Python's own precedence: the package's explicit binding of
3937
+ # the name first, the submodule only when it binds none (or
3938
+ # binds the submodule itself, `from . import name` or
3939
+ # `from pkg.x import name`); a star import is not read here
3940
+ inner = bindings(found)[1].get(binding["imported"])
3941
+ if inner is None or (inner["kind"] == "import" and inner["imported"] == binding["imported"]
3942
+ and module_file(found, inner["level"], inner["module"], top) == found):
3943
+ sub = _py_submodule(posixpath.dirname(found), binding["imported"], tree_set)
3944
+ if sub and sub != rel:
3945
+ return sub, binding["imported"], None, "submodule"
3377
3946
  rel, name = found, binding["imported"]
3378
3947
  return rel, name, None, "missing"
3379
3948
 
@@ -3391,6 +3960,31 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
3391
3960
  if what == "submodule" and where is not None:
3392
3961
  reach(where, top)
3393
3962
 
3963
+ passes: dict[tuple[str, str], list[tuple[str, str, object]]] = {}
3964
+
3965
+ def direct(rel: str, top: str) -> list[tuple[str, str, object]]:
3966
+ """What the submodule `rel` passes on (its star_names), in order:
3967
+ (name, "member", a member traced by resolve) or (name, "module",
3968
+ the file of a submodule among them); a name from another package
3969
+ has no file."""
3970
+ if (rel, top) not in passes:
3971
+ out: list[tuple[str, str, object]] = []
3972
+ for name, owner in star_names(rel, top).items():
3973
+ where, local, line, what = resolve(owner, name, top, set())
3974
+ if what == "submodule" and where is not None:
3975
+ out.append((name, "module", where))
3976
+ else:
3977
+ out.append((name, "member", _member(name, local, where, line)))
3978
+ passes[(rel, top)] = out
3979
+ return passes[(rel, top)]
3980
+
3981
+ member_lists: dict[tuple[str, str], list[dict]] = {}
3982
+
3983
+ def members(rel: str, top: str) -> list[dict]:
3984
+ if (rel, top) not in member_lists:
3985
+ member_lists[(rel, top)] = _flatten(rel, lambda module: direct(module, top), "python")
3986
+ return member_lists[(rel, top)]
3987
+
3394
3988
  for entry in [e["file"] for e in entries]:
3395
3989
  top = posixpath.dirname(entry)
3396
3990
  all_names, bound, stars = bindings(entry)
@@ -3420,7 +4014,7 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
3420
4014
  where, local, line, what = resolve(entry, name, top, chain)
3421
4015
  surface.reachable.update(chain)
3422
4016
  if what == "submodule":
3423
- surface.add(name, entry, "namespace", where, None, source, local)
4017
+ surface.add(name, entry, "namespace", where, None, source, local, members=members(where, top))
3424
4018
  reach(where, top)
3425
4019
  elif what == "import":
3426
4020
  surface.add(name, entry, "re-export", None, None, source, local)
@@ -3674,6 +4268,41 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
3674
4268
  if sub is not None:
3675
4269
  reach(lib, sub, reached)
3676
4270
 
4271
+ passes: dict[tuple[str, tuple[str, ...]], list[tuple[str, str, object]]] = {}
4272
+
4273
+ def direct(lib: str, here: tuple[str, ...]) -> list[tuple[str, str, object]]:
4274
+ """What the module at `here` passes on (exported), in order: (name,
4275
+ "member", a member traced as publish traces a name) or (name,
4276
+ "module", the path of a module among them that a file of its own
4277
+ holds); an inline or `#[path]` module is a member itself, at its
4278
+ `pub mod`, and a `pub use` of another crate's item one with no file,
4279
+ which (unlike the same form in the crate root) is not unresolved."""
4280
+ if (lib, here) not in passes:
4281
+ out: list[tuple[str, str, object]] = []
4282
+ for name in exported(lib, list(here)):
4283
+ file, local, line, sub, stop = resolve(lib, list(here), name, set())
4284
+ if stop is not None:
4285
+ out.append((name, "member", _member(name, local, None, None)))
4286
+ elif sub is not None and _rust_file(lib, sub, tree_set) is not None:
4287
+ out.append((name, "module", tuple(sub)))
4288
+ else:
4289
+ if sub is not None:
4290
+ surface.opaque.add((name, file))
4291
+ out.append((name, "member", _member(name, local, file, line)))
4292
+ passes[(lib, here)] = out
4293
+ return passes[(lib, here)]
4294
+
4295
+ member_lists: dict[tuple[str, tuple[str, ...]], list[dict]] = {}
4296
+
4297
+ def members(lib: str, here: list[str]) -> list[dict] | None:
4298
+ """The members of the module at `here`; None when no file of its own holds it."""
4299
+ if _rust_file(lib, here, tree_set) is None:
4300
+ return None
4301
+ key = (lib, tuple(here))
4302
+ if key not in member_lists:
4303
+ member_lists[key] = _flatten(tuple(here), lambda path: direct(lib, path), "rust")
4304
+ return member_lists[key]
4305
+
3677
4306
  def publish(lib: str, here: list[str], name: str, via: str, source: str | None,
3678
4307
  reached: set[tuple[str, ...]]) -> None:
3679
4308
  """Add public `name`, which the module at `here` exports, traced to
@@ -3685,7 +4314,7 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
3685
4314
  surface.add(name, lib, via, None, None, source, local)
3686
4315
  surface.unresolve(lib, name, *stop)
3687
4316
  elif sub is not None:
3688
- surface.add(name, lib, "namespace", file, line, "::".join(sub))
4317
+ surface.add(name, lib, "namespace", file, line, "::".join(sub), members=members(lib, sub))
3689
4318
  reach(lib, sub, reached)
3690
4319
  else:
3691
4320
  surface.add(name, lib, via, file, line, source, local)
@@ -3700,7 +4329,8 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
3700
4329
  continue
3701
4330
  sub = [name.removeprefix("r#")]
3702
4331
  sub_file = _rust_file(lib, sub, tree_set)
3703
- surface.add(name, lib, "namespace", sub_file or lib, None if sub_file else line)
4332
+ surface.add(name, lib, "namespace", sub_file or lib, None if sub_file else line,
4333
+ members=members(lib, sub))
3704
4334
  reach(lib, sub, reached)
3705
4335
  for path, alias, glob, _ in info["uses"]:
3706
4336
  if not glob:
@@ -3770,22 +4400,34 @@ def _go_surface(root: Path, files: list[str], surface: _Surface, entries: list[d
3770
4400
 
3771
4401
  def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries: list[dict],
3772
4402
  in_scope: set[str]) -> tuple[dict[str, str], dict[tuple[str, str], dict], list[dict],
3773
- list[tuple[str, dict]], list[tuple[str, dict]]]:
4403
+ list[tuple[str, dict]], list[tuple[str, dict]],
4404
+ list[tuple[str, dict]]]:
3774
4405
  """The entry-point diff, one language family at a time: (status by
3775
4406
  family, public names {(family, name): info}, internal export records,
3776
- and (family, info) of the extraction gaps and of the names outside
3777
- scope). A family whose entry points declare names diffs them with its
4407
+ and (family, info) of the extraction gaps, of the names outside scope
4408
+ and of the other in-scope definitions of public names, `_Surface`'s
4409
+ `others`, which no gap or outside_scope row reads: those go by name).
4410
+ A family whose entry points declare names diffs them with its
3778
4411
  exports: an export no entry point names, nor traces back to, nor makes
3779
4412
  reachable (a name, or any export of a reachable file), is internal. A
3780
- declared name no recipe found is a gap when a file in scope defines it
3781
- (or the trace found no file), and is outside scope when a file out of
3782
- scope does; a module is no recipe's target, so neither. In a family
3783
- with no entry point, or only empty ones, every export is public."""
4413
+ declared name, at the file `_Surface` kept for it (not always the first
4414
+ entry point's in path order), is outside scope when a file out of scope
4415
+ defines it, even when a recipe found its name in another file (another
4416
+ definition: no recipe reads a file out of scope), and a gap when no
4417
+ recipe found it and a file in scope defines it (or the trace found no
4418
+ file); a module is no recipe's target, so neither: a namespace with
4419
+ `members` has each member diffed in its place, found when a recipe
4420
+ found its name, or its `local` one, in its file, and given no row of
4421
+ its own when a declared name has its name and file (or another
4422
+ namespace gave it one); one whose members are unknown (None), or a
4423
+ member that is such a module (`opaque`), is no gap. In a family with
4424
+ no entry point, or only empty ones, every export is public."""
3784
4425
  statuses: dict[str, str] = {}
3785
4426
  public: dict[tuple[str, str], dict] = {}
3786
4427
  internal: list[dict] = []
3787
4428
  gaps: list[tuple[str, dict]] = []
3788
4429
  outside: list[tuple[str, dict]] = []
4430
+ others: list[tuple[str, dict]] = []
3789
4431
  entry_families = {FAMILY_OF[e["language"]] for e in entries}
3790
4432
  families = {FAMILY_OF[e["language"]] for e in exports} | entry_families
3791
4433
  for family in sorted(families | {f for f, s in surfaces.items() if s.names}):
@@ -3803,6 +4445,7 @@ def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries
3803
4445
  names = {e["export_name"] for e in records}
3804
4446
  traced = {info["local"] for info in declared.values() if info["local"]} | surface.reachable
3805
4447
  public.update(((family, name), info) for name, info in declared.items())
4448
+ others.extend((family, dict(info)) for name in sorted(surface.others) for info in surface.others[name])
3806
4449
  seen: set[str] = set()
3807
4450
  for e in records:
3808
4451
  name = e["export_name"]
@@ -3811,10 +4454,58 @@ def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries
3811
4454
  seen.add(name)
3812
4455
  internal.append(e)
3813
4456
  for name, info in declared.items():
3814
- if name in names or (info["local"] or name) in names or info["via"] == "namespace":
4457
+ if info["via"] == "namespace":
3815
4458
  continue
3816
- (gaps if info["file"] is None or info["file"] in in_scope else outside).append((family, info))
3817
- return statuses, public, internal, gaps, outside
4459
+ # A recipe never reads a file out of scope: a name it found elsewhere is another definition.
4460
+ if info["file"] is not None and info["file"] not in in_scope:
4461
+ outside.append((family, info))
4462
+ elif name not in names and (info["local"] or name) not in names:
4463
+ gaps.append((family, info))
4464
+ # A namespace's members, each by its name (or `local`) and file, or
4465
+ # by name alone when it has no file, as a declared name is: a name
4466
+ # at a file a declared name already gives counts once.
4467
+ found = {(e["export_name"], e["source_file"]) for e in records}
4468
+ given = {(info["name"], info["file"]) for info in declared.values() if not _has_members(info)}
4469
+ for info in declared.values():
4470
+ for member in info["members"] if _has_members(info) else []:
4471
+ key = (member["name"], member["file"])
4472
+ if (key in given or key in found or key in surface.opaque
4473
+ or (member["local"] and (member["local"], member["file"]) in found)
4474
+ or (member["file"] is None and {member["name"], member["local"]} & names)):
4475
+ continue
4476
+ given.add(key)
4477
+ row = {"name": member["name"], "entry": info["entry"], "file": member["file"],
4478
+ "line": member["line"]}
4479
+ (gaps if member["file"] is None or member["file"] in in_scope else outside).append((family, row))
4480
+ return statuses, public, internal, gaps, outside, others
4481
+
4482
+
4483
+ def _has_members(info: dict) -> bool:
4484
+ """A namespace item whose module's members the trace read (a list, maybe empty)."""
4485
+ return info.get("via") == "namespace" and isinstance(info.get("members"), list)
4486
+
4487
+
4488
+ def _public_keys(public: dict[tuple[str, str], dict]) -> set[tuple[str, str, str | None]]:
4489
+ """The (family, name, file) exports_public_api counts: each public name
4490
+ but a namespace with members, in its place each member, and so a member
4491
+ that is also a declared name (or another namespace's member) once."""
4492
+ keys: set[tuple[str, str, str | None]] = set()
4493
+ for (family, _), info in public.items():
4494
+ if _has_members(info):
4495
+ keys.update((family, m["name"], m["file"]) for m in info["members"])
4496
+ else:
4497
+ keys.add((family, info["name"], info["file"]))
4498
+ return keys
4499
+
4500
+
4501
+ def _other_keys(others: list[tuple[str, dict]]) -> set[tuple[str, str, str | None]]:
4502
+ """The (family, name, file) of the other in-scope definitions of public
4503
+ names (#830), which join the public keys in the set effective_denominator
4504
+ and curated_subset read, never exports_public_api: a name still counts
4505
+ once, when any of its files matches."""
4506
+ return {(family, info["name"], info["file"]) for family, info in others
4507
+ if isinstance(info.get("name"), str) and info["name"] and isinstance(info.get("file"), str)
4508
+ and info["file"]}
3818
4509
 
3819
4510
 
3820
4511
  def _head_cap(value: int | None, tier: str | None, scope_type: str | None) -> int | None:
@@ -3823,27 +4514,133 @@ def _head_cap(value: int | None, tier: str | None, scope_type: str | None) -> in
3823
4514
  return HEAD_CAPS.get((tier, scope_type), DEFAULT_HEAD_CAP)
3824
4515
 
3825
4516
 
3826
- def _monorepo(root: Path, tree: list[str], tree_set: set[str]) -> tuple[bool, str | None]:
3827
- """skf-detect-workspaces.py's verdict on the source root."""
4517
+ def _workspaces(root: Path, tree: list[str], tree_set: set[str]) -> dict:
4518
+ """skf-detect-workspaces.py's result over `tree` (its workspaces
4519
+ resolved against those paths), with the content of each root manifest
4520
+ of WORKSPACE_MANIFESTS that `tree_set` lists. No brief scope applies:
4521
+ the monorepo arm reads it through _monorepo."""
3828
4522
  manifests = {}
3829
4523
  for name in WORKSPACE_MANIFESTS:
3830
4524
  if name in tree_set:
3831
4525
  text = _read_text(root, name)
3832
4526
  if text is not None:
3833
4527
  manifests[name] = text
3834
- result = _sibling("skf-detect-workspaces.py").detect({"tree": tree, "manifests": manifests})
4528
+ return _sibling("skf-detect-workspaces.py").detect({"tree": tree, "manifests": manifests})
4529
+
4530
+
4531
+ def _monorepo(root: Path, tree: list[str], excludes: list[str],
4532
+ glob_match: Callable[[str, str], bool]) -> tuple[bool, str | None]:
4533
+ """Arm (a): (is the source a monorepo, its manifest kind), from
4534
+ skf-detect-workspaces.py over the tree the brief keeps, its
4535
+ scoped_tree: a workspace member counts only through the files no
4536
+ scope.exclude glob matches, every root manifest stays, and a workspace
4537
+ with no member left is none (#780)."""
4538
+ kept = _sibling("skf-detect-workspaces.py").scoped_tree(tree, excludes, glob_match)
4539
+ result = _workspaces(root, kept, set(kept))
3835
4540
  return bool(result.get("is_monorepo")), result.get("manifest_kind")
3836
4541
 
3837
4542
 
4543
+ def _workspace_sources(result: dict, tree: list[str], wanted: set[str]) -> list[set[str]]:
4544
+ """The source files (of a language --language selects) under each
4545
+ workspace of `result`, one set per workspace that holds any. _scope_facts
4546
+ passes the workspaces skf-detect-workspaces.py finds over the whole
4547
+ tree, scope.exclude aside (unlike arm (a)): a member the brief drops is
4548
+ still part of the repository the brief chose a subset of."""
4549
+ sources = []
4550
+ for workspace in result.get("workspaces") or [] if result.get("is_monorepo") else []:
4551
+ folder = _norm(workspace.get("path") if isinstance(workspace, dict) else "").strip("/")
4552
+ files = {f for f in tree if (not folder or f.startswith(folder + "/")) and _file_language(f) in wanted}
4553
+ if files:
4554
+ sources.append(files)
4555
+ return sources
4556
+
4557
+
4558
+ def denominator_count(counted: set[tuple[str, str, str | None]], tree: list[str], includes: list[str],
4559
+ excludes: list[str], tier_a: list[str], glob_match: Callable[[str, str], bool],
4560
+ read_all: bool) -> tuple[int, str, set[str], str | None]:
4561
+ """(effective_denominator, its basis, the files it counts, a warning or None)
4562
+ over the (family, name, file) keys `counted`: each name once whose file
4563
+ matches scope.include (any file with no glob), scope.tier_a_include when
4564
+ given (a tier A name counts only in a file scope.include matches too) and
4565
+ no scope.exclude glob. A tier A list that counts no name falls back to
4566
+ scope.include (else every file) once a name was counted and every file
4567
+ was read (`read_all`)."""
4568
+
4569
+ def files_matching(globs: list[str]) -> set[str]:
4570
+ return {f for f in tree if (not globs or any(glob_match(f, g) for g in globs))
4571
+ and not any(glob_match(f, g) for g in excludes)}
4572
+
4573
+ in_include = files_matching(includes)
4574
+ denominator_files = (files_matching(tier_a) & in_include) if tier_a else in_include
4575
+ basis = "tier_a_include" if tier_a else ("scope.include" if includes else "all-files")
4576
+ warning = None
4577
+ if tier_a and counted and read_all and not any(file in denominator_files for _, _, file in counted):
4578
+ basis = "scope.include" if includes else "all-files"
4579
+ denominator_files = in_include
4580
+ counts_what = "scope.include" if includes else "every file"
4581
+ warning = f"scope.tier_a_include counts no public name: effective_denominator counts {counts_what}"
4582
+ # Each name once, as Test Skill's sets of names count it: a name the entry points pass on
4583
+ # from two files, or in two languages, is two keys but one name.
4584
+ count = len({name for _, name, file in counted if file in denominator_files})
4585
+ return count, basis, denominator_files, warning
4586
+
4587
+
4588
+ def curated_subset(counted: set[tuple[str, str, str | None]], basis: str, tree: list[str], includes: list[str],
4589
+ excludes: list[str], tier_a: list[str], glob_match: Callable[[str, str], bool],
4590
+ arms: dict, workspace_sources: list[set[str]]) -> bool:
4591
+ """Whether the files the count read are a curated subset rather than the
4592
+ whole workspace (in a single-package repository, the whole package): the
4593
+ files the basis globs list (a tier A list's, among those scope.include
4594
+ lists; under arm (b), less those scope.exclude drops) leave out a file
4595
+ that defines a counted public name or every source file of one of
4596
+ `workspace_sources` (the workspaces found over the whole tree,
4597
+ scope.exclude aside, so excluding a member the include globs leave out
4598
+ anyway changes nothing). A basis of all-files lists none."""
4599
+ if basis == "all-files":
4600
+ return False
4601
+ files = {f for f in tree if not includes or any(glob_match(f, g) for g in includes)}
4602
+ if basis == "tier_a_include":
4603
+ files = {f for f in files if any(glob_match(f, g) for g in tier_a)}
4604
+ if arms.get("specific_modules"):
4605
+ files = {f for f in files if not any(glob_match(f, g) for g in excludes)}
4606
+ tree_set = set(tree)
4607
+ defining = {file for _, _, file in counted if file in tree_set}
4608
+ return bool(defining - files) or any(not (sources & files) for sources in workspace_sources)
4609
+
4610
+
4611
+ def effective_denominator_decision(arms: dict, scope_type: str | None, basis: str, curated: bool, count: int,
4612
+ incomplete: bool) -> dict:
4613
+ """Whether create-skill and update-skill store the count as
4614
+ stats.effective_denominator, the brief's scope.notes aside: {store, reason},
4615
+ the reason naming the first condition that fails."""
4616
+ if scope_type == "reference-app":
4617
+ reason = "reference-app"
4618
+ elif incomplete:
4619
+ reason = "incomplete-run"
4620
+ elif not any(arms.get(arm) for arm in DENOMINATOR_ARMS):
4621
+ reason = "no-arm"
4622
+ elif scope_type == "full-library":
4623
+ reason = "full-library"
4624
+ elif basis == "all-files":
4625
+ reason = "all-files"
4626
+ elif not curated:
4627
+ reason = "not-curated"
4628
+ elif count == 0:
4629
+ reason = "zero-count"
4630
+ else:
4631
+ return {"store": True, "reason": "curated-subset"}
4632
+ return {"store": False, "reason": reason}
4633
+
4634
+
3838
4635
  def _select_files(root: Path, candidates: list[str], listed: bool, includes: list[str], excludes: list[str],
3839
4636
  wanted: set[str], glob_match: Callable[[str, str], bool], issues: list[dict],
3840
- warnings: list[str]) -> tuple[list[str], dict[str, int]]:
3841
- """(the files in scope, by extension the in-scope source files no recipe
3842
- reads). A file a --files-from list names that is missing, or that no
3843
- recipe reads, is an issue; an include glob no candidate matches, a
3844
- warning."""
4637
+ warnings: list[str]) -> tuple[list[str], list[str], list[str]]:
4638
+ """(the files in scope, the in-scope source files no recipe reads, the
4639
+ include globs no candidate matches). A file a --files-from list names
4640
+ that is missing, or that no recipe reads, is an issue; an include glob
4641
+ no candidate matches, a warning too."""
3845
4642
  in_scope: list[str] = []
3846
- without_recipes: Counter = Counter()
4643
+ without_recipes: list[str] = []
3847
4644
  used = dict.fromkeys(includes, False)
3848
4645
  for rel in candidates:
3849
4646
  hits = [g for g in includes if glob_match(rel, g)]
@@ -3859,55 +4656,70 @@ def _select_files(root: Path, candidates: list[str], listed: bool, includes: lis
3859
4656
  if listed:
3860
4657
  issues.append({"file": rel, "issue": "no-recipes"})
3861
4658
  elif extension in NO_RECIPE_EXTENSIONS:
3862
- without_recipes[extension] += 1
4659
+ without_recipes.append(rel)
3863
4660
  continue
3864
4661
  in_scope.append(rel)
3865
- warnings += [f"scope.include pattern {g!r} matches no file" for g, hit in used.items() if not hit]
3866
- return in_scope, dict(sorted(without_recipes.items()))
4662
+ unmatched = [g for g, hit in used.items() if not hit]
4663
+ warnings += [f"scope.include pattern {g!r} matches no file" for g in unmatched]
4664
+ return in_scope, without_recipes, unmatched
3867
4665
 
3868
4666
 
3869
- def run_full(args: argparse.Namespace) -> tuple[dict, int]:
3870
- """Run the recipes over the source tree; (result, exit code)."""
3871
- clock = _Clock(DEFAULT_TIMEOUT_SEC if args.timeout is None else args.timeout)
3872
- root = Path(args.source_root)
3873
- if not root.is_dir():
3874
- raise RunnerError(f"source root not found: {args.source_root}")
3875
- recipes_path = Path(args.recipes) if args.recipes else RECIPES_FILE
3876
- recipes = load_recipe_file(recipes_path)
4667
+ def _by_extension(files: list[str]) -> dict[str, int]:
4668
+ """`files_without_recipes`: how many of `files` each extension has."""
4669
+ return dict(sorted(Counter(posixpath.splitext(f)[1].lower() for f in files).items()))
4670
+
4671
+
4672
+ def _unmatched_tier_a(tier_a: list[str], includes: list[str], tree: list[str],
4673
+ glob_match: Callable[[str, str], bool], warnings: list[str]) -> list[str]:
4674
+ """The tier A globs that match no file of `tree` (the whole tree, regardless
4675
+ of --files-from and before the excludes), with a warning for each, and one
4676
+ for each that matches only files outside scope.include."""
4677
+ unmatched = [g for g in dict.fromkeys(tier_a) if not any(glob_match(f, g) for f in tree)]
4678
+ warnings += [f"scope.tier_a_include pattern {g!r} matches no file" for g in unmatched]
4679
+ warnings += [f"scope.tier_a_include pattern {g!r} matches only files outside scope.include"
4680
+ for g in dict.fromkeys(tier_a) if includes and g not in unmatched
4681
+ and not any(glob_match(f, g) and any(glob_match(f, i) for i in includes) for f in tree)]
4682
+ return unmatched
4683
+
4684
+
4685
+ def _brief_scope(args: argparse.Namespace) -> dict:
4686
+ """The scope globs, scope type and languages: the flags', else the brief's."""
3877
4687
  brief = _read_brief(Path(args.brief)) if args.brief else {}
3878
- includes = [g for g in (_norm(v) for v in (args.include or brief.get("include") or [])) if g]
3879
- excludes = [g for g in (_norm(v) for v in (args.exclude or brief.get("exclude") or [])) if g]
3880
- tier_a = [g for g in (_norm(v) for v in (args.tier_a_include or brief.get("tier_a_include") or [])) if g]
3881
- scope_type = args.scope_type or brief.get("type")
3882
4688
  language_values = args.language or ([brief["language"]] if brief.get("language") else [])
3883
4689
  wanted, unknown_languages = _wanted_languages(language_values)
3884
- recipe_set = args.recipe_set or ("component-library" if scope_type == "component-library" else "standard")
3885
- head_cap = _head_cap(args.head_cap, args.tier, scope_type)
3886
- glob_match = _sibling("skf-resolve-authoritative-files.py").glob_match
4690
+ return {
4691
+ "includes": [g for g in (_norm(v) for v in (args.include or brief.get("include") or [])) if g],
4692
+ "excludes": [g for g in (_norm(v) for v in (args.exclude or brief.get("exclude") or [])) if g],
4693
+ "tier_a": [g for g in (_norm(v) for v in (args.tier_a_include or brief.get("tier_a_include") or [])) if g],
4694
+ "scope_type": args.scope_type or brief.get("type"),
4695
+ "wanted": wanted,
4696
+ "glob_match": _sibling("skf-resolve-authoritative-files.py").glob_match,
4697
+ "warnings": [f"no recipe reads {value} files" for value in unknown_languages],
4698
+ }
3887
4699
 
3888
- warnings: list[str] = [f"no recipe reads {value} files" for value in unknown_languages]
3889
- errors: list[dict] = []
4700
+
4701
+ def _scope_facts(args: argparse.Namespace, root: Path) -> dict:
4702
+ """What full mode and denominator mode know before any recipe runs: the
4703
+ brief's scope, the files in scope, the entry points and the arms."""
4704
+ scope = _brief_scope(args)
4705
+ includes, excludes, tier_a = scope["includes"], scope["excludes"], scope["tier_a"]
4706
+ scope_type, wanted, glob_match = scope["scope_type"], scope["wanted"], scope["glob_match"]
4707
+ warnings: list[str] = scope["warnings"]
3890
4708
  issues: list[dict] = []
3891
4709
  tree = list_tree(root)
3892
4710
  tree_set = set(tree)
3893
- listed = _read_file_list(Path(args.files_from)) if args.files_from else None
3894
- in_scope, without_recipes = _select_files(root, tree if listed is None else listed, listed is not None,
3895
- includes, excludes, wanted, glob_match, issues, warnings)
3896
- readable = []
3897
- for rel in in_scope:
3898
- try:
3899
- (root / rel).read_bytes().decode("utf-8")
3900
- except OSError:
3901
- issues.append({"file": rel, "issue": "unreadable"})
3902
- continue
3903
- except UnicodeDecodeError:
3904
- issues.append({"file": rel, "issue": "not-utf8"})
3905
- continue
3906
- readable.append(rel)
4711
+ files_from = getattr(args, "files_from", None)
4712
+ listed = _read_file_list(Path(files_from)) if files_from else None
4713
+ in_scope, without_recipes, unmatched_include = _select_files(
4714
+ root, tree if listed is None else listed, listed is not None, includes, excludes, wanted, glob_match,
4715
+ issues, warnings)
4716
+ # A tier A glob is matched against the whole tree under --source-root (the
4717
+ # tree effective_denominator reads).
4718
+ unmatched_tier_a = _unmatched_tier_a(tier_a, includes, tree, glob_match, warnings)
3907
4719
 
3908
4720
  # The entry points and the arms need no ast-grep.
3909
4721
  in_scope_set = set(in_scope)
3910
- surfaces = {family: _Surface() for family in FAMILIES}
4722
+ surfaces = {family: _Surface(family, in_scope_set) for family in FAMILIES}
3911
4723
  js_entries, maps, unresolved_subpaths = _js_entries(root, in_scope, tree, tree_set, warnings)
3912
4724
  py_entries = _py_entries(in_scope, tree_set)
3913
4725
  rust_entries = _rust_entries(root, in_scope, tree, tree_set)
@@ -3922,13 +4734,85 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
3922
4734
  f"{entry['package']}) is outside the files in scope: widen scope.include to document "
3923
4735
  "the names it exports (entry_point_diff.outside_scope), which effective_denominator "
3924
4736
  "leaves out")
3925
- monorepo, monorepo_kind = _monorepo(root, tree, tree_set)
4737
+ monorepo, monorepo_kind = _monorepo(root, tree, excludes, glob_match)
3926
4738
  arms = {
3927
4739
  "monorepo": monorepo,
3928
4740
  "monorepo_kind": monorepo_kind,
3929
4741
  "specific_modules": scope_type == "specific-modules" and not monorepo,
3930
4742
  "multi_subpath_exports": not monorepo and any(m["non_root_subpaths"] > 1 for m in maps),
3931
4743
  }
4744
+ return {
4745
+ "includes": includes, "excludes": excludes, "tier_a": tier_a, "scope_type": scope_type, "wanted": wanted,
4746
+ "glob_match": glob_match, "warnings": warnings, "issues": issues, "tree": tree, "tree_set": tree_set,
4747
+ "in_scope": in_scope, "without_recipes": without_recipes, "unmatched_include": unmatched_include,
4748
+ "unmatched_tier_a": unmatched_tier_a, "surfaces": surfaces, "js_entries": js_entries, "maps": maps,
4749
+ "unresolved_subpaths": unresolved_subpaths, "py_entries": py_entries, "rust_entries": rust_entries,
4750
+ "entries": entries, "arms": arms,
4751
+ "workspace_sources": _workspace_sources(_workspaces(root, tree, tree_set), tree, wanted),
4752
+ }
4753
+
4754
+
4755
+ def _scope_record(args: argparse.Namespace, facts: dict) -> dict:
4756
+ """The `scope` object full mode and denominator mode write."""
4757
+ return {"include": facts["includes"], "exclude": facts["excludes"], "tier_a_include": facts["tier_a"] or None,
4758
+ "type": facts["scope_type"], "languages": sorted(facts["wanted"]),
4759
+ "files_from": getattr(args, "files_from", None), "unmatched_include": facts["unmatched_include"],
4760
+ "unmatched_tier_a_include": facts["unmatched_tier_a"]}
4761
+
4762
+
4763
+ def _denominator_counts(facts: dict, counted: set[tuple[str, str, str | None]], public_api: int, internal: int,
4764
+ incomplete: bool, warnings: list[str]) -> dict:
4765
+ """The `counts` object: the two API counts, effective_denominator with its
4766
+ basis and files, whether those files are a curated subset, and the
4767
+ decision whether to store the count."""
4768
+ count, basis, denominator_files, warning = denominator_count(
4769
+ counted, facts["tree"], facts["includes"], facts["excludes"], facts["tier_a"], facts["glob_match"],
4770
+ not incomplete)
4771
+ if warning:
4772
+ warnings.append(warning)
4773
+ curated = curated_subset(counted, basis, facts["tree"], facts["includes"], facts["excludes"], facts["tier_a"],
4774
+ facts["glob_match"], facts["arms"], facts["workspace_sources"])
4775
+ return {
4776
+ "exports_public_api": public_api,
4777
+ "exports_internal": internal,
4778
+ "effective_denominator": count,
4779
+ "effective_denominator_basis": basis,
4780
+ "denominator_files": len(denominator_files),
4781
+ "curated_subset": curated,
4782
+ "effective_denominator_decision": effective_denominator_decision(
4783
+ facts["arms"], facts["scope_type"], basis, curated, count, incomplete),
4784
+ }
4785
+
4786
+
4787
+ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
4788
+ """Run the recipes over the source tree; (result, exit code)."""
4789
+ clock = _Clock(DEFAULT_TIMEOUT_SEC if args.timeout is None else args.timeout)
4790
+ root = Path(args.source_root)
4791
+ if not root.is_dir():
4792
+ raise RunnerError(f"source root not found: {args.source_root}")
4793
+ recipes_path = Path(args.recipes) if args.recipes else RECIPES_FILE
4794
+ recipes = load_recipe_file(recipes_path)
4795
+ facts = _scope_facts(args, root)
4796
+ scope_type, warnings, issues = facts["scope_type"], facts["warnings"], facts["issues"]
4797
+ tree_set, in_scope = facts["tree_set"], facts["in_scope"]
4798
+ in_scope_set = set(in_scope)
4799
+ surfaces, js_entries, maps = facts["surfaces"], facts["js_entries"], facts["maps"]
4800
+ py_entries, rust_entries, entries = facts["py_entries"], facts["rust_entries"], facts["entries"]
4801
+ unresolved_subpaths, arms = facts["unresolved_subpaths"], facts["arms"]
4802
+ recipe_set = args.recipe_set or ("component-library" if scope_type == "component-library" else "standard")
4803
+ head_cap = _head_cap(args.head_cap, args.tier, scope_type)
4804
+ errors: list[dict] = []
4805
+ readable = []
4806
+ for rel in in_scope:
4807
+ try:
4808
+ (root / rel).read_bytes().decode("utf-8")
4809
+ except OSError:
4810
+ issues.append({"file": rel, "issue": "unreadable"})
4811
+ continue
4812
+ except UnicodeDecodeError:
4813
+ issues.append({"file": rel, "issue": "not-utf8"})
4814
+ continue
4815
+ readable.append(rel)
3932
4816
 
3933
4817
  exe, version, shim = _ast_grep()
3934
4818
  result: dict = {
@@ -3938,12 +4822,10 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
3938
4822
  "recipes_file": recipes_path.as_posix(),
3939
4823
  "recipe_set": recipe_set,
3940
4824
  "ast_grep": {"path": exe, "version": version},
3941
- "scope": {"include": includes, "exclude": excludes, "tier_a_include": tier_a or None,
3942
- "type": scope_type, "languages": sorted(wanted),
3943
- "files_from": args.files_from},
4825
+ "scope": _scope_record(args, facts),
3944
4826
  "files_in_scope": len(in_scope),
3945
4827
  "files_by_language": dict(sorted(Counter(_file_language(f) for f in in_scope).items())),
3946
- "files_without_recipes": without_recipes,
4828
+ "files_without_recipes": _by_extension(facts["without_recipes"]),
3947
4829
  "file_issues": issues,
3948
4830
  "head_cap": head_cap,
3949
4831
  "truncated": False,
@@ -3958,6 +4840,9 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
3958
4840
  "arms": arms,
3959
4841
  "errors": errors,
3960
4842
  "warnings": warnings,
4843
+ # Last: only a run without the recipes reads them, and a large tree lists thousands.
4844
+ "files_in_scope_list": in_scope,
4845
+ "files_without_recipes_list": facts["without_recipes"],
3961
4846
  }
3962
4847
  if exe is None:
3963
4848
  result["status"] = "no-ast-grep"
@@ -4007,16 +4892,20 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
4007
4892
  for record in exports:
4008
4893
  if record["from"] and record["language"] in JS_FAMILY:
4009
4894
  record["from_file"] = resolve_js_module(record["from"], record["source_file"], tree_set)
4010
- statuses, public, internal, gaps, outside = entry_point_diff(exports, surfaces, entries, in_scope_set)
4895
+ statuses, public, internal, gaps, outside, others = entry_point_diff(exports, surfaces, entries, in_scope_set)
4011
4896
  if "barrel" in statuses.values():
4012
4897
  status = "barrel"
4013
4898
  else:
4014
4899
  status = "empty-barrel" if "empty-barrel" in statuses.values() else "no-entry-point"
4015
4900
 
4016
- basis_globs = tier_a or includes
4017
- basis = "tier_a_include" if tier_a else ("scope.include" if includes else "all-files")
4018
- denominator_files = {f for f in tree if (not basis_globs or any(glob_match(f, g) for g in basis_globs))
4019
- and not any(glob_match(f, g) for g in excludes)}
4901
+ # A tier A list that counts no public name (each glob matches no file, or
4902
+ # only files an exclude drops, that scope.include leaves out or that
4903
+ # define no public name) narrows nothing, once the run counted a name
4904
+ # and read every file (no error).
4905
+ # Each other in-scope definition of a public name counts its name where any of its files is counted (#830).
4906
+ public_keys = _public_keys(public)
4907
+ counted = public_keys | _other_keys(others)
4908
+ counts = _denominator_counts(facts, counted, len(public_keys), len(internal), bool(errors), warnings)
4020
4909
 
4021
4910
  by_type: Counter = Counter(e["export_type"] for e in exports)
4022
4911
  ordered = sorted(public.items(), key=lambda item: (item[0][1], item[0][0]))
@@ -4030,6 +4919,8 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
4030
4919
  "unresolved": unresolved_subpaths + [u for s in surfaces.values() for u in s.unresolved]},
4031
4920
  "entry_point_diff": {
4032
4921
  "public": [dict(info, language=family) for (family, _), info in ordered],
4922
+ "other_definitions": [dict(info, language=family) for family, info in
4923
+ sorted(others, key=lambda o: (o[1]["name"], o[1]["file"], o[0]))],
4033
4924
  "internal": [{"name": e["export_name"], "language": FAMILY_OF[e["language"]],
4034
4925
  "source_file": e["source_file"], "source_line": e["source_line"]} for e in internal],
4035
4926
  "extraction_gaps": [{"name": info["name"], "language": family, "entry": info["entry"],
@@ -4043,13 +4934,7 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
4043
4934
  {"name": info["name"], "language": family, "local": info["local"], "from": info["from"],
4044
4935
  "entry": info["entry"], "file": info["file"], "line": info["line"], "via": info["via"]}
4045
4936
  for (family, _), info in ordered if info["via"] in ("re-export", "star", "namespace")],
4046
- "counts": {
4047
- "exports_public_api": len(public),
4048
- "exports_internal": len(internal),
4049
- "effective_denominator": sum(1 for info in public.values() if info["file"] in denominator_files),
4050
- "effective_denominator_basis": basis,
4051
- "denominator_files": len(denominator_files),
4052
- },
4937
+ "counts": counts,
4053
4938
  })
4054
4939
  result["file_issues"] = sorted(issues, key=lambda i: (i["file"], i["issue"]))
4055
4940
  if errors:
@@ -4058,37 +4943,273 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
4058
4943
  return result, 0
4059
4944
 
4060
4945
 
4946
+ def _full_exit_2(message: str) -> int:
4947
+ """Exit 2 of full mode (an input error, a failed -o write or a crash):
4948
+ `message` on one stderr line, each line break and the whitespace around
4949
+ it joined into one space (a YAML error spans several lines) and every
4950
+ other space kept, so a path is quoted as written. Callers quote the line
4951
+ as the reason they stop or fall back."""
4952
+ sys.stderr.write("error: " + re.sub(r"\s*[\r\n]\s*", " ", message.strip("\r\n")) + "\n")
4953
+ return 2
4954
+
4955
+
4061
4956
  def _main_full(args: argparse.Namespace) -> int:
4062
4957
  if not args.source_root:
4063
- sys.stderr.write("error: --mode full needs --source-root\n")
4064
- return 2
4958
+ return _full_exit_2("--mode full needs --source-root")
4065
4959
  if args.head_cap is not None and args.head_cap < 0:
4066
- sys.stderr.write("error: --head-cap must be 0 (no cap) or more\n")
4067
- return 2
4960
+ return _full_exit_2("--head-cap must be 0 (no cap) or more")
4068
4961
  if args.timeout is not None and not args.timeout > 0:
4069
- sys.stderr.write("error: --timeout must be more than 0 seconds\n")
4070
- return 2
4962
+ return _full_exit_2("--timeout must be more than 0 seconds")
4071
4963
  try:
4072
4964
  result, code = run_full(args)
4073
4965
  except RunnerError as exc:
4074
- sys.stderr.write(f"error: {exc}\n")
4075
- return 2
4966
+ return _full_exit_2(str(exc))
4076
4967
  except Exception as exc: # noqa: BLE001 - one stderr line and exit 2, never a traceback
4077
- detail = " ".join(str(exc).split())
4078
- sys.stderr.write(f"error: the run failed: {type(exc).__name__}: {detail}\n")
4079
- return 2
4968
+ return _full_exit_2(f"the run failed: {type(exc).__name__}: {exc}")
4080
4969
  text = json.dumps(result, indent=2) + "\n"
4081
4970
  if args.output:
4082
4971
  try:
4083
4972
  Path(args.output).write_text(text, encoding="utf-8")
4084
4973
  except OSError as exc:
4085
- sys.stderr.write(f"error: cannot write {args.output}: {exc}\n")
4086
- return 2
4974
+ return _full_exit_2(f"cannot write {args.output}: {exc}")
4087
4975
  else:
4088
4976
  sys.stdout.write(text)
4089
4977
  return code
4090
4978
 
4091
4979
 
4980
+ # --------------------------------------------------------------------------
4981
+ # Denominator mode: the counts and the effective_denominator decision of
4982
+ # names counted without the runner's diff
4983
+ # --------------------------------------------------------------------------
4984
+
4985
+ NO_ARMS = {"monorepo": False, "monorepo_kind": None, "specific_modules": False, "multi_subpath_exports": False}
4986
+
4987
+
4988
+ class DenominatorInputError(Exception):
4989
+ """Exit 2 of denominator mode."""
4990
+
4991
+
4992
+ def _name_family(language: object, file: object) -> str:
4993
+ """The language family a name counts in: its file's, else its language's."""
4994
+ family = FAMILY_OF.get(_file_language(file)) if isinstance(file, str) and file else None
4995
+ if family is not None:
4996
+ return family
4997
+ value = str(language or "").strip().lower()
4998
+ selected = LANGUAGE_FAMILIES.get(value)
4999
+ return FAMILY_OF[selected[0]] if selected else value
5000
+
5001
+
5002
+ def _payload_names(items: object, where: str) -> list[dict]:
5003
+ """The {name, file, language, members} items of a payload list, checked."""
5004
+ if items is None:
5005
+ return []
5006
+ if not isinstance(items, list):
5007
+ raise DenominatorInputError(f"{where} is not a list")
5008
+ names = []
5009
+ for i, item in enumerate(items):
5010
+ if not isinstance(item, dict) or not isinstance(item.get("name"), str) or not item["name"].strip():
5011
+ raise DenominatorInputError(f"{where}[{i}] is not an object with a name")
5012
+ if item.get("file") is not None and not isinstance(item["file"], str):
5013
+ raise DenominatorInputError(f"{where}[{i}].file is not a path")
5014
+ members = item.get("members")
5015
+ if members is not None:
5016
+ if not isinstance(members, list):
5017
+ raise DenominatorInputError(f"{where}[{i}].members is not a list")
5018
+ members = _payload_names(members, f"{where}[{i}].members")
5019
+ file = _norm(item["file"]) if item.get("file") else None
5020
+ names.append({"name": item["name"].strip(), "file": file, "language": item.get("language"),
5021
+ "members": members})
5022
+ return names
5023
+
5024
+
5025
+ def _read_json_file(path: str, flag: str) -> dict:
5026
+ try:
5027
+ data = json.loads(Path(path).read_text(encoding="utf-8"))
5028
+ except (OSError, UnicodeDecodeError, ValueError) as exc:
5029
+ raise DenominatorInputError(f"cannot read {flag} {path}: {exc}") from exc
5030
+ if not isinstance(data, dict):
5031
+ raise DenominatorInputError(f"{flag} {path} is not a JSON object")
5032
+ return data
5033
+
5034
+
5035
+ def run_denominator(args: argparse.Namespace, payload: dict) -> dict:
5036
+ """The counts of the public names `payload`, --quick and --extraction
5037
+ give, with the effective_denominator decision full mode makes."""
5038
+ public: dict[tuple[str, str], dict] = {}
5039
+ runner_public: dict[tuple[str, str], dict] = {}
5040
+ # The exports found, each key at its file: the payload's and the runner's.
5041
+ found: dict[tuple[str, str], str | None] = {}
5042
+ internal: set[tuple[str, str]] = set()
5043
+ # The runner's other in-scope definitions of its public names (#830), counted where the names are.
5044
+ others: list[tuple[str, dict]] = []
5045
+ incomplete = False
5046
+ runner_status = None
5047
+ if args.extraction:
5048
+ runner = _read_json_file(args.extraction, "--extraction")
5049
+ if runner.get("mode") != "full":
5050
+ raise DenominatorInputError(f"--extraction {args.extraction} is not --mode full output")
5051
+ runner_status = runner.get("status")
5052
+ incomplete = runner_status == "incomplete" or bool(runner.get("errors"))
5053
+ diff = runner.get("entry_point_diff") if isinstance(runner.get("entry_point_diff"), dict) else {}
5054
+ for info in diff.get("public") or []:
5055
+ if isinstance(info, dict) and isinstance(info.get("name"), str):
5056
+ key = (str(info.get("language") or ""), info["name"])
5057
+ public[key] = runner_public[key] = info
5058
+ found[key] = info.get("file")
5059
+ others = [(str(info.get("language") or ""), info) for info in diff.get("other_definitions") or []
5060
+ if isinstance(info, dict)]
5061
+ for item in diff.get("internal") or []:
5062
+ if isinstance(item, dict) and isinstance(item.get("name"), str):
5063
+ internal.add((str(item.get("language") or ""), item["name"]))
5064
+ found[(str(item.get("language") or ""), item["name"])] = item.get("source_file")
5065
+ for path in args.quick or []:
5066
+ quick = _read_json_file(path, "--quick")
5067
+ if not isinstance(quick.get("exports"), list):
5068
+ raise DenominatorInputError(f"--quick {path} is not --mode quick output (no exports list)")
5069
+ for item in quick["exports"]:
5070
+ if isinstance(item, dict) and isinstance(item.get("name"), str) and item["name"]:
5071
+ file = item.get("defined_in") or item.get("source_file")
5072
+ file = _norm(file) if isinstance(file, str) and file else None
5073
+ public[(_name_family(quick.get("language"), file), item["name"])] = {
5074
+ "name": item["name"], "file": file, "via": "declaration"}
5075
+ for item in _payload_names(payload.get("public"), "public"):
5076
+ family = _name_family(item["language"], item["file"])
5077
+ info = {"name": item["name"], "file": item["file"], "via": "declaration"}
5078
+ if item["members"] is not None:
5079
+ info.update(via="namespace", members=[{"name": m["name"], "file": m["file"]} for m in item["members"]])
5080
+ public[(family, item["name"])] = info
5081
+ for item in _payload_names(payload.get("exports"), "exports"):
5082
+ key = (_name_family(item["language"], item["file"]), item["name"])
5083
+ found.setdefault(key, item["file"])
5084
+ if not _private(key[0], item["name"]):
5085
+ internal.add(key)
5086
+ named = set(public) | {(family, m["name"]) for (family, _), info in public.items() if _has_members(info)
5087
+ for m in info["members"]}
5088
+ public_keys = _public_keys(public)
5089
+ def still_held(key: tuple[str, str]) -> bool:
5090
+ """Whether the record kept for the runner's public name `key` still names the runner row's file, a
5091
+ namespace with members in neither (full mode keeps no other definition beside one)."""
5092
+ runner_row, kept = runner_public.get(key), public.get(key)
5093
+ if runner_row is None or kept is None or _has_members(runner_row) or _has_members(kept):
5094
+ return False
5095
+ files = [_norm(row["file"]) if isinstance(row.get("file"), str) and row["file"] else None
5096
+ for row in (runner_row, kept)]
5097
+ return files[0] == files[1]
5098
+
5099
+ # A runner row counts only while the record kept for its name names the runner's file: an orphan row, or
5100
+ # one of a name a payload or --quick record moved to another file, adds nothing.
5101
+ others = [(family, info) for family, info in others
5102
+ if isinstance(info.get("name"), str) and still_held((family, info["name"]))]
5103
+ other_keys = _other_keys(others)
5104
+ counted = public_keys | other_keys
5105
+
5106
+ if args.source_root:
5107
+ root = Path(args.source_root)
5108
+ if not root.is_dir():
5109
+ raise RunnerError(f"source root not found: {args.source_root}")
5110
+ facts = _scope_facts(args, root)
5111
+ else:
5112
+ # No local tree (a remote source read by eye): the repository's listing, else the names' files.
5113
+ facts = _brief_scope(args)
5114
+ in_scope = without_recipes = None
5115
+ unmatched_include: list[str] = []
5116
+ unmatched_tier_a: list[str] = []
5117
+ if args.tree_file is not None:
5118
+ try:
5119
+ listed, cut = _read_listing(args.tree_file)
5120
+ except QuickInputError as exc:
5121
+ raise DenominatorInputError(str(exc)) from exc
5122
+ tree = sorted(set(listed))
5123
+ if cut:
5124
+ facts["warnings"].append("the listing was cut short: the file lists can miss files")
5125
+ in_scope, without_recipes, unmatched_include = _select_files(
5126
+ Path("."), tree, False, facts["includes"], facts["excludes"], facts["wanted"], facts["glob_match"],
5127
+ [], facts["warnings"])
5128
+ unmatched_tier_a = _unmatched_tier_a(facts["tier_a"], facts["includes"], tree, facts["glob_match"],
5129
+ facts["warnings"])
5130
+ else:
5131
+ tree = sorted({file for _, _, file in counted if file})
5132
+ facts.update(tree=tree, in_scope=in_scope, without_recipes=without_recipes,
5133
+ unmatched_include=unmatched_include, unmatched_tier_a=unmatched_tier_a, arms=dict(NO_ARMS),
5134
+ workspace_sources=[], entries=[], maps=[])
5135
+ warnings = facts["warnings"]
5136
+ tree_set = set(facts["tree"])
5137
+ outside_tree = sorted((name, file) for _, name, file in counted if file and file not in tree_set)
5138
+ warnings += [f"public name {name} is defined in {file}, which is not in the tree: effective_denominator "
5139
+ "and curated_subset leave it out" for name, file in outside_tree]
5140
+ counts = _denominator_counts(facts, counted, len(public_keys), len(internal - named), incomplete, warnings)
5141
+ if not args.source_root and counts["effective_denominator_decision"]["reason"] not in ("reference-app",
5142
+ "incomplete-run"):
5143
+ counts["effective_denominator_decision"] = {"store": False, "reason": "no-source-tree"}
5144
+ without_recipes = facts["without_recipes"]
5145
+
5146
+ def row(key: tuple[str, str], file: str | None) -> dict:
5147
+ return {"name": key[1], "language": key[0], "file": file}
5148
+
5149
+ def by_name(keys):
5150
+ return sorted(keys, key=lambda k: (k[1], k[0]))
5151
+
5152
+ # The names the payload and --quick gave (the runner diffed its own), a namespace as its members.
5153
+ given: dict[tuple[str, str], str | None] = {}
5154
+ for key, info in public.items():
5155
+ if info is runner_public.get(key):
5156
+ continue
5157
+ for member in info["members"] if _has_members(info) else [info]:
5158
+ given.setdefault((key[0], member["name"]), member["file"])
5159
+ return {
5160
+ "mode": "denominator",
5161
+ "source_root": args.source_root,
5162
+ "scope": _scope_record(args, facts),
5163
+ "files_in_scope": None if facts["in_scope"] is None else len(facts["in_scope"]),
5164
+ "files_without_recipes": None if without_recipes is None else _by_extension(without_recipes),
5165
+ "entry_points": {"files": facts["entries"], "exports_maps": facts["maps"]},
5166
+ "arms": facts["arms"],
5167
+ "runner_status": runner_status,
5168
+ "counts": counts,
5169
+ "entry_point_diff": {
5170
+ "public": [row((family, name), file)
5171
+ for family, name, file in sorted(public_keys, key=lambda k: (k[1], k[0], k[2] or ""))],
5172
+ "other_definitions": [row((family, name), file)
5173
+ for family, name, file in sorted(other_keys, key=lambda k: (k[1], k[2], k[0]))],
5174
+ "internal": [row(key, found.get(key)) for key in by_name(internal - named)],
5175
+ "extraction_gaps": [row(key, given[key]) for key in by_name(given) if key not in found],
5176
+ },
5177
+ "warnings": warnings,
5178
+ "files_in_scope_list": facts["in_scope"],
5179
+ "files_without_recipes_list": without_recipes,
5180
+ }
5181
+
5182
+
5183
+ def _main_denominator(args: argparse.Namespace) -> int:
5184
+ if args.source_root and args.tree_file is not None:
5185
+ return _full_exit_2("--tree-file stands for a remote source with no --source-root: pass one of them")
5186
+ # With `--tree-file -` stdin holds the listing, so there is no payload.
5187
+ reads_stdin = args.tree_file != "-" and sys.stdin is not None and not sys.stdin.isatty()
5188
+ raw = sys.stdin.read() if reads_stdin else ""
5189
+ try:
5190
+ payload = json.loads(raw) if raw.strip() else {}
5191
+ except ValueError as exc:
5192
+ return _full_exit_2(f"stdin is not JSON: {exc}")
5193
+ if not isinstance(payload, dict):
5194
+ return _full_exit_2("stdin must be a JSON object: {\"public\": [...], \"exports\": [...]}")
5195
+ try:
5196
+ result = run_denominator(args, payload)
5197
+ except (RunnerError, DenominatorInputError) as exc:
5198
+ return _full_exit_2(str(exc))
5199
+ except Exception as exc: # noqa: BLE001 - one stderr line and exit 2, never a traceback
5200
+ return _full_exit_2(f"the run failed: {type(exc).__name__}: {exc}")
5201
+ text = json.dumps(result, indent=2) + "\n"
5202
+ if args.output:
5203
+ try:
5204
+ Path(args.output).parent.mkdir(parents=True, exist_ok=True)
5205
+ Path(args.output).write_text(text, encoding="utf-8")
5206
+ except OSError as exc:
5207
+ return _full_exit_2(f"cannot write {args.output}: {exc}")
5208
+ else:
5209
+ sys.stdout.write(text)
5210
+ return 0
5211
+
5212
+
4092
5213
  # --------------------------------------------------------------------------
4093
5214
  # Entries mode: the files quick mode reads, found in the listing
4094
5215
  # --------------------------------------------------------------------------
@@ -4281,10 +5402,11 @@ def _build_parser() -> argparse.ArgumentParser:
4281
5402
  parser.add_argument(
4282
5403
  "--mode",
4283
5404
  default="quick",
4284
- choices=("quick", "full", "entries"),
5405
+ choices=("quick", "full", "entries", "denominator"),
4285
5406
  help="quick (default): parse the stdin payload, or the --manifest-file and --entry-file files; "
4286
5407
  "full: run the recipes over --source-root; entries: list one package's entry files from "
4287
- "--tree-file.",
5408
+ "--tree-file; denominator: count the public names a stdin payload, --quick and --extraction give, "
5409
+ "and decide whether to store effective_denominator, as full mode does.",
4288
5410
  )
4289
5411
  quick = parser.add_argument_group("quick mode, files read from disk")
4290
5412
  quick.add_argument("--manifest-file", metavar="PATH",
@@ -4298,16 +5420,23 @@ def _build_parser() -> argparse.ArgumentParser:
4298
5420
  help=f"read each module an unlisted[] record names as one more --follow-file, round "
4299
5421
  f"after round (at most {FOLLOW_ROUNDS}), until none is left; needs --tree-file and "
4300
5422
  f"--source-root")
5423
+ quick.add_argument("--resolve-version", action="store_true",
5424
+ help="look up a version the manifest does not state: a version file, a Cargo workspace "
5425
+ "root, then a release tag (of --repo at --ref, else git describe in --source-root); "
5426
+ "needs --source-root")
4301
5427
  quick.add_argument("--repo", metavar="OWNER/REPO",
4302
5428
  help="with --follow: fetch each module from this GitHub repository into --source-root "
4303
- "first (skf-github-fetch.py)")
5429
+ "first (skf-github-fetch.py); with --resolve-version: the repository whose tags "
5430
+ "it reads, and whose files it fetches with --tree-file")
4304
5431
  quick.add_argument("--ref", metavar="REF",
4305
- help="with --repo: the branch, tag or commit to fetch at (default: the default branch)")
5432
+ help="with --repo: the branch, tag or commit to fetch and read the version at "
5433
+ "(default: the default branch)")
4306
5434
  listing = parser.add_argument_group("entries mode, and the listing quick mode follows chains in")
4307
5435
  listing.add_argument("--tree-file", metavar="PATH",
4308
5436
  help="the repository's file listing (skf-github-probe.py tree output, JSON, or one "
4309
- "path per line): entries mode finds the entry files in it, and quick mode "
4310
- "resolves each unlisted[] statement's module_file against it")
5437
+ "path per line): entries mode finds the entry files in it, quick mode "
5438
+ "resolves each unlisted[] statement's module_file against it, and denominator "
5439
+ "mode, with no --source-root, takes it as the tree (- reads it from stdin)")
4311
5440
  listing.add_argument("--scope", metavar="FOLDER",
4312
5441
  help="entries mode: the package folder to read (default: the repository root)")
4313
5442
  listing.add_argument("--fetch-list", metavar="FILE",
@@ -4322,7 +5451,11 @@ def _build_parser() -> argparse.ArgumentParser:
4322
5451
  full.add_argument("--exclude", action="append", metavar="GLOB",
4323
5452
  help="an exclude glob, repeatable (replaces the brief's scope.exclude)")
4324
5453
  full.add_argument("--tier-a-include", action="append", metavar="GLOB",
4325
- help="a tier-A glob for effective_denominator, repeatable (replaces scope.tier_a_include)")
5454
+ help="a tier-A glob for effective_denominator, repeatable (replaces scope.tier_a_include), "
5455
+ "that counts a name only in a file scope.include matches too; one that matches no "
5456
+ "file is listed in scope.unmatched_tier_a_include, one that matches only files "
5457
+ "scope.include leaves out is named in warnings, and a list that counts no public name "
5458
+ "gives way to scope.include (else every file)")
4326
5459
  full.add_argument("--scope-type", help="the scope type (replaces the brief's scope.type)")
4327
5460
  full.add_argument("--language", action="append",
4328
5461
  help="a language family to extract, repeatable (replaces the brief's language); quick "
@@ -4339,6 +5472,14 @@ def _build_parser() -> argparse.ArgumentParser:
4339
5472
  help=f"stop the ast-grep runs after this many seconds and still write the JSON, "
4340
5473
  f"status incomplete (default {DEFAULT_TIMEOUT_SEC})")
4341
5474
  full.add_argument("-o", "--output", metavar="FILE", help="write the JSON here instead of stdout")
5475
+ denominator = parser.add_argument_group("denominator mode (with --source-root, --brief, the scope flags, "
5476
+ "--language and -o)")
5477
+ denominator.add_argument("--quick", action="append", metavar="FILE",
5478
+ help="a --mode quick output whose exports are public names, each at its defined_in "
5479
+ "(else its source_file), repeatable")
5480
+ denominator.add_argument("--extraction", metavar="FILE",
5481
+ help="a --mode full output whose entry_point_diff public and internal names count "
5482
+ "too; an incomplete one is never stored")
4342
5483
  return parser
4343
5484
 
4344
5485
 
@@ -4346,10 +5487,14 @@ FULL_MODE_FLAGS = ("source_root", "brief", "include", "exclude", "tier_a_include
4346
5487
  "files_from", "recipe_set", "recipes", "tier", "head_cap", "timeout", "output")
4347
5488
  # The full mode flags quick mode takes with its file inputs, and entries mode always.
4348
5489
  QUICK_FILE_FLAGS = ("source_root", "language")
5490
+ # The full mode flags denominator mode takes, and its own.
5491
+ DENOMINATOR_FLAGS = ("source_root", "brief", "include", "exclude", "tier_a_include", "scope_type", "language",
5492
+ "output")
5493
+ DENOMINATOR_OWN_FLAGS = ("quick", "extraction")
4349
5494
  # The flags of entries mode; quick mode takes --tree-file and --fetch-list with its file inputs.
4350
5495
  LISTING_FLAGS = ("tree_file", "scope", "fetch_list")
4351
5496
  # Quick mode's own flags with its file inputs: the chains it follows itself.
4352
- FOLLOW_FLAGS = ("follow", "repo", "ref")
5497
+ FOLLOW_FLAGS = ("follow", "repo", "ref", "resolve_version")
4353
5498
  FOLLOW_ROUNDS = 5
4354
5499
 
4355
5500
 
@@ -4479,6 +5624,327 @@ def _follow(args: argparse.Namespace, result: dict) -> dict:
4479
5624
  return result
4480
5625
 
4481
5626
 
5627
+ # --resolve-version: a version the manifest does not state
5628
+ _PY_VERSION_RE = re.compile(r"""^__version__[ \t]*(?::[^=\n]*)?=[ \t]*(['"])([^'"\n]+)\1""", re.M)
5629
+ _JS_PRERELEASE_RE = re.compile(r"-(?:alpha|beta|rc|next|canary)", re.I)
5630
+ JS_FAMILY = ("js", "ts", "javascript", "typescript")
5631
+
5632
+
5633
+ def _version_reader(args: argparse.Namespace, tree: list[str] | None) -> Callable[[str], str | None]:
5634
+ """path -> the text of that file under --source-root, else None. With
5635
+ --repo and --tree-file, a file the listing holds and the folder does not
5636
+ is fetched there first, as --follow fetches a module. Raises
5637
+ QuickInputError."""
5638
+ root = Path(args.source_root)
5639
+ listed = set(tree or [])
5640
+ fetch: list = []
5641
+
5642
+ def read(path: str) -> str | None:
5643
+ target = root / path
5644
+ if not target.is_file() and args.repo is not None and args.tree_file is not None and path in listed:
5645
+ if not fetch:
5646
+ fetch.append(_follow_reader(args))
5647
+ fetch[0]([path])
5648
+ try:
5649
+ if target.is_file() and target.resolve().is_relative_to(root.resolve()):
5650
+ return target.read_bytes().decode("utf-8-sig", errors="replace")
5651
+ except OSError:
5652
+ pass
5653
+ return None
5654
+ return read
5655
+
5656
+
5657
+ def _version_value(value: object) -> str | None:
5658
+ """A version a version file writes: a literal that starts with a digit, not a placeholder."""
5659
+ if not isinstance(value, str) or not re.match(r"\d", value.strip()):
5660
+ return None
5661
+ return None if _detect_placeholder_version(value) else value.strip()
5662
+
5663
+
5664
+ def _python_version_file(manifest: str | None, entries: list[str],
5665
+ read: Callable[[str], str | None]) -> tuple[str, str] | None:
5666
+ """(version, path): the `__version__` literal of an entry __init__.py,
5667
+ then of the _version.py beside it, then setup.py's `version=` literal
5668
+ beside the manifest."""
5669
+ inits = [e for e in entries if e.rsplit("/", 1)[-1] == "__init__.py"]
5670
+ folder = posixpath.dirname(manifest or "")
5671
+ candidates = [(path, "py") for path in inits]
5672
+ candidates += [(posixpath.join(posixpath.dirname(path), "_version.py"), "py") for path in inits]
5673
+ if posixpath.basename(manifest or "") != "setup.py":
5674
+ candidates.append((posixpath.join(folder, "setup.py"), "setup.py"))
5675
+ for path, kind in dict.fromkeys(candidates):
5676
+ text = read(path)
5677
+ if text is None:
5678
+ continue
5679
+ if kind == "py":
5680
+ m = _PY_VERSION_RE.search(text)
5681
+ version = _version_value(m.group(2)) if m else None
5682
+ else:
5683
+ version = _version_value(parse_setup_py(text).get("version"))
5684
+ if version:
5685
+ return version, path
5686
+ return None
5687
+
5688
+
5689
+ def _cargo_workspace_version(manifest: str | None, read: Callable[[str], str | None]) -> tuple[str, str] | None:
5690
+ """(version, path): the `[workspace.package] version` of the workspace
5691
+ root, the nearest Cargo.toml at or above the manifest's folder that
5692
+ holds a `[workspace]` table."""
5693
+ if not manifest:
5694
+ return None
5695
+ folder = posixpath.dirname(manifest)
5696
+ while True:
5697
+ path = posixpath.join(folder, "Cargo.toml")
5698
+ text = read(path)
5699
+ try:
5700
+ data = tomllib.loads(text) if text is not None else None
5701
+ except tomllib.TOMLDecodeError:
5702
+ data = None
5703
+ workspace = data.get("workspace") if isinstance(data, dict) else None
5704
+ if isinstance(workspace, dict):
5705
+ package = workspace.get("package")
5706
+ version = _version_value(package.get("version")) if isinstance(package, dict) else None
5707
+ return (version, path) if version else None
5708
+ if not folder:
5709
+ return None
5710
+ folder = posixpath.dirname(folder)
5711
+
5712
+
5713
+ def _release_tag(args: argparse.Namespace, warnings: list[str]) -> tuple[str, str] | None:
5714
+ """(version, tag) of the tag a version is read from: with --repo, the
5715
+ `release` skf-github-probe.py's tag listing gives at --ref (the newest
5716
+ release tag at the default branch, the pinned tag itself at a tag);
5717
+ without, the tag `git describe --tags --abbrev=0` names in
5718
+ --source-root, when its name is a version. The version drops a
5719
+ leading v."""
5720
+ try:
5721
+ probe = _sibling("skf-github-probe.py")
5722
+ except RunnerError as exc:
5723
+ warnings.append(f"--resolve-version could not read tags: {exc}")
5724
+ return None
5725
+ if args.repo is not None:
5726
+ parsed = probe.parse_repo(args.repo)
5727
+ if parsed is None:
5728
+ warnings.append(f"--resolve-version read no tags: {args.repo} is not a github.com repository")
5729
+ return None
5730
+ probe._start_clock(probe.DEFAULT_TIMEOUT_SEC)
5731
+ out = probe.probe_tags(*parsed, [], 1, ref=args.ref)
5732
+ if out["status"] != "ok":
5733
+ warnings.append(f"the tags of {args.repo} could not be listed ({out['message']}), so no release "
5734
+ f"tag gave a version")
5735
+ return None
5736
+ release = out["release"]
5737
+ return (release["version"], release["tag"]) if release else None
5738
+ try:
5739
+ res = _sibling("skf-source-tree.py")._git(Path(args.source_root), "describe", "--tags", "--abbrev=0")
5740
+ except RunnerError:
5741
+ return None
5742
+ if res is None or res[0] != 0:
5743
+ return None
5744
+ tag = res[1].decode("utf-8", errors="replace").strip()
5745
+ version = probe.tag_version(tag)
5746
+ return (version, tag) if version else None
5747
+
5748
+
5749
+ def resolve_version(args: argparse.Namespace, result: dict, tree: list[str] | None) -> None:
5750
+ """--resolve-version: when the manifest states no version, look it up
5751
+ (Python: _python_version_file; Rust: _cargo_workspace_version; then any
5752
+ language: _release_tag), and for JavaScript or TypeScript read from
5753
+ --repo at the default branch, let the newest release replace a manifest
5754
+ pre-release (-alpha, -beta, -rc, -next, -canary). Sets `version` and
5755
+ `version_source`: "manifest", "file:<path>", "tag:<tag>", or null with
5756
+ a warning when nothing gave one. Raises QuickInputError."""
5757
+ warnings = result["warnings"]
5758
+ version = result["version"]
5759
+ source = "manifest" if version is not None else None
5760
+ manifest = Path(args.manifest_file).as_posix() if args.manifest_file is not None else None
5761
+ if version is None:
5762
+ read = _version_reader(args, tree)
5763
+ found = None
5764
+ if result["language"] == "python":
5765
+ found = _python_version_file(manifest, [Path(e).as_posix() for e in args.entry_file or []], read)
5766
+ elif result["language"] == "rust":
5767
+ found = _cargo_workspace_version(manifest, read)
5768
+ if found:
5769
+ version, source = found[0], f"file:{found[1]}"
5770
+ prerelease = (version is not None and source == "manifest" and result["language"] in JS_FAMILY
5771
+ and args.repo is not None and (args.ref or "").strip().lower() in ("", "null", "none", "head")
5772
+ and _JS_PRERELEASE_RE.search(version) is not None)
5773
+ if version is None or prerelease:
5774
+ tag = _release_tag(args, warnings)
5775
+ if tag and prerelease:
5776
+ warnings.append(f"manifest version {version!r} is a pre-release on the default branch; version is "
5777
+ f"the newest release, {tag[0]} (tag {tag[1]})")
5778
+ if tag:
5779
+ version, source = tag[0], f"tag:{tag[1]}"
5780
+ if version is None:
5781
+ warnings.append("no version found in the manifest, a version file or a release tag; version is null")
5782
+ result["version"] = version
5783
+ result["version_source"] = source
5784
+
5785
+
5786
+ _JS_LIST_SOURCE_RE = re.compile(r"\s*from\s*(['\"])")
5787
+ _JS_IMPORT_RE = re.compile(r"^import\s+(?:type\s+)?([A-Za-z_$*{][^;'\"]*?)\s*from\s*(['\"])", re.M)
5788
+ _JS_BRACES_RE = re.compile(r"\{([^}]*)\}")
5789
+ DEFINED_IN_FAMILIES = {"python": "python", "js": "js", "ts": "js", "javascript": "js", "typescript": "js"}
5790
+
5791
+
5792
+ def _quick_js_facts(content: str) -> dict:
5793
+ """What one JS/TS module exports, read from its text as quick mode reads
5794
+ an entry file, in _js_facts's shape: {"names": {exposed name: {"how",
5795
+ "local", "source"}}, "stars": [source], "default": the local name of
5796
+ its default export, or None}. A declaration beats a list item, and an
5797
+ exported binding an import brings in is that import's re-export (or
5798
+ namespace, for `import * as`)."""
5799
+ code = _blank(content, "js")
5800
+ imports: dict[str, tuple[str, str]] = {}
5801
+ for m in _JS_IMPORT_RE.finditer(code):
5802
+ source = _js_string(content, code, m.end(2) - 1)
5803
+ braces = _JS_BRACES_RE.search(m.group(1))
5804
+ for piece in braces.group(1).split(",") if braces else []:
5805
+ words = piece.split()
5806
+ if words and _JS_NAME_RE.fullmatch(words[-1]):
5807
+ imports.setdefault(words[-1], (_specifier_local(piece), source))
5808
+ for part in _JS_BRACES_RE.sub("", m.group(1)).split(","):
5809
+ words = part.split()
5810
+ if len(words) == 3 and words[:2] == ["*", "as"]:
5811
+ imports.setdefault(words[2], ("*", source))
5812
+ elif len(words) == 1 and _JS_NAME_RE.fullmatch(words[0]):
5813
+ imports.setdefault(words[0], ("default", source))
5814
+ names: dict[str, dict] = {}
5815
+
5816
+ def put(name: str, how: str, local: str | None, source: str | None) -> None:
5817
+ names.setdefault(name, {"how": how, "local": local, "source": source})
5818
+
5819
+ def bound(name: str, local: str) -> None:
5820
+ imported, source = imports.get(local, (None, None))
5821
+ if imported == "*":
5822
+ put(name, "namespace", None, source)
5823
+ elif imported is not None:
5824
+ put(name, "re-export", imported, source)
5825
+ else:
5826
+ put(name, "declaration", local, None)
5827
+
5828
+ for export in scan_exports_js(content, "", None, None, defaults=False):
5829
+ if export["type"] not in ("re-export", "namespace"):
5830
+ put(export["name"], "declaration", export["name"], None)
5831
+ default = None
5832
+ for m in _JS_DECL_RE.finditer(code):
5833
+ if m.group(1) and "default" in m.group(1).split():
5834
+ default = default or m.group(3)
5835
+ put("default", "declaration", m.group(3), None)
5836
+ elif m.group(1) and _js_type(m.group(2)) == "namespace":
5837
+ put(m.group(3), "declaration", m.group(3), None) # `export namespace X`, `export module X`
5838
+ stars: list[str] = []
5839
+ for m in _JS_STAR_RE.finditer(code):
5840
+ source = _js_string(content, code, m.end(2) - 1)
5841
+ if m.group(1):
5842
+ put(m.group(1), "namespace", None, source)
5843
+ else:
5844
+ stars.append(source)
5845
+ for m in _JS_LIST_RE.finditer(code):
5846
+ after = _JS_LIST_SOURCE_RE.match(code, m.end())
5847
+ source = _js_string(content, code, after.end(1) - 1) if after else None
5848
+ for piece in m.group(1).split(","):
5849
+ words = piece.split()
5850
+ if not words or not _JS_NAME_RE.fullmatch(words[-1]):
5851
+ continue
5852
+ if source is not None:
5853
+ put(words[-1], "re-export", _specifier_local(piece), source)
5854
+ else:
5855
+ bound(words[-1], _specifier_local(piece))
5856
+ for m in _JS_DEFAULT_RE.finditer(code):
5857
+ if m.group(1) not in _JS_NOT_NAMES:
5858
+ default = default or m.group(1)
5859
+ bound("default", m.group(1))
5860
+ return {"names": names, "stars": stars, "default": default}
5861
+
5862
+
5863
+ def _js_defined_in(root: Path, tree: set[str]) -> Callable[[str, str], str | None]:
5864
+ """(module, name it exports) -> the file that defines that name, as
5865
+ _js_surface's resolve traces it: through each re-export, imported
5866
+ binding and `export *` (a cycle ends the trace at the module it
5867
+ reached), a namespace export giving its module's file; None for a
5868
+ re-export or a namespace whose module no file under `root` holds."""
5869
+ cache: dict[str, dict] = {}
5870
+
5871
+ def facts(file: str) -> dict:
5872
+ if file not in cache:
5873
+ cache[file] = _quick_js_facts(_read_text(root, file) or "")
5874
+ return cache[file]
5875
+
5876
+ def owner(file: str, name: str) -> str | None:
5877
+ """The first module the `export *` chain of `file` passes `name` on from."""
5878
+ stack, visited = [file], set()
5879
+ while stack:
5880
+ module = stack.pop()
5881
+ if module in visited:
5882
+ continue
5883
+ visited.add(module)
5884
+ if module != file and name in facts(module)["names"]:
5885
+ return module
5886
+ stack += reversed([t for t in (resolve_js_module(s, module, tree) for s in facts(module)["stars"]) if t])
5887
+ return None
5888
+
5889
+ def resolve(file: str, name: str) -> str | None:
5890
+ if name not in facts(file)["names"] and name == facts(file)["default"]:
5891
+ name = "default" # quick mode names a default export by what it declares
5892
+ visited: set[tuple[str, str]] = set()
5893
+ while (file, name) not in visited:
5894
+ visited.add((file, name))
5895
+ fact = facts(file)["names"].get(name)
5896
+ if fact is None:
5897
+ found = owner(file, name) if name != "default" else None
5898
+ if found is None:
5899
+ return file
5900
+ file = found
5901
+ continue
5902
+ if fact["how"] == "declaration":
5903
+ return file
5904
+ target = resolve_js_module(fact["source"], file, tree)
5905
+ if fact["how"] == "namespace" or target is None:
5906
+ return target
5907
+ file, name = target, fact["local"]
5908
+ return file
5909
+
5910
+ return resolve
5911
+
5912
+
5913
+ def _py_defined_in(root: Path, tree: set[str]) -> Callable[[str, str], str | None]:
5914
+ """(module, name it exports) -> the file full mode's _py_surface traces
5915
+ that name to from that module (None when it gives none: a name another
5916
+ package passes on, or one the trace does not list)."""
5917
+ traced: dict[str, dict[str, str | None]] = {}
5918
+
5919
+ def resolve(file: str, name: str) -> str | None:
5920
+ if file not in traced:
5921
+ surface = _Surface("python")
5922
+ _py_surface(root, [{"file": file}], tree, surface, [])
5923
+ traced[file] = {n: info["file"] for n, info in surface.names.items()}
5924
+ return traced[file].get(name)
5925
+
5926
+ return resolve
5927
+
5928
+
5929
+ def _defined_in(root: Path, result: dict) -> None:
5930
+ """Give each export of a Python, JavaScript or TypeScript quick result
5931
+ its `defined_in` (see "Quick mode" above): the file the export is
5932
+ declared in for a name its file binds itself, else the file the trace
5933
+ reaches. The tree under `root` is listed only when a name is traced."""
5934
+ family = DEFINED_IN_FAMILIES.get(result.get("language") or "")
5935
+ if family is None:
5936
+ return
5937
+ resolve: Callable[[str, str], str | None] | None = None
5938
+ for export in result["exports"]:
5939
+ if export["type"] not in ("re-export", "namespace", "module"):
5940
+ export["defined_in"] = export["source_file"]
5941
+ continue
5942
+ if resolve is None:
5943
+ tree = set(list_tree(root))
5944
+ resolve = (_py_defined_in if family == "python" else _js_defined_in)(root, tree)
5945
+ export["defined_in"] = resolve(export["source_file"], export["name"])
5946
+
5947
+
4482
5948
  def _main_entries(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
4483
5949
  """`--mode entries`: the entry files of one package, as JSON on stdout."""
4484
5950
  if args.manifest_file is not None or args.entry_file or args.follow_file:
@@ -4517,9 +5983,20 @@ def main(argv: list[str]) -> int:
4517
5983
  args = parser.parse_args(argv)
4518
5984
  files = args.manifest_file is not None or bool(args.entry_file) or bool(args.follow_file)
4519
5985
  listing = [f"--{flag.replace('_', '-')}" for flag in LISTING_FLAGS if getattr(args, flag) is not None]
4520
- following = [f"--{flag}" for flag in FOLLOW_FLAGS if getattr(args, flag) not in (None, False)]
5986
+ following = [f"--{flag.replace('_', '-')}" for flag in FOLLOW_FLAGS if getattr(args, flag) not in (None, False)]
4521
5987
  if following and (args.mode != "quick" or not files):
4522
5988
  parser.error(f"{', '.join(following)}: quick mode only, with --manifest-file or --entry-file")
5989
+ own = [f"--{flag}" for flag in DENOMINATOR_OWN_FLAGS if getattr(args, flag) is not None]
5990
+ if own and args.mode != "denominator":
5991
+ parser.error(f"{', '.join(own)}: denominator mode only (--mode denominator)")
5992
+ if args.mode == "denominator":
5993
+ other = [f"--{flag.replace('_', '-')}" for flag in FULL_MODE_FLAGS
5994
+ if getattr(args, flag) is not None and flag not in DENOMINATOR_FLAGS]
5995
+ listing = [flag for flag in listing if flag != "--tree-file"]
5996
+ if files or listing or other:
5997
+ parser.error(f"{', '.join(other + listing) or '--manifest-file, --entry-file, --follow-file'}: "
5998
+ "not denominator mode's")
5999
+ return _main_denominator(args)
4523
6000
  if args.mode == "full":
4524
6001
  if files:
4525
6002
  parser.error("--manifest-file, --entry-file, --follow-file: quick mode only (--mode quick)")
@@ -4543,17 +6020,24 @@ def main(argv: list[str]) -> int:
4543
6020
  if files:
4544
6021
  if not args.language or len(args.language) != 1:
4545
6022
  parser.error("--manifest-file and --entry-file take one --language")
4546
- if (args.repo is not None or args.ref is not None) and not args.follow:
4547
- parser.error("--repo and --ref fetch the modules --follow reads, so pass --follow")
6023
+ if (args.repo is not None or args.ref is not None) and not (args.follow or args.resolve_version):
6024
+ parser.error("--repo and --ref fetch the modules --follow reads and name the tags --resolve-version "
6025
+ "reads, so pass --follow or --resolve-version")
4548
6026
  if args.ref is not None and args.repo is None:
4549
6027
  parser.error("--ref is the ref of --repo, so pass --repo")
4550
6028
  if args.follow and (args.tree_file is None or args.source_root is None):
4551
6029
  parser.error("--follow finds each module in --tree-file and reads it under --source-root: pass both")
6030
+ if args.resolve_version and args.source_root is None:
6031
+ parser.error("--resolve-version reads the version files under --source-root: pass it")
4552
6032
  try:
4553
6033
  payload = _quick_files_payload(args)
4554
6034
  result = extract(payload)
4555
6035
  if args.follow and "_error" not in result:
4556
6036
  result = _follow(args, result)
6037
+ if args.resolve_version and "_error" not in result:
6038
+ resolve_version(args, result, payload.get("tree"))
6039
+ if args.source_root is not None and "_error" not in result:
6040
+ _defined_in(Path(args.source_root), result)
4557
6041
  if args.fetch_list is not None and "_error" not in result:
4558
6042
  follow = [item["module_file"] for item in result["unlisted"] if item["module_file"]]
4559
6043
  _write_fetch_list(args.fetch_list, list(dict.fromkeys(follow)))