bmad-module-skill-forge 3.0.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/docs/_data/pinned.yaml +1 -1
  3. package/docs/examples.md +1 -1
  4. package/docs/troubleshooting.md +8 -0
  5. package/docs/verifying-a-skill.md +7 -3
  6. package/docs/workflows.md +3 -3
  7. package/package.json +1 -1
  8. package/src/knowledge/qmd-registry.md +1 -1
  9. package/src/knowledge/version-paths.md +1 -1
  10. package/src/shared/health-check.md +2 -50
  11. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
  12. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +2 -2
  13. package/src/shared/scripts/skf-atomic-write.py +212 -59
  14. package/src/shared/scripts/skf-build-change-manifest.py +593 -47
  15. package/src/shared/scripts/skf-classify-changed-files.py +78 -0
  16. package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
  17. package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
  18. package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
  19. package/src/shared/scripts/skf-extract-public-api.py +1309 -157
  20. package/src/shared/scripts/skf-extraction-inventory.py +580 -46
  21. package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
  22. package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
  23. package/src/shared/scripts/skf-github-probe.py +47 -8
  24. package/src/shared/scripts/skf-hash-content.py +14 -8
  25. package/src/shared/scripts/skf-load-provenance.py +3 -1
  26. package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
  27. package/src/shared/scripts/skf-preflight.py +15 -1
  28. package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
  29. package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
  30. package/src/shared/scripts/skf-structural-diff.py +133 -18
  31. package/src/shared/scripts/skf-update-active-symlink.py +115 -62
  32. package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
  33. package/src/shared/scripts/skf-verify-provenance-completeness.py +977 -49
  34. package/src/shared/scripts/skf-write-skill-brief.py +41 -8
  35. package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
  36. package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
  37. package/src/skf-audit-skill/references/headless-contract.md +1 -1
  38. package/src/skf-audit-skill/references/health-check.md +2 -0
  39. package/src/skf-audit-skill/references/init.md +2 -1
  40. package/src/skf-audit-skill/references/re-index.md +14 -9
  41. package/src/skf-audit-skill/references/report.md +2 -2
  42. package/src/skf-audit-skill/references/semantic-diff.md +15 -15
  43. package/src/skf-audit-skill/references/structural-diff.md +10 -5
  44. package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
  45. package/src/skf-brief-skill/SKILL.md +1 -1
  46. package/src/skf-brief-skill/assets/scope-templates.md +1 -1
  47. package/src/skf-brief-skill/assets/skill-brief-schema.md +2 -9
  48. package/src/skf-brief-skill/references/analyze-target.md +6 -6
  49. package/src/skf-brief-skill/references/confirm-brief.md +1 -1
  50. package/src/skf-brief-skill/references/scope-definition.md +4 -4
  51. package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
  52. package/src/skf-brief-skill/references/version-resolution.md +5 -9
  53. package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
  54. package/src/skf-create-skill/assets/skill-sections.md +6 -6
  55. package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
  56. package/src/skf-create-skill/references/compile.md +31 -60
  57. package/src/skf-create-skill/references/component-extraction.md +3 -3
  58. package/src/skf-create-skill/references/enrich.md +2 -2
  59. package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
  60. package/src/skf-create-skill/references/extract.md +22 -19
  61. package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
  62. package/src/skf-create-skill/references/extraction-patterns.md +8 -9
  63. package/src/skf-create-skill/references/generate-artifacts.md +1 -1
  64. package/src/skf-create-skill/references/report.md +16 -5
  65. package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
  66. package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
  67. package/src/skf-drop-skill/references/execute.md +5 -1
  68. package/src/skf-drop-skill/references/health-check.md +2 -0
  69. package/src/skf-drop-skill/references/invocation-contract.md +1 -1
  70. package/src/skf-drop-skill/references/report.md +2 -2
  71. package/src/skf-drop-skill/references/select.md +7 -3
  72. package/src/skf-test-skill/references/coherence-check.md +1 -1
  73. package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
  74. package/src/skf-test-skill/references/coverage-check.md +39 -57
  75. package/src/skf-test-skill/references/init.md +3 -5
  76. package/src/skf-test-skill/references/report.md +10 -8
  77. package/src/skf-test-skill/references/scoring-rules.md +6 -5
  78. package/src/skf-test-skill/references/source-access-protocol.md +14 -17
  79. package/src/skf-test-skill/scripts/build-result-context.py +114 -21
  80. package/src/skf-test-skill/scripts/gap-ledger.py +315 -30
  81. package/src/skf-test-skill/scripts/load-coverage-inputs.py +534 -96
  82. package/src/skf-test-skill/scripts/reconcile-coverage.py +276 -2
  83. package/src/skf-test-skill/scripts/score-signatures.py +36 -16
  84. package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
  85. package/src/skf-update-skill/references/detect-changes.md +32 -13
  86. package/src/skf-update-skill/references/gap-driven.md +3 -3
  87. package/src/skf-update-skill/references/init.md +6 -2
  88. package/src/skf-update-skill/references/invocation-contract.md +1 -1
  89. package/src/skf-update-skill/references/merge.md +2 -2
  90. package/src/skf-update-skill/references/re-extract.md +2 -2
  91. package/src/skf-update-skill/references/report.md +3 -1
  92. package/src/skf-update-skill/references/write.md +43 -23
  93. package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
@@ -4,7 +4,7 @@
4
4
  # ///
5
5
  """SKF Extract Public API: one package's public surface, by parser or by recipe.
6
6
 
7
- Three modes:
7
+ Four modes:
8
8
 
9
9
  --mode quick (the default) A pure parser: takes one logical package (one
10
10
  manifest, one or more entry-point files) and emits its
@@ -16,9 +16,10 @@ Three modes:
16
16
  raw fetches into a folder, or a local checkout), so a file's
17
17
  text never passes through the model or a shell string.
18
18
  --follow also reads the modules an `export *` chain passes
19
- names on from, fetching them first with --repo. Multi-module
20
- monorepos are caller-orchestrated: invoke once per module,
21
- aggregate.
19
+ names on from, fetching them first with --repo, and
20
+ --resolve-version looks up a version the manifest does
21
+ not state. Multi-module monorepos are caller-orchestrated:
22
+ invoke once per module, aggregate.
22
23
  --mode full The ast-grep recipe runner (skf-create-skill and the
23
24
  workflows that extract as it does): runs the recipes in
24
25
  src/shared/data/ast-grep-recipes.yaml over a source tree
@@ -30,6 +31,11 @@ Three modes:
30
31
  rules, so no caller derives an entry file from a
31
32
  distribution name or reads a manifest by eye. See "Entries
32
33
  mode" below.
34
+ --mode denominator
35
+ Full mode's counts and effective_denominator decision for
36
+ public names counted without its diff (by quick mode or by
37
+ eye), and its scope facts with no recipe run. See
38
+ "Denominator mode" below.
33
39
 
34
40
  Quick mode
35
41
  ----------
@@ -120,7 +126,7 @@ or the files, read from disk (stdin is then never read):
120
126
  --mode quick --language <language> [--source-root <dir>]
121
127
  [--manifest-file <path>] [--entry-file <path>]...
122
128
  [--follow-file <path>]... [--tree-file <listing>] [--fetch-list <file>]
123
- [--follow [--repo <owner/repo> [--ref <ref>]]]
129
+ [--follow] [--resolve-version] [--repo <owner/repo> [--ref <ref>]]
124
130
 
125
131
  An entry with `followed` true (a --follow-file) is a module an `unlisted[]`
126
132
  record named, read as an entry is, less a JS/TS `export default`, which
@@ -138,6 +144,41 @@ A module that cannot be fetched or read stays in `unlisted[]`, and a
138
144
  warning names it and why. The output then also lists `followed`, the
139
145
  modules read this way, in the order they were read.
140
146
 
147
+ --resolve-version (with --source-root) looks up a version the manifest does
148
+ not state, in this order, and names where it came from in `version_source`:
149
+
150
+ python the `__version__` literal of an entry `__init__.py`, then of the
151
+ `_version.py` beside it, then the `version=` literal of the
152
+ setup.py beside the manifest (a dynamic version, `attr:` or
153
+ `file:`, or a version setup.py computes)
154
+ rust the `[workspace.package] version` of the workspace root, the
155
+ nearest Cargo.toml at or above the manifest's folder that holds
156
+ a `[workspace]` table (`version.workspace = true`)
157
+ any the release tag: with --repo, skf-github-probe.py's tag listing
158
+ at --ref (the newest tag whose name is a release version, X.Y.Z,
159
+ at the default branch; the pinned tag itself at a tag; none at a
160
+ branch or a commit); without, the tag `git describe --tags
161
+ --abbrev=0` names in --source-root, when its name is a version
162
+
163
+ A file the lookups read is read under --source-root; with --repo and
164
+ --tree-file, one the listing holds is fetched there first, at --ref. A
165
+ version a file gives must start with a digit and be no placeholder, and
166
+ one a tag gives drops a leading `v`. For JavaScript and TypeScript read
167
+ with --repo at the default branch, a manifest pre-release (`-alpha`,
168
+ `-beta`, `-rc`, `-next`, `-canary`) gives way to the newest release, with
169
+ a warning. When nothing gives a version, a warning says so.
170
+
171
+ With --source-root, each Python, JavaScript and TypeScript export also
172
+ carries `defined_in`: the file that defines it, traced as full mode traces
173
+ an entry point (through each import, re-export, `export *` and star import
174
+ to the end of its chain, a cycle cut), so a name an entry file only passes
175
+ on names its own module. It reads only files under --source-root (the
176
+ files full mode lists there, never the --tree-file listing) and needs no
177
+ ast-grep: Python with full mode's module reader, JavaScript and TypeScript
178
+ from each module's text as quick mode reads an entry file. It is null when
179
+ the trace finds no file, as for a name passed on from another package or a
180
+ module --source-root does not hold. `source_file` stays the file read.
181
+
141
182
  Each path is relative to --source-root, the folder the files were staged in
142
183
  laid out like the repository (or a local checkout), else to the current
143
184
  folder, and is the `path` the output names (`source_file`), written with
@@ -155,9 +196,20 @@ Output JSON shape (stdout):
155
196
  {
156
197
  "language": "python",
157
198
  "package_name": "foo",
158
- "version": "1.2.3",
199
+ "version": "1.2.3", (the manifest's literal
200
+ version, else null: a
201
+ go.mod, a dynamic or
202
+ workspace-inherited one,
203
+ a placeholder; with
204
+ --resolve-version, the
205
+ one it looked up)
206
+ "version_source": "manifest" | "file:<path>" | "tag:<tag>" | null,
207
+ (--resolve-version only)
159
208
  "description": "...",
160
- "exports": [{"name": "Bar", "type": "class", "source_file": "..."}, ...],
209
+ "exports": [{"name": "Bar", "type": "class", "source_file": "...",
210
+ "defined_in": "..."}, ...], (defined_in: Python,
211
+ JS and TS, with
212
+ --source-root)
161
213
  "dependencies": ["requests", "pydantic", ...],
162
214
  "modules": ["server", "client"], (Maven/Gradle only)
163
215
  "module_folders": ["server", "client"], (each module's folder in
@@ -193,10 +245,12 @@ Exit codes (quick):
193
245
  or given twice with the file inputs, a --manifest-file, --entry-file
194
246
  or --follow-file that cannot be read or leads outside --source-root,
195
247
  a --tree-file that cannot be read or a --fetch-list that cannot be
196
- written, --follow without --tree-file and --source-root, --repo or
197
- --ref without --follow (--ref without --repo), or a --repo or --ref
198
- skf-github-fetch.py does not accept. A module --follow could not
199
- read changes no exit code.
248
+ written, --follow without --tree-file and --source-root,
249
+ --resolve-version without --source-root, --repo or --ref without
250
+ --follow or --resolve-version (--ref without --repo), or a --repo or
251
+ --ref skf-github-fetch.py does not accept. A module --follow could
252
+ not read, and a version --resolve-version could not find, change no
253
+ exit code.
200
254
 
201
255
  Entries mode
202
256
  ------------
@@ -276,7 +330,15 @@ is one the recipes read (python, typescript, tsx, javascript, rust, go and
276
330
  .vue files) and --language allows it. Globs follow
277
331
  skf-resolve-authoritative-files.py's rules: `**` spans any number of path
278
332
  segments, none included, and `*` and `?` stay inside one, so `src/**/*.ts`
279
- matches `src/index.ts` and `**/test_*` matches a top-level `test_x.py`.
333
+ matches `src/index.ts` and `**/test_*` matches a top-level `test_x.py`. An
334
+ include glob that matches no file (a brief written for an older layout,
335
+ say) is named in `warnings` and listed in `scope.unmatched_include`. A
336
+ scope.tier_a_include glob is matched against the whole tree under
337
+ --source-root, regardless of --files-from and before the excludes: one
338
+ that matches no file is named in `warnings` too and listed in
339
+ `scope.unmatched_tier_a_include`, and, when scope.include has a glob, one
340
+ that matches only files no scope.include glob matches is named in
341
+ `warnings`.
280
342
 
281
343
  --brief supplies scope.include, scope.exclude, scope.tier_a_include,
282
344
  scope.type and language; --include, --exclude, --tier-a-include,
@@ -337,11 +399,27 @@ JS/TS one that is not). Chains are followed to the end, a cycle cut: an
337
399
  `export *`, a star import or a glob passes on what its module exports,
338
400
  that module's own chains included; a re-exported name is traced through
339
401
  each re-export, import and glob between to the file that defines it (a
340
- default export to the name it declares). A namespace (`export * as ns`, a
341
- Python submodule, a Rust `pub mod`) counts once, and what it makes
342
- reachable is not internal: the names its module passes on, the modules
343
- among them in turn, and for Rust every recipe match in the crate root and
344
- in each module file a `pub mod` or `pub use` of a reachable module names.
402
+ default export to the name it declares). A Python package's explicit
403
+ binding of a name (an import, or a def or class in its __init__.py)
404
+ comes before its submodule: `from .x import y` imports the y
405
+ x/__init__.py binds, and the submodule x/y.py only when x/__init__.py
406
+ binds no y or binds the submodule itself (`from . import y`, `from
407
+ pkg.x import y`); a y x/__init__.py binds only through a star import
408
+ (`from .impl import *`) is still read as the submodule x/y.py. A
409
+ namespace (`export * as ns`, a Python submodule, a Rust `pub mod`)
410
+ lists in `members` the names its module passes on, which users reach as
411
+ `pkg.ns.name`, each traced to the file that defines it as a public name
412
+ is, and in place of a namespace among them its members (each module once,
413
+ so a cycle is cut); `members` is null when the trace cannot read the
414
+ module (a JavaScript target no file here holds, a Rust inline or
415
+ `#[path]` module), and a nested one the trace cannot read is a member
416
+ itself. A member re-exported from another package has no file, is no
417
+ gap when a recipe found its name, marks its name public, and, unlike the
418
+ same form in an entry point, adds no `unresolved` item. What a namespace
419
+ makes reachable is not internal: the names its module passes on, the
420
+ modules among them in turn, and for Rust every recipe match in the crate
421
+ root and in each module file a `pub mod` or `pub use` of a reachable
422
+ module names.
345
423
  A chain the trace cannot follow (a module no file here holds, a `#[path]`
346
424
  module) is listed in `unresolved`. The diff against the recipe inventory,
347
425
  one language family at a time (JavaScript/TypeScript, Python, Rust, Go):
@@ -350,21 +428,59 @@ that no entry point exports or makes reachable), `extraction_gaps` (names
350
428
  an entry point exports that no recipe found, defined in a file in scope or
351
429
  in none the trace reached: read them by eye) and `outside_scope` (such
352
430
  names defined in a file out of scope: widen scope.include to document
353
- them). In a family with no entry point, or only empty ones, every name the
431
+ them). A namespace with `members` is diffed as its members: each is found
432
+ when a recipe found its name, or its `local` one, in its file (one with
433
+ no file by its name alone, as a public name is), and has no row of its
434
+ own when a public name (or another namespace's member) has its name and
435
+ file; one with null `members`, and a member that is such a module, is no
436
+ gap. In a family with no entry point, or only empty ones, every name the
354
437
  recipes found is public.
355
438
 
356
439
  Counts. `exports_public_api` counts the public names (the package's whole
357
- public API, `outside_scope` names included) and `exports_internal` the
358
- internal ones. `effective_denominator` counts the public names whose
359
- definition file matches scope.tier_a_include (else scope.include) and no
360
- scope.exclude glob: each name once, so a type counts once whatever its
361
- methods, and a barrel that only re-exports adds nothing of its own. `arms` gives compile's effective_denominator arms:
440
+ public API, `outside_scope` names included), each distinct name and file
441
+ once: a namespace with `members` counts as its members (none for an empty
442
+ list, and a member that is also a public name, or another namespace's
443
+ member, once), one with null `members` as one. `exports_internal` counts
444
+ the internal names. `effective_denominator` counts those public names
445
+ whose definition file matches scope.include (any file when it has no
446
+ glob), scope.tier_a_include when given, and no scope.exclude glob: each
447
+ name once, so a type counts once whatever its methods, a barrel that
448
+ only re-exports adds nothing of its own, and a name the entry points
449
+ pass on from two files (a namespace member named like another public
450
+ name, or like another namespace's member, at another file) or in two
451
+ languages counts once, when any one of its files matches, as Test
452
+ Skill's sets count it. A tier A list that counts no public name (each
453
+ glob matches no file, or only files scope.exclude drops, that
454
+ scope.include leaves out or that define no public name) narrows nothing
455
+ on a run that counted a public name and read every file (no `errors`
456
+ item): the count falls back to scope.include (else every file), with a
457
+ warning.
458
+ `arms` gives compile's effective_denominator arms:
362
459
  (a) `monorepo` from skf-detect-workspaces.py, (b) `specific_modules`
363
460
  (scope type specific-modules on a repository that is not a monorepo) and
364
461
  (c) `multi_subpath_exports` (an in-scope package.json whose `exports` map
365
462
  has more than one non-root subpath without a `*`, on a repository that is
366
463
  not a monorepo).
367
464
 
465
+ Store or not. `curated_subset` is true when the files the count's basis
466
+ globs list (scope.include's, else every file; a tier A count's, those
467
+ among them a tier A glob matches too; under arm (b), less those a
468
+ scope.exclude glob matches) leave out a file that defines a counted public
469
+ name (an `outside_scope` name's included) or, in a monorepo, every source
470
+ file of a skf-detect-workspaces.py workspace; a basis of `all-files` lists
471
+ none. `effective_denominator_decision` says whether create-skill and
472
+ update-skill store `effective_denominator` as stats.effective_denominator,
473
+ the brief's scope.notes aside (documenting the stratification is the one
474
+ judgment left to them): `store` is true only when none of these holds,
475
+ and `reason` names the first that does, else `curated-subset`:
476
+ `reference-app` (scope type reference-app: pattern_surfaces_documented is
477
+ its basis), `incomplete-run` (an `errors` item: a file a failed call left
478
+ unread can hide a public name, so the count can be 0 or partial),
479
+ `no-arm` (no arm is true), `full-library` (scope type full-library),
480
+ `all-files` (the basis), `not-curated` (`curated_subset` false) and
481
+ `zero-count` (a stored 0 is no denominator: Test Skill reads it as
482
+ deflation).
483
+
368
484
  Output JSON (stdout, or -o):
369
485
 
370
486
  {
@@ -376,7 +492,9 @@ Output JSON (stdout, or -o):
376
492
  "ast_grep": {"path": "<exe>" | null, "version": "<x.y.z>" | null},
377
493
  "scope": {"include": [...], "exclude": [...],
378
494
  "tier_a_include": [...] | null, "type": "<type>" | null,
379
- "languages": [...], "files_from": "<file>" | null},
495
+ "languages": [...], "files_from": "<file>" | null,
496
+ "unmatched_include": [...], # include globs no file matches
497
+ "unmatched_tier_a_include": [...]}, # tier A globs no file matches
380
498
  "files_in_scope": N,
381
499
  "files_by_language": {"<language>": N, ...},
382
500
  "files_without_recipes": {".java": N, ...}, # in scope, no recipe reads them
@@ -409,35 +527,44 @@ Output JSON (stdout, or -o):
409
527
  | {"entry", "name", "from", "file"}, ...]},
410
528
  "entry_point_diff": {
411
529
  "public": [{"name", "language", "entry", "via", "from", "local",
412
- "file", "line"}, ...],
530
+ "file", "line",
531
+ "members": [{"name", "local", "file", "line"}, ...]
532
+ | null}, ...], # members: via namespace only
413
533
  "internal": [{"name", "language", "source_file", "source_line"}, ...],
414
534
  "extraction_gaps": [{"name", "language", "entry", "file", "line"}, ...],
415
535
  "outside_scope": [{"name", "language", "entry", "file", "line"}, ...]},
416
536
  "reexport_targets": [{"name", "language", "local", "from", "entry",
417
537
  "file", "line", "via"}, ...],
418
- "counts": {"exports_public_api": N, "exports_internal": N,
419
- "effective_denominator": N,
538
+ "counts": {"exports_public_api": N, # (family, name, file) keys
539
+ "exports_internal": N,
540
+ "effective_denominator": N, # names, each once
420
541
  "effective_denominator_basis": "tier_a_include" | "scope.include"
421
- | "all-files",
422
- "denominator_files": N},
542
+ | "all-files", # not tier_a_include when it counts no scope.include name
543
+ "denominator_files": N,
544
+ "curated_subset": bool,
545
+ "effective_denominator_decision": {"store": bool, "reason": "<reason>"}},
423
546
  "arms": {"monorepo": bool, "monorepo_kind": "<kind>" | null,
424
547
  "specific_modules": bool, "multi_subpath_exports": bool},
425
548
  "errors": [{"reason": "ast-grep-error" | "ast-grep-timeout" | "time-limit",
426
549
  "files": N, "first_file", "detail",
427
- "unread": [...]}, ...], # unread: time-limit only
428
- "warnings": ["..."]
550
+ "unread": [...]}, ...], # unread: the files its call(s) may have left unread
551
+ "warnings": ["..."],
552
+ "files_in_scope_list": ["<file>", ...], # the files files_in_scope counts, in tree order
553
+ "files_without_recipes_list": ["<file>", ...] # the files files_without_recipes counts
429
554
  }
430
555
 
431
556
  `via` says how an entry point exports a name: `declaration` (defined in the
432
557
  entry point), `re-export` (from another file, which `file` names, `local`
433
558
  giving the name it has there when it differs), `star` (through `export *`,
434
559
  `from .x import *` or `pub use x::*`) or `namespace` (a module: `export *
435
- as ns`, a Python submodule, a Rust module). `line` is null where no line
436
- defines the name (a module, a name from another package). An `unresolved`
437
- item names a subpath whose targets name no file, or a chain met tracing
438
- `entry` that stops at `file`, the module `from` naming no file here.
439
- Without ast-grep the JSON still has the entry points' `files`, the
440
- `exports_maps`, the unresolved subpaths and the `arms`.
560
+ as ns`, a Python submodule, a Rust module, its names in `members`). `line`
561
+ is null where no line defines the name (a module, a name from another
562
+ package), and a member's `file` is null for a name from another package.
563
+ An `unresolved` item names a subpath whose targets name no file, or a
564
+ chain met tracing `entry` that stops at `file`, the module `from` naming
565
+ no file here.
566
+ Without ast-grep the JSON still has the files in scope, the entry points'
567
+ `files`, the `exports_maps`, the unresolved subpaths and the `arms`.
441
568
  `file_issues` lists the files to read by eye: a file ast-grep skips (not
442
569
  UTF-8), and a file where the parser met code it could not read (an ERROR
443
570
  node), where a recipe can miss an export. `t1_low` is 0: every export here
@@ -449,14 +576,18 @@ under the 10-minute limit an agent's shell call often has) have passed
449
576
  since the run started: the run going on is stopped, no other starts, and
450
577
  an `errors[]` item with the reason `time-limit` lists in `unread` the
451
578
  files left unread (each ast-grep call also stops after 600 seconds,
452
- `ast-grep-timeout`). The JSON is still written, status "incomplete".
579
+ `ast-grep-timeout`). A call that times out keeps no match; one that exited
580
+ keeps the matches it printed before any line that is not JSON, but a kill
581
+ can cut a file's matches anywhere, so the item of every failed call lists
582
+ all its files in `unread`. The JSON is still written, status "incomplete".
453
583
 
454
584
  Exit codes (full):
455
585
 
456
586
  0 the recipes ran on every file in scope ("ok")
457
- 1 an ast-grep run failed (exited non-zero) or timed out, or the
458
- --timeout ran out: its files may be unread and the result is
459
- incomplete ("incomplete"); `errors` lists them
587
+ 1 an ast-grep call could not start, exited non-zero, printed other
588
+ than JSON lines or timed out, or the --timeout ran out: the result
589
+ is incomplete ("incomplete"), and each `errors` item lists in
590
+ `unread` the files it may have left unread
460
591
  2 input error (no source root, an unreadable brief, file list or
461
592
  recipe file, a recipe without its metadata, no PyYAML, an -o file
462
593
  that cannot be written, a --timeout not above 0), or a failure the
@@ -466,6 +597,63 @@ Exit codes (full):
466
597
  beside it (cmd.exe would read a repository path's `&` as syntax);
467
598
  the JSON has the scope and no exports; extract by source reading
468
599
 
600
+ Denominator mode
601
+ ----------------
602
+
603
+ --mode denominator [--source-root <dir> | --tree-file <listing>]
604
+ [--brief <skill-brief.yaml>] [--include GLOB]... [--exclude GLOB]...
605
+ [--tier-a-include GLOB]... [--scope-type TYPE] [--language LANG]...
606
+ [--quick <quick.json>]... [--extraction <full.json>] [-o <out.json>]
607
+ < {payload_json}
608
+
609
+ The counts full mode writes, for public names no runner diff gave: those
610
+ a stdin payload lists,
611
+
612
+ {"public": [{"name", "file", "language",
613
+ "members": [{"name", "file"}, ...]}, ...],
614
+ "exports": [{"name", "file", "language"}, ...]}
615
+
616
+ (`file` relative to --source-root, with `/`; `members` for a submodule
617
+ the entry points export by name, which counts as the names its module
618
+ passes on), each export of a --quick output at its `defined_in` (else its
619
+ `source_file`), and the --extraction runner's `entry_point_diff.public`;
620
+ a payload name replaces a --quick or runner one of its family and name.
621
+ `exports_internal` counts the names of the payload's `exports` (the
622
+ exports extraction found; `_` names left out) and of the runner's
623
+ `internal` that no public name or member has. `effective_denominator`,
624
+ its basis, `curated_subset` and `effective_denominator_decision` follow
625
+ full mode's rules, over the tree under --source-root, with the scope and
626
+ arms full mode works out there; an --extraction that is `incomplete` is
627
+ never stored. A name whose `file` is not in that tree counts in
628
+ `exports_public_api` alone, with a warning. With no --source-root (a
629
+ remote source read by eye) the tree is the --tree-file listing (a file,
630
+ or `-` for stdin, which then holds no payload: JSON with a `tree` list, a
631
+ list, or one path per line; one that lists no file is an input error),
632
+ else the names' files; the arms are all false, there is no entry point,
633
+ and the decision is never to store (`no-source-tree`). An empty payload
634
+ (`{}`, or no stdin) gives the scope facts alone, with no recipe run:
635
+ `scope`, the two file lists (by full mode's Which files rule, null with
636
+ neither a tree nor a listing), `entry_points` (`files`, `exports_maps`)
637
+ and `arms`.
638
+
639
+ `entry_point_diff` replaces a compare by eye, each item {"name",
640
+ "language", "file"}: `public`, the names `exports_public_api` counts (a
641
+ namespace as its `members`); `internal`, the exports `exports_internal`
642
+ counts; `extraction_gaps`, the public names the payload and --quick give
643
+ that no export found has (the payload's `exports`, the --extraction
644
+ runner's names), to trace (the runner lists its own gaps).
645
+
646
+ Output JSON: `mode` "denominator", `source_root`, `scope` (as full mode
647
+ writes it), `files_in_scope`, `files_without_recipes`, `entry_points`,
648
+ `arms`, `runner_status` (the --extraction's `status`, else null),
649
+ `counts` (as full mode writes it), `entry_point_diff`, `warnings`, and
650
+ last `files_in_scope_list` and `files_without_recipes_list` (as full
651
+ mode writes them), to -o when given (its folder created). Exit 0, or 2
652
+ on an input error (a --source-root that is no folder, a brief, --quick,
653
+ --extraction or --tree-file that cannot be read, --tree-file beside
654
+ --source-root, a payload that is not this shape, a flag of another
655
+ mode).
656
+
469
657
  CLI examples:
470
658
  uv run skf-extract-public-api.py --mode quick < {payload_json}
471
659
  uv run skf-extract-public-api.py --mode entries --language python \\
@@ -722,7 +910,7 @@ def parse_package_swift(content: str) -> dict:
722
910
  """Best-effort parse of a SwiftPM Package.swift manifest.
723
911
 
724
912
  SwiftPM has no version field in the manifest (versions come from git tags),
725
- so `version` is always None; the brief falls back to target_version / default.
913
+ so `version` is always None; a caller reads it from a git tag.
726
914
  """
727
915
  name_m = re.search(r"\bPackage\s*\(\s*name:\s*['\"]([^'\"]+)['\"]", content, re.DOTALL)
728
916
  deps: list[str] = []
@@ -1332,6 +1520,35 @@ def _detect_placeholder_version(version: str | None) -> str | None:
1332
1520
  return None
1333
1521
 
1334
1522
 
1523
+ def _literal_version(version: object, warnings: list[str]) -> str | None:
1524
+ """The version the manifest writes as a literal string, else None.
1525
+
1526
+ A placeholder and a version that is not a string (Cargo's
1527
+ `version = { workspace = true }`, which the workspace root's Cargo.toml
1528
+ holds) give None with a warning, so the output's `version` is a string
1529
+ or null, which --resolve-version looks up."""
1530
+ if version is None:
1531
+ return None
1532
+ if not isinstance(version, str):
1533
+ if isinstance(version, dict) and version.get("workspace") is True:
1534
+ warnings.append(
1535
+ "manifest version is inherited from the workspace (version.workspace = true), "
1536
+ "which the workspace root's Cargo.toml holds, so the manifest states none"
1537
+ )
1538
+ else:
1539
+ warnings.append(f"manifest version {version!r} is not a string, so the manifest states none")
1540
+ return None
1541
+ placeholder = _detect_placeholder_version(version)
1542
+ if placeholder is not None:
1543
+ warnings.append(
1544
+ f"manifest version {placeholder!r} is a release-time placeholder "
1545
+ f"(workspace protocol or semantic-release sentinel) and is not a real version, "
1546
+ f"so the manifest states none"
1547
+ )
1548
+ return None
1549
+ return version
1550
+
1551
+
1335
1552
  def extract(payload: dict) -> dict:
1336
1553
  """Orchestrate manifest parse + export scan for one logical package."""
1337
1554
  warnings: list[str] = []
@@ -1353,14 +1570,7 @@ def extract(payload: dict) -> dict:
1353
1570
  warnings.append(parsed["_parse_error"])
1354
1571
  parsed = {}
1355
1572
 
1356
- placeholder = _detect_placeholder_version(parsed.get("version"))
1357
- if placeholder is not None:
1358
- warnings.append(
1359
- f"manifest version {placeholder!r} is a release-time placeholder "
1360
- f"(workspace protocol or semantic-release sentinel) and is not a real version; "
1361
- f"the brief will fall back to user-supplied target_version or the default"
1362
- )
1363
- parsed["version"] = None
1573
+ parsed["version"] = _literal_version(parsed.get("version"), warnings)
1364
1574
 
1365
1575
  _, scanner = LANGUAGE_DISPATCH[language]
1366
1576
  exports: list[dict] = []
@@ -1536,6 +1746,8 @@ JS_SOURCE_OF = {
1536
1746
  JS_TARGET_SUFFIXES = (".d.mts", ".d.cts", ".d.ts", ".mjs", ".cjs", ".jsx", ".js", ".mts", ".cts", ".tsx", ".ts")
1537
1747
  PACKAGE_ENTRY_FIELDS = ("types", "typings", "module", "main", "browser")
1538
1748
  WORKSPACE_MANIFESTS = ("package.json", "pnpm-workspace.yaml", "lerna.json", "Cargo.toml", "rush.json", "nx.json")
1749
+ # The arms under which a curated count can be stored as stats.effective_denominator.
1750
+ DENOMINATOR_ARMS = ("monorepo", "specific_modules", "multi_subpath_exports")
1539
1751
 
1540
1752
 
1541
1753
  class RunnerError(Exception):
@@ -1560,8 +1772,9 @@ def _sibling(filename: str):
1560
1772
  """A helper installed beside this script, loaded once as a module:
1561
1773
  skf-source-tree.py (git without the caller's hooks and git variables,
1562
1774
  and a tool lookup that never takes a shim planted in the current
1563
- folder), skf-resolve-authoritative-files.py (the brief's glob rules) and
1564
- skf-detect-workspaces.py (monorepo detection). Raises RunnerError."""
1775
+ folder), skf-resolve-authoritative-files.py (the brief's YAML and glob
1776
+ rules) and skf-detect-workspaces.py (monorepo detection). Raises
1777
+ RunnerError."""
1565
1778
  module = _SIBLINGS.get(filename)
1566
1779
  if module is None:
1567
1780
  path = Path(__file__).resolve().parent / filename
@@ -1856,8 +2069,9 @@ def _read_file_list(path: Path) -> list[str]:
1856
2069
  def _read_brief(path: Path) -> dict:
1857
2070
  """The scope fields and language of a skill brief."""
1858
2071
  yaml = _yaml()
2072
+ parse_brief_yaml = _sibling("skf-resolve-authoritative-files.py").parse_brief_yaml
1859
2073
  try:
1860
- data = yaml.safe_load(path.read_text(encoding="utf-8"))
2074
+ data = parse_brief_yaml(path.read_text(encoding="utf-8"))
1861
2075
  except (OSError, UnicodeDecodeError) as exc:
1862
2076
  raise RunnerError(f"cannot read brief {path}: {exc}") from exc
1863
2077
  except yaml.YAMLError as exc:
@@ -2003,12 +2217,17 @@ def _scan(exe: str, root: Path, folder: Path, rules: list[dict], files: list[str
2003
2217
  errors: list[dict], clock: _Clock | None = None) -> list[dict]:
2004
2218
  """ast-grep's matches of `rules` in `files` (relative to `root`), in
2005
2219
  batches. The rules go in a file as JSON documents (YAML reads JSON),
2006
- never on the command line. A call that times out, exits non-zero (its
2007
- rules carry no severity, so a scan that read every file exits 0, matches
2008
- or not) or prints other than JSON lines adds to `errors`: its batch's
2009
- files may be unread. The matches it printed are kept. When `clock` (the
2010
- run's --timeout) runs out, the call running is stopped and no other
2011
- starts: one `time-limit` item names every file left unread."""
2220
+ never on the command line. A call that cannot start, times out, exits
2221
+ non-zero (its rules carry no severity, so a scan that read every file
2222
+ exits 0, matches or not) or prints other than JSON lines adds to
2223
+ `errors` an item whose `unread` lists every file of its batch: ast-grep
2224
+ prints a file's matches as one block per rule, with other files' blocks
2225
+ between them, and a kill can stop it anywhere, so the output cannot show
2226
+ which files it printed in full. A call that exited keeps the matches it
2227
+ printed before any line that is not JSON; a call that timed out keeps
2228
+ none. When `clock` (the run's --timeout) runs out, a call running is
2229
+ stopped (its `time-limit` item lists its batch) and no other starts: a
2230
+ `time-limit` item lists the files no call read, if any."""
2012
2231
  if not rules or not files:
2013
2232
  return []
2014
2233
  rule_file = folder / "rules.yml"
@@ -2055,10 +2274,8 @@ def _scan(exe: str, root: Path, folder: Path, rules: list[dict], files: list[str
2055
2274
  if match is not None:
2056
2275
  found.append(match)
2057
2276
  if failure is not None:
2058
- error = {"reason": failure[0], "files": len(batch), "first_file": batch[0], "detail": failure[1]}
2059
- if failure[0] == "time-limit":
2060
- error["unread"] = list(batch)
2061
- errors.append(error)
2277
+ errors.append({"reason": failure[0], "files": len(batch), "first_file": batch[0], "detail": failure[1],
2278
+ "unread": list(batch)})
2062
2279
  return found
2063
2280
 
2064
2281
 
@@ -2861,26 +3078,66 @@ def _specifier_local(text: str) -> str:
2861
3078
  return re.split(r"\s+as\s+", item, maxsplit=1)[0].strip().strip("'\"")
2862
3079
 
2863
3080
 
3081
+ def _member(name: str, local: str | None, file: str | None, line: int | None) -> dict:
3082
+ """One name a namespace's module passes on, traced to the file that
3083
+ defines it (`local` the name it has there when it differs)."""
3084
+ return {"name": name, "local": local if local and local != name else None, "file": file, "line": line}
3085
+
3086
+
3087
+ def _flatten(start, direct: Callable[[object], list[tuple[str, str, object]]]) -> list[dict]:
3088
+ """The members of the namespace module `start`: what `direct` says it
3089
+ passes on, (name, "member", member) or (name, "module", the module),
3090
+ each nested namespace module's members spliced in its place (each
3091
+ module once, so a cycle is cut), each (name, file) once, and no name
3092
+ that starts with `_`, which is never public."""
3093
+ out: list[dict] = []
3094
+ seen: set[tuple[str, str | None]] = set()
3095
+ visited: set = set()
3096
+
3097
+ def walk(module) -> None:
3098
+ visited.add(module)
3099
+ for name, kind, value in direct(module):
3100
+ if not name or name.startswith("_"):
3101
+ continue
3102
+ if kind == "module":
3103
+ if value not in visited:
3104
+ walk(value)
3105
+ elif (value["name"], value["file"]) not in seen:
3106
+ seen.add((value["name"], value["file"]))
3107
+ out.append(value)
3108
+
3109
+ walk(start)
3110
+ return out
3111
+
3112
+
2864
3113
  class _Surface:
2865
3114
  """The public names one language family's entry points declare:
2866
3115
  {name: info}, the first entry point naming it wins; the names, and the
2867
3116
  files, a namespace export (a module) makes reachable, which are not
2868
- internal; and the re-exports the trace could not follow."""
3117
+ internal; and the re-exports the trace could not follow. A namespace
3118
+ item also carries its `members`, the names its module passes on (None
3119
+ when the trace cannot read that module); `opaque` holds the (name,
3120
+ file) of each member that is itself a module the trace cannot read,
3121
+ which, as such a namespace, is no recipe's target."""
2869
3122
 
2870
3123
  def __init__(self) -> None:
2871
3124
  self.names: dict[str, dict] = {}
2872
3125
  self.reachable: set[str] = set()
2873
3126
  self.reachable_files: set[str] = set()
2874
3127
  self.unresolved: list[dict] = []
3128
+ self.opaque: set[tuple[str, str | None]] = set()
2875
3129
 
2876
3130
  def add(self, name: str, entry: str, via: str, file: str | None, line: int | None,
2877
- source: str | None = None, local: str | None = None) -> dict | None:
3131
+ source: str | None = None, local: str | None = None,
3132
+ members: list[dict] | None = None) -> dict | None:
2878
3133
  if not name or name.startswith("_"):
2879
3134
  return None
2880
3135
  if name not in self.names:
2881
3136
  self.names[name] = {"name": name, "entry": entry, "via": via, "from": source,
2882
3137
  "local": local if local and local != name else None,
2883
3138
  "file": file, "line": line}
3139
+ if via == "namespace":
3140
+ self.names[name]["members"] = [dict(m) for m in members] if members is not None else None
2884
3141
  return self.names[name]
2885
3142
 
2886
3143
  def unresolve(self, entry: str, name: str, source: str | None, file: str) -> None:
@@ -3053,6 +3310,40 @@ def _js_surface(root: Path, entries: list[dict], tree_set: set[str], forms: dict
3053
3310
  if fact is not None and fact["how"] == "namespace":
3054
3311
  reach(target(fact["source"], where))
3055
3312
 
3313
+ passes: dict[str, list[tuple[str, str, object]]] = {}
3314
+
3315
+ def direct(file: str) -> list[tuple[str, str, object]]:
3316
+ """What the namespace module `file` passes on, in order: (name,
3317
+ "member", a member traced as publish traces a name) or (name,
3318
+ "module", the file of a namespace among them); a namespace whose
3319
+ module no file here holds is a member itself, with no file, and so
3320
+ is a re-export from another package, which (unlike the same form in
3321
+ an entry point) is not unresolved."""
3322
+ if file not in passes:
3323
+ out: list[tuple[str, str, object]] = []
3324
+ for name, owner in passed_on(file).items():
3325
+ where, local, line, fact = resolve(owner, name, set())
3326
+ how = fact["how"] if fact is not None else None
3327
+ namespace = target(fact["source"], where) if how == "namespace" else None
3328
+ if namespace is not None:
3329
+ out.append((name, "module", namespace))
3330
+ elif how == "namespace":
3331
+ surface.opaque.add((name, None))
3332
+ out.append((name, "member", _member(name, None, None, None)))
3333
+ elif how == "re-export":
3334
+ out.append((name, "member", _member(name, local, None, None)))
3335
+ else:
3336
+ out.append((name, "member", _member(name, local, where, line)))
3337
+ passes[file] = out
3338
+ return passes[file]
3339
+
3340
+ member_lists: dict[str, list[dict]] = {}
3341
+
3342
+ def members(file: str) -> list[dict]:
3343
+ if file not in member_lists:
3344
+ member_lists[file] = _flatten(file, direct)
3345
+ return member_lists[file]
3346
+
3056
3347
  def publish(entry: str, name: str, via: str, module: str, source: str | None) -> None:
3057
3348
  """Add public `name`, which `module` exports, traced to what it is."""
3058
3349
  chain: set[str] = set()
@@ -3061,7 +3352,8 @@ def _js_surface(root: Path, entries: list[dict], tree_set: set[str], forms: dict
3061
3352
  how = fact["how"] if fact is not None else None
3062
3353
  if how == "namespace":
3063
3354
  namespace = target(fact["source"], where)
3064
- surface.add(name, entry, "namespace", namespace, None, source)
3355
+ surface.add(name, entry, "namespace", namespace, None, source,
3356
+ members=members(namespace) if namespace is not None else None)
3065
3357
  if namespace is None:
3066
3358
  surface.unresolve(entry, name, fact["source"], where)
3067
3359
  reach(namespace)
@@ -3371,9 +3663,16 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
3371
3663
  if found is None:
3372
3664
  return None, binding["imported"], None, "import"
3373
3665
  if found.endswith("__init__.py"):
3374
- sub = _py_submodule(posixpath.dirname(found), binding["imported"], tree_set)
3375
- if sub and sub != rel:
3376
- return sub, binding["imported"], None, "submodule"
3666
+ # Python's own precedence: the package's explicit binding of
3667
+ # the name first, the submodule only when it binds none (or
3668
+ # binds the submodule itself, `from . import name` or
3669
+ # `from pkg.x import name`); a star import is not read here
3670
+ inner = bindings(found)[1].get(binding["imported"])
3671
+ if inner is None or (inner["kind"] == "import" and inner["imported"] == binding["imported"]
3672
+ and module_file(found, inner["level"], inner["module"], top) == found):
3673
+ sub = _py_submodule(posixpath.dirname(found), binding["imported"], tree_set)
3674
+ if sub and sub != rel:
3675
+ return sub, binding["imported"], None, "submodule"
3377
3676
  rel, name = found, binding["imported"]
3378
3677
  return rel, name, None, "missing"
3379
3678
 
@@ -3391,6 +3690,31 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
3391
3690
  if what == "submodule" and where is not None:
3392
3691
  reach(where, top)
3393
3692
 
3693
+ passes: dict[tuple[str, str], list[tuple[str, str, object]]] = {}
3694
+
3695
+ def direct(rel: str, top: str) -> list[tuple[str, str, object]]:
3696
+ """What the submodule `rel` passes on (its star_names), in order:
3697
+ (name, "member", a member traced by resolve) or (name, "module",
3698
+ the file of a submodule among them); a name from another package
3699
+ has no file."""
3700
+ if (rel, top) not in passes:
3701
+ out: list[tuple[str, str, object]] = []
3702
+ for name, owner in star_names(rel, top).items():
3703
+ where, local, line, what = resolve(owner, name, top, set())
3704
+ if what == "submodule" and where is not None:
3705
+ out.append((name, "module", where))
3706
+ else:
3707
+ out.append((name, "member", _member(name, local, where, line)))
3708
+ passes[(rel, top)] = out
3709
+ return passes[(rel, top)]
3710
+
3711
+ member_lists: dict[tuple[str, str], list[dict]] = {}
3712
+
3713
+ def members(rel: str, top: str) -> list[dict]:
3714
+ if (rel, top) not in member_lists:
3715
+ member_lists[(rel, top)] = _flatten(rel, lambda module: direct(module, top))
3716
+ return member_lists[(rel, top)]
3717
+
3394
3718
  for entry in [e["file"] for e in entries]:
3395
3719
  top = posixpath.dirname(entry)
3396
3720
  all_names, bound, stars = bindings(entry)
@@ -3420,7 +3744,7 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
3420
3744
  where, local, line, what = resolve(entry, name, top, chain)
3421
3745
  surface.reachable.update(chain)
3422
3746
  if what == "submodule":
3423
- surface.add(name, entry, "namespace", where, None, source, local)
3747
+ surface.add(name, entry, "namespace", where, None, source, local, members=members(where, top))
3424
3748
  reach(where, top)
3425
3749
  elif what == "import":
3426
3750
  surface.add(name, entry, "re-export", None, None, source, local)
@@ -3674,6 +3998,41 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
3674
3998
  if sub is not None:
3675
3999
  reach(lib, sub, reached)
3676
4000
 
4001
+ passes: dict[tuple[str, tuple[str, ...]], list[tuple[str, str, object]]] = {}
4002
+
4003
+ def direct(lib: str, here: tuple[str, ...]) -> list[tuple[str, str, object]]:
4004
+ """What the module at `here` passes on (exported), in order: (name,
4005
+ "member", a member traced as publish traces a name) or (name,
4006
+ "module", the path of a module among them that a file of its own
4007
+ holds); an inline or `#[path]` module is a member itself, at its
4008
+ `pub mod`, and a `pub use` of another crate's item one with no file,
4009
+ which (unlike the same form in the crate root) is not unresolved."""
4010
+ if (lib, here) not in passes:
4011
+ out: list[tuple[str, str, object]] = []
4012
+ for name in exported(lib, list(here)):
4013
+ file, local, line, sub, stop = resolve(lib, list(here), name, set())
4014
+ if stop is not None:
4015
+ out.append((name, "member", _member(name, local, None, None)))
4016
+ elif sub is not None and _rust_file(lib, sub, tree_set) is not None:
4017
+ out.append((name, "module", tuple(sub)))
4018
+ else:
4019
+ if sub is not None:
4020
+ surface.opaque.add((name, file))
4021
+ out.append((name, "member", _member(name, local, file, line)))
4022
+ passes[(lib, here)] = out
4023
+ return passes[(lib, here)]
4024
+
4025
+ member_lists: dict[tuple[str, tuple[str, ...]], list[dict]] = {}
4026
+
4027
+ def members(lib: str, here: list[str]) -> list[dict] | None:
4028
+ """The members of the module at `here`; None when no file of its own holds it."""
4029
+ if _rust_file(lib, here, tree_set) is None:
4030
+ return None
4031
+ key = (lib, tuple(here))
4032
+ if key not in member_lists:
4033
+ member_lists[key] = _flatten(tuple(here), lambda path: direct(lib, path))
4034
+ return member_lists[key]
4035
+
3677
4036
  def publish(lib: str, here: list[str], name: str, via: str, source: str | None,
3678
4037
  reached: set[tuple[str, ...]]) -> None:
3679
4038
  """Add public `name`, which the module at `here` exports, traced to
@@ -3685,7 +4044,7 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
3685
4044
  surface.add(name, lib, via, None, None, source, local)
3686
4045
  surface.unresolve(lib, name, *stop)
3687
4046
  elif sub is not None:
3688
- surface.add(name, lib, "namespace", file, line, "::".join(sub))
4047
+ surface.add(name, lib, "namespace", file, line, "::".join(sub), members=members(lib, sub))
3689
4048
  reach(lib, sub, reached)
3690
4049
  else:
3691
4050
  surface.add(name, lib, via, file, line, source, local)
@@ -3700,7 +4059,8 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
3700
4059
  continue
3701
4060
  sub = [name.removeprefix("r#")]
3702
4061
  sub_file = _rust_file(lib, sub, tree_set)
3703
- surface.add(name, lib, "namespace", sub_file or lib, None if sub_file else line)
4062
+ surface.add(name, lib, "namespace", sub_file or lib, None if sub_file else line,
4063
+ members=members(lib, sub))
3704
4064
  reach(lib, sub, reached)
3705
4065
  for path, alias, glob, _ in info["uses"]:
3706
4066
  if not glob:
@@ -3779,8 +4139,13 @@ def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries
3779
4139
  reachable (a name, or any export of a reachable file), is internal. A
3780
4140
  declared name no recipe found is a gap when a file in scope defines it
3781
4141
  (or the trace found no file), and is outside scope when a file out of
3782
- scope does; a module is no recipe's target, so neither. In a family
3783
- with no entry point, or only empty ones, every export is public."""
4142
+ scope does; a module is no recipe's target, so neither: a namespace
4143
+ with `members` has each member diffed in its place, found when a
4144
+ recipe found its name, or its `local` one, in its file, and given no
4145
+ row of its own when a declared name has its name and file (or another
4146
+ namespace gave it one); one whose members are unknown (None), or a
4147
+ member that is such a module (`opaque`), is no gap. In a family with
4148
+ no entry point, or only empty ones, every export is public."""
3784
4149
  statuses: dict[str, str] = {}
3785
4150
  public: dict[tuple[str, str], dict] = {}
3786
4151
  internal: list[dict] = []
@@ -3814,16 +4179,50 @@ def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries
3814
4179
  if name in names or (info["local"] or name) in names or info["via"] == "namespace":
3815
4180
  continue
3816
4181
  (gaps if info["file"] is None or info["file"] in in_scope else outside).append((family, info))
4182
+ # A namespace's members, each by its name (or `local`) and file, or
4183
+ # by name alone when it has no file, as a declared name is: a name
4184
+ # at a file a declared name already gives counts once.
4185
+ found = {(e["export_name"], e["source_file"]) for e in records}
4186
+ given = {(info["name"], info["file"]) for info in declared.values() if not _has_members(info)}
4187
+ for info in declared.values():
4188
+ for member in info["members"] if _has_members(info) else []:
4189
+ key = (member["name"], member["file"])
4190
+ if (key in given or key in found or key in surface.opaque
4191
+ or (member["local"] and (member["local"], member["file"]) in found)
4192
+ or (member["file"] is None and {member["name"], member["local"]} & names)):
4193
+ continue
4194
+ given.add(key)
4195
+ row = {"name": member["name"], "entry": info["entry"], "file": member["file"],
4196
+ "line": member["line"]}
4197
+ (gaps if member["file"] is None or member["file"] in in_scope else outside).append((family, row))
3817
4198
  return statuses, public, internal, gaps, outside
3818
4199
 
3819
4200
 
4201
+ def _has_members(info: dict) -> bool:
4202
+ """A namespace item whose module's members the trace read (a list, maybe empty)."""
4203
+ return info.get("via") == "namespace" and isinstance(info.get("members"), list)
4204
+
4205
+
4206
+ def _public_keys(public: dict[tuple[str, str], dict]) -> set[tuple[str, str, str | None]]:
4207
+ """The (family, name, file) the counts count: each public name but a
4208
+ namespace with members, in its place each member, and so a member that
4209
+ is also a declared name (or another namespace's member) once."""
4210
+ keys: set[tuple[str, str, str | None]] = set()
4211
+ for (family, _), info in public.items():
4212
+ if _has_members(info):
4213
+ keys.update((family, m["name"], m["file"]) for m in info["members"])
4214
+ else:
4215
+ keys.add((family, info["name"], info["file"]))
4216
+ return keys
4217
+
4218
+
3820
4219
  def _head_cap(value: int | None, tier: str | None, scope_type: str | None) -> int | None:
3821
4220
  if value is not None:
3822
4221
  return value or None
3823
4222
  return HEAD_CAPS.get((tier, scope_type), DEFAULT_HEAD_CAP)
3824
4223
 
3825
4224
 
3826
- def _monorepo(root: Path, tree: list[str], tree_set: set[str]) -> tuple[bool, str | None]:
4225
+ def _workspaces(root: Path, tree: list[str], tree_set: set[str]) -> dict:
3827
4226
  """skf-detect-workspaces.py's verdict on the source root."""
3828
4227
  manifests = {}
3829
4228
  for name in WORKSPACE_MANIFESTS:
@@ -3831,19 +4230,111 @@ def _monorepo(root: Path, tree: list[str], tree_set: set[str]) -> tuple[bool, st
3831
4230
  text = _read_text(root, name)
3832
4231
  if text is not None:
3833
4232
  manifests[name] = text
3834
- result = _sibling("skf-detect-workspaces.py").detect({"tree": tree, "manifests": manifests})
4233
+ return _sibling("skf-detect-workspaces.py").detect({"tree": tree, "manifests": manifests})
4234
+
4235
+
4236
+ def _monorepo(root: Path, tree: list[str], tree_set: set[str]) -> tuple[bool, str | None]:
4237
+ """skf-detect-workspaces.py's verdict on the source root: (is it a monorepo, its manifest kind)."""
4238
+ result = _workspaces(root, tree, tree_set)
3835
4239
  return bool(result.get("is_monorepo")), result.get("manifest_kind")
3836
4240
 
3837
4241
 
4242
+ def _workspace_sources(result: dict, tree: list[str], wanted: set[str]) -> list[set[str]]:
4243
+ """The source files (of a language --language selects) of each workspace
4244
+ skf-detect-workspaces.py found, one set per workspace that holds any."""
4245
+ sources = []
4246
+ for workspace in result.get("workspaces") or [] if result.get("is_monorepo") else []:
4247
+ folder = _norm(workspace.get("path") if isinstance(workspace, dict) else "").strip("/")
4248
+ files = {f for f in tree if (not folder or f.startswith(folder + "/")) and _file_language(f) in wanted}
4249
+ if files:
4250
+ sources.append(files)
4251
+ return sources
4252
+
4253
+
4254
+ def denominator_count(counted: set[tuple[str, str, str | None]], tree: list[str], includes: list[str],
4255
+ excludes: list[str], tier_a: list[str], glob_match: Callable[[str, str], bool],
4256
+ read_all: bool) -> tuple[int, str, set[str], str | None]:
4257
+ """(effective_denominator, its basis, the files it counts, a warning or None)
4258
+ over the (family, name, file) keys `counted`: each name once whose file
4259
+ matches scope.include (any file with no glob), scope.tier_a_include when
4260
+ given (a tier A name counts only in a file scope.include matches too) and
4261
+ no scope.exclude glob. A tier A list that counts no name falls back to
4262
+ scope.include (else every file) once a name was counted and every file
4263
+ was read (`read_all`)."""
4264
+
4265
+ def files_matching(globs: list[str]) -> set[str]:
4266
+ return {f for f in tree if (not globs or any(glob_match(f, g) for g in globs))
4267
+ and not any(glob_match(f, g) for g in excludes)}
4268
+
4269
+ in_include = files_matching(includes)
4270
+ denominator_files = (files_matching(tier_a) & in_include) if tier_a else in_include
4271
+ basis = "tier_a_include" if tier_a else ("scope.include" if includes else "all-files")
4272
+ warning = None
4273
+ if tier_a and counted and read_all and not any(file in denominator_files for _, _, file in counted):
4274
+ basis = "scope.include" if includes else "all-files"
4275
+ denominator_files = in_include
4276
+ counts_what = "scope.include" if includes else "every file"
4277
+ warning = f"scope.tier_a_include counts no public name: effective_denominator counts {counts_what}"
4278
+ # Each name once, as Test Skill's sets of names count it: a name the entry points pass on
4279
+ # from two files, or in two languages, is two keys but one name.
4280
+ count = len({name for _, name, file in counted if file in denominator_files})
4281
+ return count, basis, denominator_files, warning
4282
+
4283
+
4284
+ def curated_subset(counted: set[tuple[str, str, str | None]], basis: str, tree: list[str], includes: list[str],
4285
+ excludes: list[str], tier_a: list[str], glob_match: Callable[[str, str], bool],
4286
+ arms: dict, workspace_sources: list[set[str]]) -> bool:
4287
+ """Whether the files the count read are a curated subset rather than the
4288
+ whole workspace (in a single-package repository, the whole package): the
4289
+ files the basis globs list (a tier A list's, among those scope.include
4290
+ lists; under arm (b), less those scope.exclude drops) leave out a file
4291
+ that defines a counted public name or, in a monorepo, every source file
4292
+ of a workspace. A basis of all-files lists none."""
4293
+ if basis == "all-files":
4294
+ return False
4295
+ files = {f for f in tree if not includes or any(glob_match(f, g) for g in includes)}
4296
+ if basis == "tier_a_include":
4297
+ files = {f for f in files if any(glob_match(f, g) for g in tier_a)}
4298
+ if arms.get("specific_modules"):
4299
+ files = {f for f in files if not any(glob_match(f, g) for g in excludes)}
4300
+ tree_set = set(tree)
4301
+ defining = {file for _, _, file in counted if file in tree_set}
4302
+ return bool(defining - files) or any(not (sources & files) for sources in workspace_sources)
4303
+
4304
+
4305
+ def effective_denominator_decision(arms: dict, scope_type: str | None, basis: str, curated: bool, count: int,
4306
+ incomplete: bool) -> dict:
4307
+ """Whether create-skill and update-skill store the count as
4308
+ stats.effective_denominator, the brief's scope.notes aside: {store, reason},
4309
+ the reason naming the first condition that fails."""
4310
+ if scope_type == "reference-app":
4311
+ reason = "reference-app"
4312
+ elif incomplete:
4313
+ reason = "incomplete-run"
4314
+ elif not any(arms.get(arm) for arm in DENOMINATOR_ARMS):
4315
+ reason = "no-arm"
4316
+ elif scope_type == "full-library":
4317
+ reason = "full-library"
4318
+ elif basis == "all-files":
4319
+ reason = "all-files"
4320
+ elif not curated:
4321
+ reason = "not-curated"
4322
+ elif count == 0:
4323
+ reason = "zero-count"
4324
+ else:
4325
+ return {"store": True, "reason": "curated-subset"}
4326
+ return {"store": False, "reason": reason}
4327
+
4328
+
3838
4329
  def _select_files(root: Path, candidates: list[str], listed: bool, includes: list[str], excludes: list[str],
3839
4330
  wanted: set[str], glob_match: Callable[[str, str], bool], issues: list[dict],
3840
- warnings: list[str]) -> tuple[list[str], dict[str, int]]:
3841
- """(the files in scope, by extension the in-scope source files no recipe
3842
- reads). A file a --files-from list names that is missing, or that no
3843
- recipe reads, is an issue; an include glob no candidate matches, a
3844
- warning."""
4331
+ warnings: list[str]) -> tuple[list[str], list[str], list[str]]:
4332
+ """(the files in scope, the in-scope source files no recipe reads, the
4333
+ include globs no candidate matches). A file a --files-from list names
4334
+ that is missing, or that no recipe reads, is an issue; an include glob
4335
+ no candidate matches, a warning too."""
3845
4336
  in_scope: list[str] = []
3846
- without_recipes: Counter = Counter()
4337
+ without_recipes: list[str] = []
3847
4338
  used = dict.fromkeys(includes, False)
3848
4339
  for rel in candidates:
3849
4340
  hits = [g for g in includes if glob_match(rel, g)]
@@ -3859,51 +4350,66 @@ def _select_files(root: Path, candidates: list[str], listed: bool, includes: lis
3859
4350
  if listed:
3860
4351
  issues.append({"file": rel, "issue": "no-recipes"})
3861
4352
  elif extension in NO_RECIPE_EXTENSIONS:
3862
- without_recipes[extension] += 1
4353
+ without_recipes.append(rel)
3863
4354
  continue
3864
4355
  in_scope.append(rel)
3865
- warnings += [f"scope.include pattern {g!r} matches no file" for g, hit in used.items() if not hit]
3866
- return in_scope, dict(sorted(without_recipes.items()))
4356
+ unmatched = [g for g, hit in used.items() if not hit]
4357
+ warnings += [f"scope.include pattern {g!r} matches no file" for g in unmatched]
4358
+ return in_scope, without_recipes, unmatched
3867
4359
 
3868
4360
 
3869
- def run_full(args: argparse.Namespace) -> tuple[dict, int]:
3870
- """Run the recipes over the source tree; (result, exit code)."""
3871
- clock = _Clock(DEFAULT_TIMEOUT_SEC if args.timeout is None else args.timeout)
3872
- root = Path(args.source_root)
3873
- if not root.is_dir():
3874
- raise RunnerError(f"source root not found: {args.source_root}")
3875
- recipes_path = Path(args.recipes) if args.recipes else RECIPES_FILE
3876
- recipes = load_recipe_file(recipes_path)
4361
+ def _by_extension(files: list[str]) -> dict[str, int]:
4362
+ """`files_without_recipes`: how many of `files` each extension has."""
4363
+ return dict(sorted(Counter(posixpath.splitext(f)[1].lower() for f in files).items()))
4364
+
4365
+
4366
+ def _unmatched_tier_a(tier_a: list[str], includes: list[str], tree: list[str],
4367
+ glob_match: Callable[[str, str], bool], warnings: list[str]) -> list[str]:
4368
+ """The tier A globs that match no file of `tree` (the whole tree, regardless
4369
+ of --files-from and before the excludes), with a warning for each, and one
4370
+ for each that matches only files outside scope.include."""
4371
+ unmatched = [g for g in dict.fromkeys(tier_a) if not any(glob_match(f, g) for f in tree)]
4372
+ warnings += [f"scope.tier_a_include pattern {g!r} matches no file" for g in unmatched]
4373
+ warnings += [f"scope.tier_a_include pattern {g!r} matches only files outside scope.include"
4374
+ for g in dict.fromkeys(tier_a) if includes and g not in unmatched
4375
+ and not any(glob_match(f, g) and any(glob_match(f, i) for i in includes) for f in tree)]
4376
+ return unmatched
4377
+
4378
+
4379
+ def _brief_scope(args: argparse.Namespace) -> dict:
4380
+ """The scope globs, scope type and languages: the flags', else the brief's."""
3877
4381
  brief = _read_brief(Path(args.brief)) if args.brief else {}
3878
- includes = [g for g in (_norm(v) for v in (args.include or brief.get("include") or [])) if g]
3879
- excludes = [g for g in (_norm(v) for v in (args.exclude or brief.get("exclude") or [])) if g]
3880
- tier_a = [g for g in (_norm(v) for v in (args.tier_a_include or brief.get("tier_a_include") or [])) if g]
3881
- scope_type = args.scope_type or brief.get("type")
3882
4382
  language_values = args.language or ([brief["language"]] if brief.get("language") else [])
3883
4383
  wanted, unknown_languages = _wanted_languages(language_values)
3884
- recipe_set = args.recipe_set or ("component-library" if scope_type == "component-library" else "standard")
3885
- head_cap = _head_cap(args.head_cap, args.tier, scope_type)
3886
- glob_match = _sibling("skf-resolve-authoritative-files.py").glob_match
4384
+ return {
4385
+ "includes": [g for g in (_norm(v) for v in (args.include or brief.get("include") or [])) if g],
4386
+ "excludes": [g for g in (_norm(v) for v in (args.exclude or brief.get("exclude") or [])) if g],
4387
+ "tier_a": [g for g in (_norm(v) for v in (args.tier_a_include or brief.get("tier_a_include") or [])) if g],
4388
+ "scope_type": args.scope_type or brief.get("type"),
4389
+ "wanted": wanted,
4390
+ "glob_match": _sibling("skf-resolve-authoritative-files.py").glob_match,
4391
+ "warnings": [f"no recipe reads {value} files" for value in unknown_languages],
4392
+ }
3887
4393
 
3888
- warnings: list[str] = [f"no recipe reads {value} files" for value in unknown_languages]
3889
- errors: list[dict] = []
4394
+
4395
+ def _scope_facts(args: argparse.Namespace, root: Path) -> dict:
4396
+ """What full mode and denominator mode know before any recipe runs: the
4397
+ brief's scope, the files in scope, the entry points and the arms."""
4398
+ scope = _brief_scope(args)
4399
+ includes, excludes, tier_a = scope["includes"], scope["excludes"], scope["tier_a"]
4400
+ scope_type, wanted, glob_match = scope["scope_type"], scope["wanted"], scope["glob_match"]
4401
+ warnings: list[str] = scope["warnings"]
3890
4402
  issues: list[dict] = []
3891
4403
  tree = list_tree(root)
3892
4404
  tree_set = set(tree)
3893
- listed = _read_file_list(Path(args.files_from)) if args.files_from else None
3894
- in_scope, without_recipes = _select_files(root, tree if listed is None else listed, listed is not None,
3895
- includes, excludes, wanted, glob_match, issues, warnings)
3896
- readable = []
3897
- for rel in in_scope:
3898
- try:
3899
- (root / rel).read_bytes().decode("utf-8")
3900
- except OSError:
3901
- issues.append({"file": rel, "issue": "unreadable"})
3902
- continue
3903
- except UnicodeDecodeError:
3904
- issues.append({"file": rel, "issue": "not-utf8"})
3905
- continue
3906
- readable.append(rel)
4405
+ files_from = getattr(args, "files_from", None)
4406
+ listed = _read_file_list(Path(files_from)) if files_from else None
4407
+ in_scope, without_recipes, unmatched_include = _select_files(
4408
+ root, tree if listed is None else listed, listed is not None, includes, excludes, wanted, glob_match,
4409
+ issues, warnings)
4410
+ # A tier A glob is matched against the whole tree under --source-root (the
4411
+ # tree effective_denominator reads).
4412
+ unmatched_tier_a = _unmatched_tier_a(tier_a, includes, tree, glob_match, warnings)
3907
4413
 
3908
4414
  # The entry points and the arms need no ast-grep.
3909
4415
  in_scope_set = set(in_scope)
@@ -3922,13 +4428,85 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
3922
4428
  f"{entry['package']}) is outside the files in scope: widen scope.include to document "
3923
4429
  "the names it exports (entry_point_diff.outside_scope), which effective_denominator "
3924
4430
  "leaves out")
3925
- monorepo, monorepo_kind = _monorepo(root, tree, tree_set)
4431
+ workspaces = _workspaces(root, tree, tree_set)
4432
+ monorepo = bool(workspaces.get("is_monorepo"))
3926
4433
  arms = {
3927
4434
  "monorepo": monorepo,
3928
- "monorepo_kind": monorepo_kind,
4435
+ "monorepo_kind": workspaces.get("manifest_kind"),
3929
4436
  "specific_modules": scope_type == "specific-modules" and not monorepo,
3930
4437
  "multi_subpath_exports": not monorepo and any(m["non_root_subpaths"] > 1 for m in maps),
3931
4438
  }
4439
+ return {
4440
+ "includes": includes, "excludes": excludes, "tier_a": tier_a, "scope_type": scope_type, "wanted": wanted,
4441
+ "glob_match": glob_match, "warnings": warnings, "issues": issues, "tree": tree, "tree_set": tree_set,
4442
+ "in_scope": in_scope, "without_recipes": without_recipes, "unmatched_include": unmatched_include,
4443
+ "unmatched_tier_a": unmatched_tier_a, "surfaces": surfaces, "js_entries": js_entries, "maps": maps,
4444
+ "unresolved_subpaths": unresolved_subpaths, "py_entries": py_entries, "rust_entries": rust_entries,
4445
+ "entries": entries, "arms": arms, "workspace_sources": _workspace_sources(workspaces, tree, wanted),
4446
+ }
4447
+
4448
+
4449
+ def _scope_record(args: argparse.Namespace, facts: dict) -> dict:
4450
+ """The `scope` object full mode and denominator mode write."""
4451
+ return {"include": facts["includes"], "exclude": facts["excludes"], "tier_a_include": facts["tier_a"] or None,
4452
+ "type": facts["scope_type"], "languages": sorted(facts["wanted"]),
4453
+ "files_from": getattr(args, "files_from", None), "unmatched_include": facts["unmatched_include"],
4454
+ "unmatched_tier_a_include": facts["unmatched_tier_a"]}
4455
+
4456
+
4457
+ def _denominator_counts(facts: dict, counted: set[tuple[str, str, str | None]], public_api: int, internal: int,
4458
+ incomplete: bool, warnings: list[str]) -> dict:
4459
+ """The `counts` object: the two API counts, effective_denominator with its
4460
+ basis and files, whether those files are a curated subset, and the
4461
+ decision whether to store the count."""
4462
+ count, basis, denominator_files, warning = denominator_count(
4463
+ counted, facts["tree"], facts["includes"], facts["excludes"], facts["tier_a"], facts["glob_match"],
4464
+ not incomplete)
4465
+ if warning:
4466
+ warnings.append(warning)
4467
+ curated = curated_subset(counted, basis, facts["tree"], facts["includes"], facts["excludes"], facts["tier_a"],
4468
+ facts["glob_match"], facts["arms"], facts["workspace_sources"])
4469
+ return {
4470
+ "exports_public_api": public_api,
4471
+ "exports_internal": internal,
4472
+ "effective_denominator": count,
4473
+ "effective_denominator_basis": basis,
4474
+ "denominator_files": len(denominator_files),
4475
+ "curated_subset": curated,
4476
+ "effective_denominator_decision": effective_denominator_decision(
4477
+ facts["arms"], facts["scope_type"], basis, curated, count, incomplete),
4478
+ }
4479
+
4480
+
4481
+ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
4482
+ """Run the recipes over the source tree; (result, exit code)."""
4483
+ clock = _Clock(DEFAULT_TIMEOUT_SEC if args.timeout is None else args.timeout)
4484
+ root = Path(args.source_root)
4485
+ if not root.is_dir():
4486
+ raise RunnerError(f"source root not found: {args.source_root}")
4487
+ recipes_path = Path(args.recipes) if args.recipes else RECIPES_FILE
4488
+ recipes = load_recipe_file(recipes_path)
4489
+ facts = _scope_facts(args, root)
4490
+ scope_type, warnings, issues = facts["scope_type"], facts["warnings"], facts["issues"]
4491
+ tree_set, in_scope = facts["tree_set"], facts["in_scope"]
4492
+ in_scope_set = set(in_scope)
4493
+ surfaces, js_entries, maps = facts["surfaces"], facts["js_entries"], facts["maps"]
4494
+ py_entries, rust_entries, entries = facts["py_entries"], facts["rust_entries"], facts["entries"]
4495
+ unresolved_subpaths, arms = facts["unresolved_subpaths"], facts["arms"]
4496
+ recipe_set = args.recipe_set or ("component-library" if scope_type == "component-library" else "standard")
4497
+ head_cap = _head_cap(args.head_cap, args.tier, scope_type)
4498
+ errors: list[dict] = []
4499
+ readable = []
4500
+ for rel in in_scope:
4501
+ try:
4502
+ (root / rel).read_bytes().decode("utf-8")
4503
+ except OSError:
4504
+ issues.append({"file": rel, "issue": "unreadable"})
4505
+ continue
4506
+ except UnicodeDecodeError:
4507
+ issues.append({"file": rel, "issue": "not-utf8"})
4508
+ continue
4509
+ readable.append(rel)
3932
4510
 
3933
4511
  exe, version, shim = _ast_grep()
3934
4512
  result: dict = {
@@ -3938,12 +4516,10 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
3938
4516
  "recipes_file": recipes_path.as_posix(),
3939
4517
  "recipe_set": recipe_set,
3940
4518
  "ast_grep": {"path": exe, "version": version},
3941
- "scope": {"include": includes, "exclude": excludes, "tier_a_include": tier_a or None,
3942
- "type": scope_type, "languages": sorted(wanted),
3943
- "files_from": args.files_from},
4519
+ "scope": _scope_record(args, facts),
3944
4520
  "files_in_scope": len(in_scope),
3945
4521
  "files_by_language": dict(sorted(Counter(_file_language(f) for f in in_scope).items())),
3946
- "files_without_recipes": without_recipes,
4522
+ "files_without_recipes": _by_extension(facts["without_recipes"]),
3947
4523
  "file_issues": issues,
3948
4524
  "head_cap": head_cap,
3949
4525
  "truncated": False,
@@ -3958,6 +4534,9 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
3958
4534
  "arms": arms,
3959
4535
  "errors": errors,
3960
4536
  "warnings": warnings,
4537
+ # Last: only a run without the recipes reads them, and a large tree lists thousands.
4538
+ "files_in_scope_list": in_scope,
4539
+ "files_without_recipes_list": facts["without_recipes"],
3961
4540
  }
3962
4541
  if exe is None:
3963
4542
  result["status"] = "no-ast-grep"
@@ -4013,10 +4592,12 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
4013
4592
  else:
4014
4593
  status = "empty-barrel" if "empty-barrel" in statuses.values() else "no-entry-point"
4015
4594
 
4016
- basis_globs = tier_a or includes
4017
- basis = "tier_a_include" if tier_a else ("scope.include" if includes else "all-files")
4018
- denominator_files = {f for f in tree if (not basis_globs or any(glob_match(f, g) for g in basis_globs))
4019
- and not any(glob_match(f, g) for g in excludes)}
4595
+ # A tier A list that counts no public name (each glob matches no file, or
4596
+ # only files an exclude drops, that scope.include leaves out or that
4597
+ # define no public name) narrows nothing, once the run counted a name
4598
+ # and read every file (no error).
4599
+ counted = _public_keys(public)
4600
+ counts = _denominator_counts(facts, counted, len(counted), len(internal), bool(errors), warnings)
4020
4601
 
4021
4602
  by_type: Counter = Counter(e["export_type"] for e in exports)
4022
4603
  ordered = sorted(public.items(), key=lambda item: (item[0][1], item[0][0]))
@@ -4043,13 +4624,7 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
4043
4624
  {"name": info["name"], "language": family, "local": info["local"], "from": info["from"],
4044
4625
  "entry": info["entry"], "file": info["file"], "line": info["line"], "via": info["via"]}
4045
4626
  for (family, _), info in ordered if info["via"] in ("re-export", "star", "namespace")],
4046
- "counts": {
4047
- "exports_public_api": len(public),
4048
- "exports_internal": len(internal),
4049
- "effective_denominator": sum(1 for info in public.values() if info["file"] in denominator_files),
4050
- "effective_denominator_basis": basis,
4051
- "denominator_files": len(denominator_files),
4052
- },
4627
+ "counts": counts,
4053
4628
  })
4054
4629
  result["file_issues"] = sorted(issues, key=lambda i: (i["file"], i["issue"]))
4055
4630
  if errors:
@@ -4058,37 +4633,251 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
4058
4633
  return result, 0
4059
4634
 
4060
4635
 
4636
+ def _full_exit_2(message: str) -> int:
4637
+ """Exit 2 of full mode (an input error, a failed -o write or a crash):
4638
+ `message` on one stderr line, each line break and the whitespace around
4639
+ it joined into one space (a YAML error spans several lines) and every
4640
+ other space kept, so a path is quoted as written. Callers quote the line
4641
+ as the reason they stop or fall back."""
4642
+ sys.stderr.write("error: " + re.sub(r"\s*[\r\n]\s*", " ", message.strip("\r\n")) + "\n")
4643
+ return 2
4644
+
4645
+
4061
4646
  def _main_full(args: argparse.Namespace) -> int:
4062
4647
  if not args.source_root:
4063
- sys.stderr.write("error: --mode full needs --source-root\n")
4064
- return 2
4648
+ return _full_exit_2("--mode full needs --source-root")
4065
4649
  if args.head_cap is not None and args.head_cap < 0:
4066
- sys.stderr.write("error: --head-cap must be 0 (no cap) or more\n")
4067
- return 2
4650
+ return _full_exit_2("--head-cap must be 0 (no cap) or more")
4068
4651
  if args.timeout is not None and not args.timeout > 0:
4069
- sys.stderr.write("error: --timeout must be more than 0 seconds\n")
4070
- return 2
4652
+ return _full_exit_2("--timeout must be more than 0 seconds")
4071
4653
  try:
4072
4654
  result, code = run_full(args)
4073
4655
  except RunnerError as exc:
4074
- sys.stderr.write(f"error: {exc}\n")
4075
- return 2
4656
+ return _full_exit_2(str(exc))
4076
4657
  except Exception as exc: # noqa: BLE001 - one stderr line and exit 2, never a traceback
4077
- detail = " ".join(str(exc).split())
4078
- sys.stderr.write(f"error: the run failed: {type(exc).__name__}: {detail}\n")
4079
- return 2
4658
+ return _full_exit_2(f"the run failed: {type(exc).__name__}: {exc}")
4080
4659
  text = json.dumps(result, indent=2) + "\n"
4081
4660
  if args.output:
4082
4661
  try:
4083
4662
  Path(args.output).write_text(text, encoding="utf-8")
4084
4663
  except OSError as exc:
4085
- sys.stderr.write(f"error: cannot write {args.output}: {exc}\n")
4086
- return 2
4664
+ return _full_exit_2(f"cannot write {args.output}: {exc}")
4087
4665
  else:
4088
4666
  sys.stdout.write(text)
4089
4667
  return code
4090
4668
 
4091
4669
 
4670
+ # --------------------------------------------------------------------------
4671
+ # Denominator mode: the counts and the effective_denominator decision of
4672
+ # names counted without the runner's diff
4673
+ # --------------------------------------------------------------------------
4674
+
4675
+ NO_ARMS = {"monorepo": False, "monorepo_kind": None, "specific_modules": False, "multi_subpath_exports": False}
4676
+
4677
+
4678
+ class DenominatorInputError(Exception):
4679
+ """Exit 2 of denominator mode."""
4680
+
4681
+
4682
+ def _name_family(language: object, file: object) -> str:
4683
+ """The language family a name counts in: its file's, else its language's."""
4684
+ family = FAMILY_OF.get(_file_language(file)) if isinstance(file, str) and file else None
4685
+ if family is not None:
4686
+ return family
4687
+ value = str(language or "").strip().lower()
4688
+ selected = LANGUAGE_FAMILIES.get(value)
4689
+ return FAMILY_OF[selected[0]] if selected else value
4690
+
4691
+
4692
+ def _payload_names(items: object, where: str) -> list[dict]:
4693
+ """The {name, file, language, members} items of a payload list, checked."""
4694
+ if items is None:
4695
+ return []
4696
+ if not isinstance(items, list):
4697
+ raise DenominatorInputError(f"{where} is not a list")
4698
+ names = []
4699
+ for i, item in enumerate(items):
4700
+ if not isinstance(item, dict) or not isinstance(item.get("name"), str) or not item["name"].strip():
4701
+ raise DenominatorInputError(f"{where}[{i}] is not an object with a name")
4702
+ if item.get("file") is not None and not isinstance(item["file"], str):
4703
+ raise DenominatorInputError(f"{where}[{i}].file is not a path")
4704
+ members = item.get("members")
4705
+ if members is not None:
4706
+ if not isinstance(members, list):
4707
+ raise DenominatorInputError(f"{where}[{i}].members is not a list")
4708
+ members = _payload_names(members, f"{where}[{i}].members")
4709
+ file = _norm(item["file"]) if item.get("file") else None
4710
+ names.append({"name": item["name"].strip(), "file": file, "language": item.get("language"),
4711
+ "members": members})
4712
+ return names
4713
+
4714
+
4715
+ def _read_json_file(path: str, flag: str) -> dict:
4716
+ try:
4717
+ data = json.loads(Path(path).read_text(encoding="utf-8"))
4718
+ except (OSError, UnicodeDecodeError, ValueError) as exc:
4719
+ raise DenominatorInputError(f"cannot read {flag} {path}: {exc}") from exc
4720
+ if not isinstance(data, dict):
4721
+ raise DenominatorInputError(f"{flag} {path} is not a JSON object")
4722
+ return data
4723
+
4724
+
4725
+ def run_denominator(args: argparse.Namespace, payload: dict) -> dict:
4726
+ """The counts of the public names `payload`, --quick and --extraction
4727
+ give, with the effective_denominator decision full mode makes."""
4728
+ public: dict[tuple[str, str], dict] = {}
4729
+ runner_public: dict[tuple[str, str], dict] = {}
4730
+ # The exports found, each key at its file: the payload's and the runner's.
4731
+ found: dict[tuple[str, str], str | None] = {}
4732
+ internal: set[tuple[str, str]] = set()
4733
+ incomplete = False
4734
+ runner_status = None
4735
+ if args.extraction:
4736
+ runner = _read_json_file(args.extraction, "--extraction")
4737
+ if runner.get("mode") != "full":
4738
+ raise DenominatorInputError(f"--extraction {args.extraction} is not --mode full output")
4739
+ runner_status = runner.get("status")
4740
+ incomplete = runner_status == "incomplete" or bool(runner.get("errors"))
4741
+ diff = runner.get("entry_point_diff") if isinstance(runner.get("entry_point_diff"), dict) else {}
4742
+ for info in diff.get("public") or []:
4743
+ if isinstance(info, dict) and isinstance(info.get("name"), str):
4744
+ key = (str(info.get("language") or ""), info["name"])
4745
+ public[key] = runner_public[key] = info
4746
+ found[key] = info.get("file")
4747
+ for item in diff.get("internal") or []:
4748
+ if isinstance(item, dict) and isinstance(item.get("name"), str):
4749
+ internal.add((str(item.get("language") or ""), item["name"]))
4750
+ found[(str(item.get("language") or ""), item["name"])] = item.get("source_file")
4751
+ for path in args.quick or []:
4752
+ quick = _read_json_file(path, "--quick")
4753
+ if not isinstance(quick.get("exports"), list):
4754
+ raise DenominatorInputError(f"--quick {path} is not --mode quick output (no exports list)")
4755
+ for item in quick["exports"]:
4756
+ if isinstance(item, dict) and isinstance(item.get("name"), str) and item["name"]:
4757
+ file = item.get("defined_in") or item.get("source_file")
4758
+ file = _norm(file) if isinstance(file, str) and file else None
4759
+ public[(_name_family(quick.get("language"), file), item["name"])] = {
4760
+ "name": item["name"], "file": file, "via": "declaration"}
4761
+ for item in _payload_names(payload.get("public"), "public"):
4762
+ family = _name_family(item["language"], item["file"])
4763
+ info = {"name": item["name"], "file": item["file"], "via": "declaration"}
4764
+ if item["members"] is not None:
4765
+ info.update(via="namespace", members=[{"name": m["name"], "file": m["file"]} for m in item["members"]])
4766
+ public[(family, item["name"])] = info
4767
+ for item in _payload_names(payload.get("exports"), "exports"):
4768
+ key = (_name_family(item["language"], item["file"]), item["name"])
4769
+ found.setdefault(key, item["file"])
4770
+ if not item["name"].startswith("_"):
4771
+ internal.add(key)
4772
+ named = set(public) | {(family, m["name"]) for (family, _), info in public.items() if _has_members(info)
4773
+ for m in info["members"]}
4774
+ counted = _public_keys(public)
4775
+
4776
+ if args.source_root:
4777
+ root = Path(args.source_root)
4778
+ if not root.is_dir():
4779
+ raise RunnerError(f"source root not found: {args.source_root}")
4780
+ facts = _scope_facts(args, root)
4781
+ else:
4782
+ # No local tree (a remote source read by eye): the repository's listing, else the names' files.
4783
+ facts = _brief_scope(args)
4784
+ in_scope = without_recipes = None
4785
+ unmatched_include: list[str] = []
4786
+ unmatched_tier_a: list[str] = []
4787
+ if args.tree_file is not None:
4788
+ try:
4789
+ listed, cut = _read_listing(args.tree_file)
4790
+ except QuickInputError as exc:
4791
+ raise DenominatorInputError(str(exc)) from exc
4792
+ tree = sorted(set(listed))
4793
+ if cut:
4794
+ facts["warnings"].append("the listing was cut short: the file lists can miss files")
4795
+ in_scope, without_recipes, unmatched_include = _select_files(
4796
+ Path("."), tree, False, facts["includes"], facts["excludes"], facts["wanted"], facts["glob_match"],
4797
+ [], facts["warnings"])
4798
+ unmatched_tier_a = _unmatched_tier_a(facts["tier_a"], facts["includes"], tree, facts["glob_match"],
4799
+ facts["warnings"])
4800
+ else:
4801
+ tree = sorted({file for _, _, file in counted if file})
4802
+ facts.update(tree=tree, in_scope=in_scope, without_recipes=without_recipes,
4803
+ unmatched_include=unmatched_include, unmatched_tier_a=unmatched_tier_a, arms=dict(NO_ARMS),
4804
+ workspace_sources=[], entries=[], maps=[])
4805
+ warnings = facts["warnings"]
4806
+ tree_set = set(facts["tree"])
4807
+ outside_tree = sorted((name, file) for _, name, file in counted if file and file not in tree_set)
4808
+ warnings += [f"public name {name} is defined in {file}, which is not in the tree: effective_denominator "
4809
+ "and curated_subset leave it out" for name, file in outside_tree]
4810
+ counts = _denominator_counts(facts, counted, len(counted), len(internal - named), incomplete, warnings)
4811
+ if not args.source_root and counts["effective_denominator_decision"]["reason"] not in ("reference-app",
4812
+ "incomplete-run"):
4813
+ counts["effective_denominator_decision"] = {"store": False, "reason": "no-source-tree"}
4814
+ without_recipes = facts["without_recipes"]
4815
+
4816
+ def row(key: tuple[str, str], file: str | None) -> dict:
4817
+ return {"name": key[1], "language": key[0], "file": file}
4818
+
4819
+ def by_name(keys):
4820
+ return sorted(keys, key=lambda k: (k[1], k[0]))
4821
+
4822
+ # The names the payload and --quick gave (the runner diffed its own), a namespace as its members.
4823
+ given: dict[tuple[str, str], str | None] = {}
4824
+ for key, info in public.items():
4825
+ if info is runner_public.get(key):
4826
+ continue
4827
+ for member in info["members"] if _has_members(info) else [info]:
4828
+ given.setdefault((key[0], member["name"]), member["file"])
4829
+ return {
4830
+ "mode": "denominator",
4831
+ "source_root": args.source_root,
4832
+ "scope": _scope_record(args, facts),
4833
+ "files_in_scope": None if facts["in_scope"] is None else len(facts["in_scope"]),
4834
+ "files_without_recipes": None if without_recipes is None else _by_extension(without_recipes),
4835
+ "entry_points": {"files": facts["entries"], "exports_maps": facts["maps"]},
4836
+ "arms": facts["arms"],
4837
+ "runner_status": runner_status,
4838
+ "counts": counts,
4839
+ "entry_point_diff": {
4840
+ "public": [row((family, name), file)
4841
+ for family, name, file in sorted(counted, key=lambda k: (k[1], k[0], k[2] or ""))],
4842
+ "internal": [row(key, found.get(key)) for key in by_name(internal - named)],
4843
+ "extraction_gaps": [row(key, given[key]) for key in by_name(given) if key not in found],
4844
+ },
4845
+ "warnings": warnings,
4846
+ "files_in_scope_list": facts["in_scope"],
4847
+ "files_without_recipes_list": without_recipes,
4848
+ }
4849
+
4850
+
4851
+ def _main_denominator(args: argparse.Namespace) -> int:
4852
+ if args.source_root and args.tree_file is not None:
4853
+ return _full_exit_2("--tree-file stands for a remote source with no --source-root: pass one of them")
4854
+ # With `--tree-file -` stdin holds the listing, so there is no payload.
4855
+ reads_stdin = args.tree_file != "-" and sys.stdin is not None and not sys.stdin.isatty()
4856
+ raw = sys.stdin.read() if reads_stdin else ""
4857
+ try:
4858
+ payload = json.loads(raw) if raw.strip() else {}
4859
+ except ValueError as exc:
4860
+ return _full_exit_2(f"stdin is not JSON: {exc}")
4861
+ if not isinstance(payload, dict):
4862
+ return _full_exit_2("stdin must be a JSON object: {\"public\": [...], \"exports\": [...]}")
4863
+ try:
4864
+ result = run_denominator(args, payload)
4865
+ except (RunnerError, DenominatorInputError) as exc:
4866
+ return _full_exit_2(str(exc))
4867
+ except Exception as exc: # noqa: BLE001 - one stderr line and exit 2, never a traceback
4868
+ return _full_exit_2(f"the run failed: {type(exc).__name__}: {exc}")
4869
+ text = json.dumps(result, indent=2) + "\n"
4870
+ if args.output:
4871
+ try:
4872
+ Path(args.output).parent.mkdir(parents=True, exist_ok=True)
4873
+ Path(args.output).write_text(text, encoding="utf-8")
4874
+ except OSError as exc:
4875
+ return _full_exit_2(f"cannot write {args.output}: {exc}")
4876
+ else:
4877
+ sys.stdout.write(text)
4878
+ return 0
4879
+
4880
+
4092
4881
  # --------------------------------------------------------------------------
4093
4882
  # Entries mode: the files quick mode reads, found in the listing
4094
4883
  # --------------------------------------------------------------------------
@@ -4281,10 +5070,11 @@ def _build_parser() -> argparse.ArgumentParser:
4281
5070
  parser.add_argument(
4282
5071
  "--mode",
4283
5072
  default="quick",
4284
- choices=("quick", "full", "entries"),
5073
+ choices=("quick", "full", "entries", "denominator"),
4285
5074
  help="quick (default): parse the stdin payload, or the --manifest-file and --entry-file files; "
4286
5075
  "full: run the recipes over --source-root; entries: list one package's entry files from "
4287
- "--tree-file.",
5076
+ "--tree-file; denominator: count the public names a stdin payload, --quick and --extraction give, "
5077
+ "and decide whether to store effective_denominator, as full mode does.",
4288
5078
  )
4289
5079
  quick = parser.add_argument_group("quick mode, files read from disk")
4290
5080
  quick.add_argument("--manifest-file", metavar="PATH",
@@ -4298,16 +5088,23 @@ def _build_parser() -> argparse.ArgumentParser:
4298
5088
  help=f"read each module an unlisted[] record names as one more --follow-file, round "
4299
5089
  f"after round (at most {FOLLOW_ROUNDS}), until none is left; needs --tree-file and "
4300
5090
  f"--source-root")
5091
+ quick.add_argument("--resolve-version", action="store_true",
5092
+ help="look up a version the manifest does not state: a version file, a Cargo workspace "
5093
+ "root, then a release tag (of --repo at --ref, else git describe in --source-root); "
5094
+ "needs --source-root")
4301
5095
  quick.add_argument("--repo", metavar="OWNER/REPO",
4302
5096
  help="with --follow: fetch each module from this GitHub repository into --source-root "
4303
- "first (skf-github-fetch.py)")
5097
+ "first (skf-github-fetch.py); with --resolve-version: the repository whose tags "
5098
+ "it reads, and whose files it fetches with --tree-file")
4304
5099
  quick.add_argument("--ref", metavar="REF",
4305
- help="with --repo: the branch, tag or commit to fetch at (default: the default branch)")
5100
+ help="with --repo: the branch, tag or commit to fetch and read the version at "
5101
+ "(default: the default branch)")
4306
5102
  listing = parser.add_argument_group("entries mode, and the listing quick mode follows chains in")
4307
5103
  listing.add_argument("--tree-file", metavar="PATH",
4308
5104
  help="the repository's file listing (skf-github-probe.py tree output, JSON, or one "
4309
- "path per line): entries mode finds the entry files in it, and quick mode "
4310
- "resolves each unlisted[] statement's module_file against it")
5105
+ "path per line): entries mode finds the entry files in it, quick mode "
5106
+ "resolves each unlisted[] statement's module_file against it, and denominator "
5107
+ "mode, with no --source-root, takes it as the tree (- reads it from stdin)")
4311
5108
  listing.add_argument("--scope", metavar="FOLDER",
4312
5109
  help="entries mode: the package folder to read (default: the repository root)")
4313
5110
  listing.add_argument("--fetch-list", metavar="FILE",
@@ -4322,7 +5119,11 @@ def _build_parser() -> argparse.ArgumentParser:
4322
5119
  full.add_argument("--exclude", action="append", metavar="GLOB",
4323
5120
  help="an exclude glob, repeatable (replaces the brief's scope.exclude)")
4324
5121
  full.add_argument("--tier-a-include", action="append", metavar="GLOB",
4325
- help="a tier-A glob for effective_denominator, repeatable (replaces scope.tier_a_include)")
5122
+ help="a tier-A glob for effective_denominator, repeatable (replaces scope.tier_a_include), "
5123
+ "that counts a name only in a file scope.include matches too; one that matches no "
5124
+ "file is listed in scope.unmatched_tier_a_include, one that matches only files "
5125
+ "scope.include leaves out is named in warnings, and a list that counts no public name "
5126
+ "gives way to scope.include (else every file)")
4326
5127
  full.add_argument("--scope-type", help="the scope type (replaces the brief's scope.type)")
4327
5128
  full.add_argument("--language", action="append",
4328
5129
  help="a language family to extract, repeatable (replaces the brief's language); quick "
@@ -4339,6 +5140,14 @@ def _build_parser() -> argparse.ArgumentParser:
4339
5140
  help=f"stop the ast-grep runs after this many seconds and still write the JSON, "
4340
5141
  f"status incomplete (default {DEFAULT_TIMEOUT_SEC})")
4341
5142
  full.add_argument("-o", "--output", metavar="FILE", help="write the JSON here instead of stdout")
5143
+ denominator = parser.add_argument_group("denominator mode (with --source-root, --brief, the scope flags, "
5144
+ "--language and -o)")
5145
+ denominator.add_argument("--quick", action="append", metavar="FILE",
5146
+ help="a --mode quick output whose exports are public names, each at its defined_in "
5147
+ "(else its source_file), repeatable")
5148
+ denominator.add_argument("--extraction", metavar="FILE",
5149
+ help="a --mode full output whose entry_point_diff public and internal names count "
5150
+ "too; an incomplete one is never stored")
4342
5151
  return parser
4343
5152
 
4344
5153
 
@@ -4346,10 +5155,14 @@ FULL_MODE_FLAGS = ("source_root", "brief", "include", "exclude", "tier_a_include
4346
5155
  "files_from", "recipe_set", "recipes", "tier", "head_cap", "timeout", "output")
4347
5156
  # The full mode flags quick mode takes with its file inputs, and entries mode always.
4348
5157
  QUICK_FILE_FLAGS = ("source_root", "language")
5158
+ # The full mode flags denominator mode takes, and its own.
5159
+ DENOMINATOR_FLAGS = ("source_root", "brief", "include", "exclude", "tier_a_include", "scope_type", "language",
5160
+ "output")
5161
+ DENOMINATOR_OWN_FLAGS = ("quick", "extraction")
4349
5162
  # The flags of entries mode; quick mode takes --tree-file and --fetch-list with its file inputs.
4350
5163
  LISTING_FLAGS = ("tree_file", "scope", "fetch_list")
4351
5164
  # Quick mode's own flags with its file inputs: the chains it follows itself.
4352
- FOLLOW_FLAGS = ("follow", "repo", "ref")
5165
+ FOLLOW_FLAGS = ("follow", "repo", "ref", "resolve_version")
4353
5166
  FOLLOW_ROUNDS = 5
4354
5167
 
4355
5168
 
@@ -4479,6 +5292,327 @@ def _follow(args: argparse.Namespace, result: dict) -> dict:
4479
5292
  return result
4480
5293
 
4481
5294
 
5295
+ # --resolve-version: a version the manifest does not state
5296
+ _PY_VERSION_RE = re.compile(r"""^__version__[ \t]*(?::[^=\n]*)?=[ \t]*(['"])([^'"\n]+)\1""", re.M)
5297
+ _JS_PRERELEASE_RE = re.compile(r"-(?:alpha|beta|rc|next|canary)", re.I)
5298
+ JS_FAMILY = ("js", "ts", "javascript", "typescript")
5299
+
5300
+
5301
+ def _version_reader(args: argparse.Namespace, tree: list[str] | None) -> Callable[[str], str | None]:
5302
+ """path -> the text of that file under --source-root, else None. With
5303
+ --repo and --tree-file, a file the listing holds and the folder does not
5304
+ is fetched there first, as --follow fetches a module. Raises
5305
+ QuickInputError."""
5306
+ root = Path(args.source_root)
5307
+ listed = set(tree or [])
5308
+ fetch: list = []
5309
+
5310
+ def read(path: str) -> str | None:
5311
+ target = root / path
5312
+ if not target.is_file() and args.repo is not None and args.tree_file is not None and path in listed:
5313
+ if not fetch:
5314
+ fetch.append(_follow_reader(args))
5315
+ fetch[0]([path])
5316
+ try:
5317
+ if target.is_file() and target.resolve().is_relative_to(root.resolve()):
5318
+ return target.read_bytes().decode("utf-8-sig", errors="replace")
5319
+ except OSError:
5320
+ pass
5321
+ return None
5322
+ return read
5323
+
5324
+
5325
+ def _version_value(value: object) -> str | None:
5326
+ """A version a version file writes: a literal that starts with a digit, not a placeholder."""
5327
+ if not isinstance(value, str) or not re.match(r"\d", value.strip()):
5328
+ return None
5329
+ return None if _detect_placeholder_version(value) else value.strip()
5330
+
5331
+
5332
+ def _python_version_file(manifest: str | None, entries: list[str],
5333
+ read: Callable[[str], str | None]) -> tuple[str, str] | None:
5334
+ """(version, path): the `__version__` literal of an entry __init__.py,
5335
+ then of the _version.py beside it, then setup.py's `version=` literal
5336
+ beside the manifest."""
5337
+ inits = [e for e in entries if e.rsplit("/", 1)[-1] == "__init__.py"]
5338
+ folder = posixpath.dirname(manifest or "")
5339
+ candidates = [(path, "py") for path in inits]
5340
+ candidates += [(posixpath.join(posixpath.dirname(path), "_version.py"), "py") for path in inits]
5341
+ if posixpath.basename(manifest or "") != "setup.py":
5342
+ candidates.append((posixpath.join(folder, "setup.py"), "setup.py"))
5343
+ for path, kind in dict.fromkeys(candidates):
5344
+ text = read(path)
5345
+ if text is None:
5346
+ continue
5347
+ if kind == "py":
5348
+ m = _PY_VERSION_RE.search(text)
5349
+ version = _version_value(m.group(2)) if m else None
5350
+ else:
5351
+ version = _version_value(parse_setup_py(text).get("version"))
5352
+ if version:
5353
+ return version, path
5354
+ return None
5355
+
5356
+
5357
+ def _cargo_workspace_version(manifest: str | None, read: Callable[[str], str | None]) -> tuple[str, str] | None:
5358
+ """(version, path): the `[workspace.package] version` of the workspace
5359
+ root, the nearest Cargo.toml at or above the manifest's folder that
5360
+ holds a `[workspace]` table."""
5361
+ if not manifest:
5362
+ return None
5363
+ folder = posixpath.dirname(manifest)
5364
+ while True:
5365
+ path = posixpath.join(folder, "Cargo.toml")
5366
+ text = read(path)
5367
+ try:
5368
+ data = tomllib.loads(text) if text is not None else None
5369
+ except tomllib.TOMLDecodeError:
5370
+ data = None
5371
+ workspace = data.get("workspace") if isinstance(data, dict) else None
5372
+ if isinstance(workspace, dict):
5373
+ package = workspace.get("package")
5374
+ version = _version_value(package.get("version")) if isinstance(package, dict) else None
5375
+ return (version, path) if version else None
5376
+ if not folder:
5377
+ return None
5378
+ folder = posixpath.dirname(folder)
5379
+
5380
+
5381
+ def _release_tag(args: argparse.Namespace, warnings: list[str]) -> tuple[str, str] | None:
5382
+ """(version, tag) of the tag a version is read from: with --repo, the
5383
+ `release` skf-github-probe.py's tag listing gives at --ref (the newest
5384
+ release tag at the default branch, the pinned tag itself at a tag);
5385
+ without, the tag `git describe --tags --abbrev=0` names in
5386
+ --source-root, when its name is a version. The version drops a
5387
+ leading v."""
5388
+ try:
5389
+ probe = _sibling("skf-github-probe.py")
5390
+ except RunnerError as exc:
5391
+ warnings.append(f"--resolve-version could not read tags: {exc}")
5392
+ return None
5393
+ if args.repo is not None:
5394
+ parsed = probe.parse_repo(args.repo)
5395
+ if parsed is None:
5396
+ warnings.append(f"--resolve-version read no tags: {args.repo} is not a github.com repository")
5397
+ return None
5398
+ probe._start_clock(probe.DEFAULT_TIMEOUT_SEC)
5399
+ out = probe.probe_tags(*parsed, [], 1, ref=args.ref)
5400
+ if out["status"] != "ok":
5401
+ warnings.append(f"the tags of {args.repo} could not be listed ({out['message']}), so no release "
5402
+ f"tag gave a version")
5403
+ return None
5404
+ release = out["release"]
5405
+ return (release["version"], release["tag"]) if release else None
5406
+ try:
5407
+ res = _sibling("skf-source-tree.py")._git(Path(args.source_root), "describe", "--tags", "--abbrev=0")
5408
+ except RunnerError:
5409
+ return None
5410
+ if res is None or res[0] != 0:
5411
+ return None
5412
+ tag = res[1].decode("utf-8", errors="replace").strip()
5413
+ version = probe.tag_version(tag)
5414
+ return (version, tag) if version else None
5415
+
5416
+
5417
+ def resolve_version(args: argparse.Namespace, result: dict, tree: list[str] | None) -> None:
5418
+ """--resolve-version: when the manifest states no version, look it up
5419
+ (Python: _python_version_file; Rust: _cargo_workspace_version; then any
5420
+ language: _release_tag), and for JavaScript or TypeScript read from
5421
+ --repo at the default branch, let the newest release replace a manifest
5422
+ pre-release (-alpha, -beta, -rc, -next, -canary). Sets `version` and
5423
+ `version_source`: "manifest", "file:<path>", "tag:<tag>", or null with
5424
+ a warning when nothing gave one. Raises QuickInputError."""
5425
+ warnings = result["warnings"]
5426
+ version = result["version"]
5427
+ source = "manifest" if version is not None else None
5428
+ manifest = Path(args.manifest_file).as_posix() if args.manifest_file is not None else None
5429
+ if version is None:
5430
+ read = _version_reader(args, tree)
5431
+ found = None
5432
+ if result["language"] == "python":
5433
+ found = _python_version_file(manifest, [Path(e).as_posix() for e in args.entry_file or []], read)
5434
+ elif result["language"] == "rust":
5435
+ found = _cargo_workspace_version(manifest, read)
5436
+ if found:
5437
+ version, source = found[0], f"file:{found[1]}"
5438
+ prerelease = (version is not None and source == "manifest" and result["language"] in JS_FAMILY
5439
+ and args.repo is not None and (args.ref or "").strip().lower() in ("", "null", "none", "head")
5440
+ and _JS_PRERELEASE_RE.search(version) is not None)
5441
+ if version is None or prerelease:
5442
+ tag = _release_tag(args, warnings)
5443
+ if tag and prerelease:
5444
+ warnings.append(f"manifest version {version!r} is a pre-release on the default branch; version is "
5445
+ f"the newest release, {tag[0]} (tag {tag[1]})")
5446
+ if tag:
5447
+ version, source = tag[0], f"tag:{tag[1]}"
5448
+ if version is None:
5449
+ warnings.append("no version found in the manifest, a version file or a release tag; version is null")
5450
+ result["version"] = version
5451
+ result["version_source"] = source
5452
+
5453
+
5454
+ _JS_LIST_SOURCE_RE = re.compile(r"\s*from\s*(['\"])")
5455
+ _JS_IMPORT_RE = re.compile(r"^import\s+(?:type\s+)?([A-Za-z_$*{][^;'\"]*?)\s*from\s*(['\"])", re.M)
5456
+ _JS_BRACES_RE = re.compile(r"\{([^}]*)\}")
5457
+ DEFINED_IN_FAMILIES = {"python": "python", "js": "js", "ts": "js", "javascript": "js", "typescript": "js"}
5458
+
5459
+
5460
+ def _quick_js_facts(content: str) -> dict:
5461
+ """What one JS/TS module exports, read from its text as quick mode reads
5462
+ an entry file, in _js_facts's shape: {"names": {exposed name: {"how",
5463
+ "local", "source"}}, "stars": [source], "default": the local name of
5464
+ its default export, or None}. A declaration beats a list item, and an
5465
+ exported binding an import brings in is that import's re-export (or
5466
+ namespace, for `import * as`)."""
5467
+ code = _blank(content, "js")
5468
+ imports: dict[str, tuple[str, str]] = {}
5469
+ for m in _JS_IMPORT_RE.finditer(code):
5470
+ source = _js_string(content, code, m.end(2) - 1)
5471
+ braces = _JS_BRACES_RE.search(m.group(1))
5472
+ for piece in braces.group(1).split(",") if braces else []:
5473
+ words = piece.split()
5474
+ if words and _JS_NAME_RE.fullmatch(words[-1]):
5475
+ imports.setdefault(words[-1], (_specifier_local(piece), source))
5476
+ for part in _JS_BRACES_RE.sub("", m.group(1)).split(","):
5477
+ words = part.split()
5478
+ if len(words) == 3 and words[:2] == ["*", "as"]:
5479
+ imports.setdefault(words[2], ("*", source))
5480
+ elif len(words) == 1 and _JS_NAME_RE.fullmatch(words[0]):
5481
+ imports.setdefault(words[0], ("default", source))
5482
+ names: dict[str, dict] = {}
5483
+
5484
+ def put(name: str, how: str, local: str | None, source: str | None) -> None:
5485
+ names.setdefault(name, {"how": how, "local": local, "source": source})
5486
+
5487
+ def bound(name: str, local: str) -> None:
5488
+ imported, source = imports.get(local, (None, None))
5489
+ if imported == "*":
5490
+ put(name, "namespace", None, source)
5491
+ elif imported is not None:
5492
+ put(name, "re-export", imported, source)
5493
+ else:
5494
+ put(name, "declaration", local, None)
5495
+
5496
+ for export in scan_exports_js(content, "", None, None, defaults=False):
5497
+ if export["type"] not in ("re-export", "namespace"):
5498
+ put(export["name"], "declaration", export["name"], None)
5499
+ default = None
5500
+ for m in _JS_DECL_RE.finditer(code):
5501
+ if m.group(1) and "default" in m.group(1).split():
5502
+ default = default or m.group(3)
5503
+ put("default", "declaration", m.group(3), None)
5504
+ elif m.group(1) and _js_type(m.group(2)) == "namespace":
5505
+ put(m.group(3), "declaration", m.group(3), None) # `export namespace X`, `export module X`
5506
+ stars: list[str] = []
5507
+ for m in _JS_STAR_RE.finditer(code):
5508
+ source = _js_string(content, code, m.end(2) - 1)
5509
+ if m.group(1):
5510
+ put(m.group(1), "namespace", None, source)
5511
+ else:
5512
+ stars.append(source)
5513
+ for m in _JS_LIST_RE.finditer(code):
5514
+ after = _JS_LIST_SOURCE_RE.match(code, m.end())
5515
+ source = _js_string(content, code, after.end(1) - 1) if after else None
5516
+ for piece in m.group(1).split(","):
5517
+ words = piece.split()
5518
+ if not words or not _JS_NAME_RE.fullmatch(words[-1]):
5519
+ continue
5520
+ if source is not None:
5521
+ put(words[-1], "re-export", _specifier_local(piece), source)
5522
+ else:
5523
+ bound(words[-1], _specifier_local(piece))
5524
+ for m in _JS_DEFAULT_RE.finditer(code):
5525
+ if m.group(1) not in _JS_NOT_NAMES:
5526
+ default = default or m.group(1)
5527
+ bound("default", m.group(1))
5528
+ return {"names": names, "stars": stars, "default": default}
5529
+
5530
+
5531
+ def _js_defined_in(root: Path, tree: set[str]) -> Callable[[str, str], str | None]:
5532
+ """(module, name it exports) -> the file that defines that name, as
5533
+ _js_surface's resolve traces it: through each re-export, imported
5534
+ binding and `export *` (a cycle ends the trace at the module it
5535
+ reached), a namespace export giving its module's file; None for a
5536
+ re-export or a namespace whose module no file under `root` holds."""
5537
+ cache: dict[str, dict] = {}
5538
+
5539
+ def facts(file: str) -> dict:
5540
+ if file not in cache:
5541
+ cache[file] = _quick_js_facts(_read_text(root, file) or "")
5542
+ return cache[file]
5543
+
5544
+ def owner(file: str, name: str) -> str | None:
5545
+ """The first module the `export *` chain of `file` passes `name` on from."""
5546
+ stack, visited = [file], set()
5547
+ while stack:
5548
+ module = stack.pop()
5549
+ if module in visited:
5550
+ continue
5551
+ visited.add(module)
5552
+ if module != file and name in facts(module)["names"]:
5553
+ return module
5554
+ stack += reversed([t for t in (resolve_js_module(s, module, tree) for s in facts(module)["stars"]) if t])
5555
+ return None
5556
+
5557
+ def resolve(file: str, name: str) -> str | None:
5558
+ if name not in facts(file)["names"] and name == facts(file)["default"]:
5559
+ name = "default" # quick mode names a default export by what it declares
5560
+ visited: set[tuple[str, str]] = set()
5561
+ while (file, name) not in visited:
5562
+ visited.add((file, name))
5563
+ fact = facts(file)["names"].get(name)
5564
+ if fact is None:
5565
+ found = owner(file, name) if name != "default" else None
5566
+ if found is None:
5567
+ return file
5568
+ file = found
5569
+ continue
5570
+ if fact["how"] == "declaration":
5571
+ return file
5572
+ target = resolve_js_module(fact["source"], file, tree)
5573
+ if fact["how"] == "namespace" or target is None:
5574
+ return target
5575
+ file, name = target, fact["local"]
5576
+ return file
5577
+
5578
+ return resolve
5579
+
5580
+
5581
+ def _py_defined_in(root: Path, tree: set[str]) -> Callable[[str, str], str | None]:
5582
+ """(module, name it exports) -> the file full mode's _py_surface traces
5583
+ that name to from that module (None when it gives none: a name another
5584
+ package passes on, or one the trace does not list)."""
5585
+ traced: dict[str, dict[str, str | None]] = {}
5586
+
5587
+ def resolve(file: str, name: str) -> str | None:
5588
+ if file not in traced:
5589
+ surface = _Surface()
5590
+ _py_surface(root, [{"file": file}], tree, surface, [])
5591
+ traced[file] = {n: info["file"] for n, info in surface.names.items()}
5592
+ return traced[file].get(name)
5593
+
5594
+ return resolve
5595
+
5596
+
5597
+ def _defined_in(root: Path, result: dict) -> None:
5598
+ """Give each export of a Python, JavaScript or TypeScript quick result
5599
+ its `defined_in` (see "Quick mode" above): the file the export is
5600
+ declared in for a name its file binds itself, else the file the trace
5601
+ reaches. The tree under `root` is listed only when a name is traced."""
5602
+ family = DEFINED_IN_FAMILIES.get(result.get("language") or "")
5603
+ if family is None:
5604
+ return
5605
+ resolve: Callable[[str, str], str | None] | None = None
5606
+ for export in result["exports"]:
5607
+ if export["type"] not in ("re-export", "namespace", "module"):
5608
+ export["defined_in"] = export["source_file"]
5609
+ continue
5610
+ if resolve is None:
5611
+ tree = set(list_tree(root))
5612
+ resolve = (_py_defined_in if family == "python" else _js_defined_in)(root, tree)
5613
+ export["defined_in"] = resolve(export["source_file"], export["name"])
5614
+
5615
+
4482
5616
  def _main_entries(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
4483
5617
  """`--mode entries`: the entry files of one package, as JSON on stdout."""
4484
5618
  if args.manifest_file is not None or args.entry_file or args.follow_file:
@@ -4517,9 +5651,20 @@ def main(argv: list[str]) -> int:
4517
5651
  args = parser.parse_args(argv)
4518
5652
  files = args.manifest_file is not None or bool(args.entry_file) or bool(args.follow_file)
4519
5653
  listing = [f"--{flag.replace('_', '-')}" for flag in LISTING_FLAGS if getattr(args, flag) is not None]
4520
- following = [f"--{flag}" for flag in FOLLOW_FLAGS if getattr(args, flag) not in (None, False)]
5654
+ following = [f"--{flag.replace('_', '-')}" for flag in FOLLOW_FLAGS if getattr(args, flag) not in (None, False)]
4521
5655
  if following and (args.mode != "quick" or not files):
4522
5656
  parser.error(f"{', '.join(following)}: quick mode only, with --manifest-file or --entry-file")
5657
+ own = [f"--{flag}" for flag in DENOMINATOR_OWN_FLAGS if getattr(args, flag) is not None]
5658
+ if own and args.mode != "denominator":
5659
+ parser.error(f"{', '.join(own)}: denominator mode only (--mode denominator)")
5660
+ if args.mode == "denominator":
5661
+ other = [f"--{flag.replace('_', '-')}" for flag in FULL_MODE_FLAGS
5662
+ if getattr(args, flag) is not None and flag not in DENOMINATOR_FLAGS]
5663
+ listing = [flag for flag in listing if flag != "--tree-file"]
5664
+ if files or listing or other:
5665
+ parser.error(f"{', '.join(other + listing) or '--manifest-file, --entry-file, --follow-file'}: "
5666
+ "not denominator mode's")
5667
+ return _main_denominator(args)
4523
5668
  if args.mode == "full":
4524
5669
  if files:
4525
5670
  parser.error("--manifest-file, --entry-file, --follow-file: quick mode only (--mode quick)")
@@ -4543,17 +5688,24 @@ def main(argv: list[str]) -> int:
4543
5688
  if files:
4544
5689
  if not args.language or len(args.language) != 1:
4545
5690
  parser.error("--manifest-file and --entry-file take one --language")
4546
- if (args.repo is not None or args.ref is not None) and not args.follow:
4547
- parser.error("--repo and --ref fetch the modules --follow reads, so pass --follow")
5691
+ if (args.repo is not None or args.ref is not None) and not (args.follow or args.resolve_version):
5692
+ parser.error("--repo and --ref fetch the modules --follow reads and name the tags --resolve-version "
5693
+ "reads, so pass --follow or --resolve-version")
4548
5694
  if args.ref is not None and args.repo is None:
4549
5695
  parser.error("--ref is the ref of --repo, so pass --repo")
4550
5696
  if args.follow and (args.tree_file is None or args.source_root is None):
4551
5697
  parser.error("--follow finds each module in --tree-file and reads it under --source-root: pass both")
5698
+ if args.resolve_version and args.source_root is None:
5699
+ parser.error("--resolve-version reads the version files under --source-root: pass it")
4552
5700
  try:
4553
5701
  payload = _quick_files_payload(args)
4554
5702
  result = extract(payload)
4555
5703
  if args.follow and "_error" not in result:
4556
5704
  result = _follow(args, result)
5705
+ if args.resolve_version and "_error" not in result:
5706
+ resolve_version(args, result, payload.get("tree"))
5707
+ if args.source_root is not None and "_error" not in result:
5708
+ _defined_in(Path(args.source_root), result)
4557
5709
  if args.fetch_list is not None and "_error" not in result:
4558
5710
  follow = [item["module_file"] for item in result["unlisted"] if item["module_file"]]
4559
5711
  _write_fetch_list(args.fetch_list, list(dict.fromkeys(follow)))