bmad-module-skill-forge 3.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/docs/_data/pinned.yaml +1 -1
- package/docs/examples.md +1 -1
- package/docs/troubleshooting.md +8 -0
- package/docs/verifying-a-skill.md +7 -3
- package/docs/workflows.md +3 -3
- package/package.json +1 -1
- package/src/knowledge/qmd-registry.md +1 -1
- package/src/knowledge/version-paths.md +1 -1
- package/src/shared/health-check.md +2 -50
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +2 -2
- package/src/shared/scripts/skf-atomic-write.py +212 -59
- package/src/shared/scripts/skf-build-change-manifest.py +593 -47
- package/src/shared/scripts/skf-classify-changed-files.py +78 -0
- package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
- package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
- package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
- package/src/shared/scripts/skf-extract-public-api.py +1309 -157
- package/src/shared/scripts/skf-extraction-inventory.py +580 -46
- package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
- package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
- package/src/shared/scripts/skf-github-probe.py +47 -8
- package/src/shared/scripts/skf-hash-content.py +14 -8
- package/src/shared/scripts/skf-load-provenance.py +3 -1
- package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
- package/src/shared/scripts/skf-preflight.py +15 -1
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
- package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
- package/src/shared/scripts/skf-structural-diff.py +133 -18
- package/src/shared/scripts/skf-update-active-symlink.py +115 -62
- package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
- package/src/shared/scripts/skf-verify-provenance-completeness.py +977 -49
- package/src/shared/scripts/skf-write-skill-brief.py +41 -8
- package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
- package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
- package/src/skf-audit-skill/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +2 -1
- package/src/skf-audit-skill/references/re-index.md +14 -9
- package/src/skf-audit-skill/references/report.md +2 -2
- package/src/skf-audit-skill/references/semantic-diff.md +15 -15
- package/src/skf-audit-skill/references/structural-diff.md +10 -5
- package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
- package/src/skf-brief-skill/SKILL.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +1 -1
- package/src/skf-brief-skill/assets/skill-brief-schema.md +2 -9
- package/src/skf-brief-skill/references/analyze-target.md +6 -6
- package/src/skf-brief-skill/references/confirm-brief.md +1 -1
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
- package/src/skf-brief-skill/references/version-resolution.md +5 -9
- package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
- package/src/skf-create-skill/assets/skill-sections.md +6 -6
- package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
- package/src/skf-create-skill/references/compile.md +31 -60
- package/src/skf-create-skill/references/component-extraction.md +3 -3
- package/src/skf-create-skill/references/enrich.md +2 -2
- package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
- package/src/skf-create-skill/references/extract.md +22 -19
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
- package/src/skf-create-skill/references/extraction-patterns.md +8 -9
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +16 -5
- package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
- package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
- package/src/skf-drop-skill/references/execute.md +5 -1
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +2 -2
- package/src/skf-drop-skill/references/select.md +7 -3
- package/src/skf-test-skill/references/coherence-check.md +1 -1
- package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
- package/src/skf-test-skill/references/coverage-check.md +39 -57
- package/src/skf-test-skill/references/init.md +3 -5
- package/src/skf-test-skill/references/report.md +10 -8
- package/src/skf-test-skill/references/scoring-rules.md +6 -5
- package/src/skf-test-skill/references/source-access-protocol.md +14 -17
- package/src/skf-test-skill/scripts/build-result-context.py +114 -21
- package/src/skf-test-skill/scripts/gap-ledger.py +315 -30
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +534 -96
- package/src/skf-test-skill/scripts/reconcile-coverage.py +276 -2
- package/src/skf-test-skill/scripts/score-signatures.py +36 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
- package/src/skf-update-skill/references/detect-changes.md +32 -13
- package/src/skf-update-skill/references/gap-driven.md +3 -3
- package/src/skf-update-skill/references/init.md +6 -2
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +2 -2
- package/src/skf-update-skill/references/re-extract.md +2 -2
- package/src/skf-update-skill/references/report.md +3 -1
- package/src/skf-update-skill/references/write.md +43 -23
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
# ///
|
|
5
5
|
"""SKF Extract Public API: one package's public surface, by parser or by recipe.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Four modes:
|
|
8
8
|
|
|
9
9
|
--mode quick (the default) A pure parser: takes one logical package (one
|
|
10
10
|
manifest, one or more entry-point files) and emits its
|
|
@@ -16,9 +16,10 @@ Three modes:
|
|
|
16
16
|
raw fetches into a folder, or a local checkout), so a file's
|
|
17
17
|
text never passes through the model or a shell string.
|
|
18
18
|
--follow also reads the modules an `export *` chain passes
|
|
19
|
-
names on from, fetching them first with --repo
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
names on from, fetching them first with --repo, and
|
|
20
|
+
--resolve-version looks up a version the manifest does
|
|
21
|
+
not state. Multi-module monorepos are caller-orchestrated:
|
|
22
|
+
invoke once per module, aggregate.
|
|
22
23
|
--mode full The ast-grep recipe runner (skf-create-skill and the
|
|
23
24
|
workflows that extract as it does): runs the recipes in
|
|
24
25
|
src/shared/data/ast-grep-recipes.yaml over a source tree
|
|
@@ -30,6 +31,11 @@ Three modes:
|
|
|
30
31
|
rules, so no caller derives an entry file from a
|
|
31
32
|
distribution name or reads a manifest by eye. See "Entries
|
|
32
33
|
mode" below.
|
|
34
|
+
--mode denominator
|
|
35
|
+
Full mode's counts and effective_denominator decision for
|
|
36
|
+
public names counted without its diff (by quick mode or by
|
|
37
|
+
eye), and its scope facts with no recipe run. See
|
|
38
|
+
"Denominator mode" below.
|
|
33
39
|
|
|
34
40
|
Quick mode
|
|
35
41
|
----------
|
|
@@ -120,7 +126,7 @@ or the files, read from disk (stdin is then never read):
|
|
|
120
126
|
--mode quick --language <language> [--source-root <dir>]
|
|
121
127
|
[--manifest-file <path>] [--entry-file <path>]...
|
|
122
128
|
[--follow-file <path>]... [--tree-file <listing>] [--fetch-list <file>]
|
|
123
|
-
[--follow [--repo <owner/repo> [--ref <ref>]]
|
|
129
|
+
[--follow] [--resolve-version] [--repo <owner/repo> [--ref <ref>]]
|
|
124
130
|
|
|
125
131
|
An entry with `followed` true (a --follow-file) is a module an `unlisted[]`
|
|
126
132
|
record named, read as an entry is, less a JS/TS `export default`, which
|
|
@@ -138,6 +144,41 @@ A module that cannot be fetched or read stays in `unlisted[]`, and a
|
|
|
138
144
|
warning names it and why. The output then also lists `followed`, the
|
|
139
145
|
modules read this way, in the order they were read.
|
|
140
146
|
|
|
147
|
+
--resolve-version (with --source-root) looks up a version the manifest does
|
|
148
|
+
not state, in this order, and names where it came from in `version_source`:
|
|
149
|
+
|
|
150
|
+
python the `__version__` literal of an entry `__init__.py`, then of the
|
|
151
|
+
`_version.py` beside it, then the `version=` literal of the
|
|
152
|
+
setup.py beside the manifest (a dynamic version, `attr:` or
|
|
153
|
+
`file:`, or a version setup.py computes)
|
|
154
|
+
rust the `[workspace.package] version` of the workspace root, the
|
|
155
|
+
nearest Cargo.toml at or above the manifest's folder that holds
|
|
156
|
+
a `[workspace]` table (`version.workspace = true`)
|
|
157
|
+
any the release tag: with --repo, skf-github-probe.py's tag listing
|
|
158
|
+
at --ref (the newest tag whose name is a release version, X.Y.Z,
|
|
159
|
+
at the default branch; the pinned tag itself at a tag; none at a
|
|
160
|
+
branch or a commit); without, the tag `git describe --tags
|
|
161
|
+
--abbrev=0` names in --source-root, when its name is a version
|
|
162
|
+
|
|
163
|
+
A file the lookups read is read under --source-root; with --repo and
|
|
164
|
+
--tree-file, one the listing holds is fetched there first, at --ref. A
|
|
165
|
+
version a file gives must start with a digit and be no placeholder, and
|
|
166
|
+
one a tag gives drops a leading `v`. For JavaScript and TypeScript read
|
|
167
|
+
with --repo at the default branch, a manifest pre-release (`-alpha`,
|
|
168
|
+
`-beta`, `-rc`, `-next`, `-canary`) gives way to the newest release, with
|
|
169
|
+
a warning. When nothing gives a version, a warning says so.
|
|
170
|
+
|
|
171
|
+
With --source-root, each Python, JavaScript and TypeScript export also
|
|
172
|
+
carries `defined_in`: the file that defines it, traced as full mode traces
|
|
173
|
+
an entry point (through each import, re-export, `export *` and star import
|
|
174
|
+
to the end of its chain, a cycle cut), so a name an entry file only passes
|
|
175
|
+
on names its own module. It reads only files under --source-root (the
|
|
176
|
+
files full mode lists there, never the --tree-file listing) and needs no
|
|
177
|
+
ast-grep: Python with full mode's module reader, JavaScript and TypeScript
|
|
178
|
+
from each module's text as quick mode reads an entry file. It is null when
|
|
179
|
+
the trace finds no file, as for a name passed on from another package or a
|
|
180
|
+
module --source-root does not hold. `source_file` stays the file read.
|
|
181
|
+
|
|
141
182
|
Each path is relative to --source-root, the folder the files were staged in
|
|
142
183
|
laid out like the repository (or a local checkout), else to the current
|
|
143
184
|
folder, and is the `path` the output names (`source_file`), written with
|
|
@@ -155,9 +196,20 @@ Output JSON shape (stdout):
|
|
|
155
196
|
{
|
|
156
197
|
"language": "python",
|
|
157
198
|
"package_name": "foo",
|
|
158
|
-
"version": "1.2.3",
|
|
199
|
+
"version": "1.2.3", (the manifest's literal
|
|
200
|
+
version, else null: a
|
|
201
|
+
go.mod, a dynamic or
|
|
202
|
+
workspace-inherited one,
|
|
203
|
+
a placeholder; with
|
|
204
|
+
--resolve-version, the
|
|
205
|
+
one it looked up)
|
|
206
|
+
"version_source": "manifest" | "file:<path>" | "tag:<tag>" | null,
|
|
207
|
+
(--resolve-version only)
|
|
159
208
|
"description": "...",
|
|
160
|
-
"exports": [{"name": "Bar", "type": "class", "source_file": "..."
|
|
209
|
+
"exports": [{"name": "Bar", "type": "class", "source_file": "...",
|
|
210
|
+
"defined_in": "..."}, ...], (defined_in: Python,
|
|
211
|
+
JS and TS, with
|
|
212
|
+
--source-root)
|
|
161
213
|
"dependencies": ["requests", "pydantic", ...],
|
|
162
214
|
"modules": ["server", "client"], (Maven/Gradle only)
|
|
163
215
|
"module_folders": ["server", "client"], (each module's folder in
|
|
@@ -193,10 +245,12 @@ Exit codes (quick):
|
|
|
193
245
|
or given twice with the file inputs, a --manifest-file, --entry-file
|
|
194
246
|
or --follow-file that cannot be read or leads outside --source-root,
|
|
195
247
|
a --tree-file that cannot be read or a --fetch-list that cannot be
|
|
196
|
-
written, --follow without --tree-file and --source-root,
|
|
197
|
-
--
|
|
198
|
-
|
|
199
|
-
|
|
248
|
+
written, --follow without --tree-file and --source-root,
|
|
249
|
+
--resolve-version without --source-root, --repo or --ref without
|
|
250
|
+
--follow or --resolve-version (--ref without --repo), or a --repo or
|
|
251
|
+
--ref skf-github-fetch.py does not accept. A module --follow could
|
|
252
|
+
not read, and a version --resolve-version could not find, change no
|
|
253
|
+
exit code.
|
|
200
254
|
|
|
201
255
|
Entries mode
|
|
202
256
|
------------
|
|
@@ -276,7 +330,15 @@ is one the recipes read (python, typescript, tsx, javascript, rust, go and
|
|
|
276
330
|
.vue files) and --language allows it. Globs follow
|
|
277
331
|
skf-resolve-authoritative-files.py's rules: `**` spans any number of path
|
|
278
332
|
segments, none included, and `*` and `?` stay inside one, so `src/**/*.ts`
|
|
279
|
-
matches `src/index.ts` and `**/test_*` matches a top-level `test_x.py`.
|
|
333
|
+
matches `src/index.ts` and `**/test_*` matches a top-level `test_x.py`. An
|
|
334
|
+
include glob that matches no file (a brief written for an older layout,
|
|
335
|
+
say) is named in `warnings` and listed in `scope.unmatched_include`. A
|
|
336
|
+
scope.tier_a_include glob is matched against the whole tree under
|
|
337
|
+
--source-root, regardless of --files-from and before the excludes: one
|
|
338
|
+
that matches no file is named in `warnings` too and listed in
|
|
339
|
+
`scope.unmatched_tier_a_include`, and, when scope.include has a glob, one
|
|
340
|
+
that matches only files no scope.include glob matches is named in
|
|
341
|
+
`warnings`.
|
|
280
342
|
|
|
281
343
|
--brief supplies scope.include, scope.exclude, scope.tier_a_include,
|
|
282
344
|
scope.type and language; --include, --exclude, --tier-a-include,
|
|
@@ -337,11 +399,27 @@ JS/TS one that is not). Chains are followed to the end, a cycle cut: an
|
|
|
337
399
|
`export *`, a star import or a glob passes on what its module exports,
|
|
338
400
|
that module's own chains included; a re-exported name is traced through
|
|
339
401
|
each re-export, import and glob between to the file that defines it (a
|
|
340
|
-
default export to the name it declares). A
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
402
|
+
default export to the name it declares). A Python package's explicit
|
|
403
|
+
binding of a name (an import, or a def or class in its __init__.py)
|
|
404
|
+
comes before its submodule: `from .x import y` imports the y
|
|
405
|
+
x/__init__.py binds, and the submodule x/y.py only when x/__init__.py
|
|
406
|
+
binds no y or binds the submodule itself (`from . import y`, `from
|
|
407
|
+
pkg.x import y`); a y x/__init__.py binds only through a star import
|
|
408
|
+
(`from .impl import *`) is still read as the submodule x/y.py. A
|
|
409
|
+
namespace (`export * as ns`, a Python submodule, a Rust `pub mod`)
|
|
410
|
+
lists in `members` the names its module passes on, which users reach as
|
|
411
|
+
`pkg.ns.name`, each traced to the file that defines it as a public name
|
|
412
|
+
is, and in place of a namespace among them its members (each module once,
|
|
413
|
+
so a cycle is cut); `members` is null when the trace cannot read the
|
|
414
|
+
module (a JavaScript target no file here holds, a Rust inline or
|
|
415
|
+
`#[path]` module), and a nested one the trace cannot read is a member
|
|
416
|
+
itself. A member re-exported from another package has no file, is no
|
|
417
|
+
gap when a recipe found its name, marks its name public, and, unlike the
|
|
418
|
+
same form in an entry point, adds no `unresolved` item. What a namespace
|
|
419
|
+
makes reachable is not internal: the names its module passes on, the
|
|
420
|
+
modules among them in turn, and for Rust every recipe match in the crate
|
|
421
|
+
root and in each module file a `pub mod` or `pub use` of a reachable
|
|
422
|
+
module names.
|
|
345
423
|
A chain the trace cannot follow (a module no file here holds, a `#[path]`
|
|
346
424
|
module) is listed in `unresolved`. The diff against the recipe inventory,
|
|
347
425
|
one language family at a time (JavaScript/TypeScript, Python, Rust, Go):
|
|
@@ -350,21 +428,59 @@ that no entry point exports or makes reachable), `extraction_gaps` (names
|
|
|
350
428
|
an entry point exports that no recipe found, defined in a file in scope or
|
|
351
429
|
in none the trace reached: read them by eye) and `outside_scope` (such
|
|
352
430
|
names defined in a file out of scope: widen scope.include to document
|
|
353
|
-
them).
|
|
431
|
+
them). A namespace with `members` is diffed as its members: each is found
|
|
432
|
+
when a recipe found its name, or its `local` one, in its file (one with
|
|
433
|
+
no file by its name alone, as a public name is), and has no row of its
|
|
434
|
+
own when a public name (or another namespace's member) has its name and
|
|
435
|
+
file; one with null `members`, and a member that is such a module, is no
|
|
436
|
+
gap. In a family with no entry point, or only empty ones, every name the
|
|
354
437
|
recipes found is public.
|
|
355
438
|
|
|
356
439
|
Counts. `exports_public_api` counts the public names (the package's whole
|
|
357
|
-
public API, `outside_scope` names included) and
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
440
|
+
public API, `outside_scope` names included), each distinct name and file
|
|
441
|
+
once: a namespace with `members` counts as its members (none for an empty
|
|
442
|
+
list, and a member that is also a public name, or another namespace's
|
|
443
|
+
member, once), one with null `members` as one. `exports_internal` counts
|
|
444
|
+
the internal names. `effective_denominator` counts those public names
|
|
445
|
+
whose definition file matches scope.include (any file when it has no
|
|
446
|
+
glob), scope.tier_a_include when given, and no scope.exclude glob: each
|
|
447
|
+
name once, so a type counts once whatever its methods, a barrel that
|
|
448
|
+
only re-exports adds nothing of its own, and a name the entry points
|
|
449
|
+
pass on from two files (a namespace member named like another public
|
|
450
|
+
name, or like another namespace's member, at another file) or in two
|
|
451
|
+
languages counts once, when any one of its files matches, as Test
|
|
452
|
+
Skill's sets count it. A tier A list that counts no public name (each
|
|
453
|
+
glob matches no file, or only files scope.exclude drops, that
|
|
454
|
+
scope.include leaves out or that define no public name) narrows nothing
|
|
455
|
+
on a run that counted a public name and read every file (no `errors`
|
|
456
|
+
item): the count falls back to scope.include (else every file), with a
|
|
457
|
+
warning.
|
|
458
|
+
`arms` gives compile's effective_denominator arms:
|
|
362
459
|
(a) `monorepo` from skf-detect-workspaces.py, (b) `specific_modules`
|
|
363
460
|
(scope type specific-modules on a repository that is not a monorepo) and
|
|
364
461
|
(c) `multi_subpath_exports` (an in-scope package.json whose `exports` map
|
|
365
462
|
has more than one non-root subpath without a `*`, on a repository that is
|
|
366
463
|
not a monorepo).
|
|
367
464
|
|
|
465
|
+
Store or not. `curated_subset` is true when the files the count's basis
|
|
466
|
+
globs list (scope.include's, else every file; a tier A count's, those
|
|
467
|
+
among them a tier A glob matches too; under arm (b), less those a
|
|
468
|
+
scope.exclude glob matches) leave out a file that defines a counted public
|
|
469
|
+
name (an `outside_scope` name's included) or, in a monorepo, every source
|
|
470
|
+
file of a skf-detect-workspaces.py workspace; a basis of `all-files` lists
|
|
471
|
+
none. `effective_denominator_decision` says whether create-skill and
|
|
472
|
+
update-skill store `effective_denominator` as stats.effective_denominator,
|
|
473
|
+
the brief's scope.notes aside (documenting the stratification is the one
|
|
474
|
+
judgment left to them): `store` is true only when none of these holds,
|
|
475
|
+
and `reason` names the first that does, else `curated-subset`:
|
|
476
|
+
`reference-app` (scope type reference-app: pattern_surfaces_documented is
|
|
477
|
+
its basis), `incomplete-run` (an `errors` item: a file a failed call left
|
|
478
|
+
unread can hide a public name, so the count can be 0 or partial),
|
|
479
|
+
`no-arm` (no arm is true), `full-library` (scope type full-library),
|
|
480
|
+
`all-files` (the basis), `not-curated` (`curated_subset` false) and
|
|
481
|
+
`zero-count` (a stored 0 is no denominator: Test Skill reads it as
|
|
482
|
+
deflation).
|
|
483
|
+
|
|
368
484
|
Output JSON (stdout, or -o):
|
|
369
485
|
|
|
370
486
|
{
|
|
@@ -376,7 +492,9 @@ Output JSON (stdout, or -o):
|
|
|
376
492
|
"ast_grep": {"path": "<exe>" | null, "version": "<x.y.z>" | null},
|
|
377
493
|
"scope": {"include": [...], "exclude": [...],
|
|
378
494
|
"tier_a_include": [...] | null, "type": "<type>" | null,
|
|
379
|
-
"languages": [...], "files_from": "<file>" | null
|
|
495
|
+
"languages": [...], "files_from": "<file>" | null,
|
|
496
|
+
"unmatched_include": [...], # include globs no file matches
|
|
497
|
+
"unmatched_tier_a_include": [...]}, # tier A globs no file matches
|
|
380
498
|
"files_in_scope": N,
|
|
381
499
|
"files_by_language": {"<language>": N, ...},
|
|
382
500
|
"files_without_recipes": {".java": N, ...}, # in scope, no recipe reads them
|
|
@@ -409,35 +527,44 @@ Output JSON (stdout, or -o):
|
|
|
409
527
|
| {"entry", "name", "from", "file"}, ...]},
|
|
410
528
|
"entry_point_diff": {
|
|
411
529
|
"public": [{"name", "language", "entry", "via", "from", "local",
|
|
412
|
-
"file", "line"
|
|
530
|
+
"file", "line",
|
|
531
|
+
"members": [{"name", "local", "file", "line"}, ...]
|
|
532
|
+
| null}, ...], # members: via namespace only
|
|
413
533
|
"internal": [{"name", "language", "source_file", "source_line"}, ...],
|
|
414
534
|
"extraction_gaps": [{"name", "language", "entry", "file", "line"}, ...],
|
|
415
535
|
"outside_scope": [{"name", "language", "entry", "file", "line"}, ...]},
|
|
416
536
|
"reexport_targets": [{"name", "language", "local", "from", "entry",
|
|
417
537
|
"file", "line", "via"}, ...],
|
|
418
|
-
"counts": {"exports_public_api": N,
|
|
419
|
-
"
|
|
538
|
+
"counts": {"exports_public_api": N, # (family, name, file) keys
|
|
539
|
+
"exports_internal": N,
|
|
540
|
+
"effective_denominator": N, # names, each once
|
|
420
541
|
"effective_denominator_basis": "tier_a_include" | "scope.include"
|
|
421
|
-
| "all-files",
|
|
422
|
-
"denominator_files": N
|
|
542
|
+
| "all-files", # not tier_a_include when it counts no scope.include name
|
|
543
|
+
"denominator_files": N,
|
|
544
|
+
"curated_subset": bool,
|
|
545
|
+
"effective_denominator_decision": {"store": bool, "reason": "<reason>"}},
|
|
423
546
|
"arms": {"monorepo": bool, "monorepo_kind": "<kind>" | null,
|
|
424
547
|
"specific_modules": bool, "multi_subpath_exports": bool},
|
|
425
548
|
"errors": [{"reason": "ast-grep-error" | "ast-grep-timeout" | "time-limit",
|
|
426
549
|
"files": N, "first_file", "detail",
|
|
427
|
-
"unread": [...]}, ...], # unread:
|
|
428
|
-
"warnings": ["..."]
|
|
550
|
+
"unread": [...]}, ...], # unread: the files its call(s) may have left unread
|
|
551
|
+
"warnings": ["..."],
|
|
552
|
+
"files_in_scope_list": ["<file>", ...], # the files files_in_scope counts, in tree order
|
|
553
|
+
"files_without_recipes_list": ["<file>", ...] # the files files_without_recipes counts
|
|
429
554
|
}
|
|
430
555
|
|
|
431
556
|
`via` says how an entry point exports a name: `declaration` (defined in the
|
|
432
557
|
entry point), `re-export` (from another file, which `file` names, `local`
|
|
433
558
|
giving the name it has there when it differs), `star` (through `export *`,
|
|
434
559
|
`from .x import *` or `pub use x::*`) or `namespace` (a module: `export *
|
|
435
|
-
as ns`, a Python submodule, a Rust module). `line`
|
|
436
|
-
defines the name (a module, a name from another
|
|
437
|
-
|
|
438
|
-
`
|
|
439
|
-
|
|
440
|
-
|
|
560
|
+
as ns`, a Python submodule, a Rust module, its names in `members`). `line`
|
|
561
|
+
is null where no line defines the name (a module, a name from another
|
|
562
|
+
package), and a member's `file` is null for a name from another package.
|
|
563
|
+
An `unresolved` item names a subpath whose targets name no file, or a
|
|
564
|
+
chain met tracing `entry` that stops at `file`, the module `from` naming
|
|
565
|
+
no file here.
|
|
566
|
+
Without ast-grep the JSON still has the files in scope, the entry points'
|
|
567
|
+
`files`, the `exports_maps`, the unresolved subpaths and the `arms`.
|
|
441
568
|
`file_issues` lists the files to read by eye: a file ast-grep skips (not
|
|
442
569
|
UTF-8), and a file where the parser met code it could not read (an ERROR
|
|
443
570
|
node), where a recipe can miss an export. `t1_low` is 0: every export here
|
|
@@ -449,14 +576,18 @@ under the 10-minute limit an agent's shell call often has) have passed
|
|
|
449
576
|
since the run started: the run going on is stopped, no other starts, and
|
|
450
577
|
an `errors[]` item with the reason `time-limit` lists in `unread` the
|
|
451
578
|
files left unread (each ast-grep call also stops after 600 seconds,
|
|
452
|
-
`ast-grep-timeout`).
|
|
579
|
+
`ast-grep-timeout`). A call that times out keeps no match; one that exited
|
|
580
|
+
keeps the matches it printed before any line that is not JSON, but a kill
|
|
581
|
+
can cut a file's matches anywhere, so the item of every failed call lists
|
|
582
|
+
all its files in `unread`. The JSON is still written, status "incomplete".
|
|
453
583
|
|
|
454
584
|
Exit codes (full):
|
|
455
585
|
|
|
456
586
|
0 the recipes ran on every file in scope ("ok")
|
|
457
|
-
1 an ast-grep
|
|
458
|
-
|
|
459
|
-
incomplete ("incomplete")
|
|
587
|
+
1 an ast-grep call could not start, exited non-zero, printed other
|
|
588
|
+
than JSON lines or timed out, or the --timeout ran out: the result
|
|
589
|
+
is incomplete ("incomplete"), and each `errors` item lists in
|
|
590
|
+
`unread` the files it may have left unread
|
|
460
591
|
2 input error (no source root, an unreadable brief, file list or
|
|
461
592
|
recipe file, a recipe without its metadata, no PyYAML, an -o file
|
|
462
593
|
that cannot be written, a --timeout not above 0), or a failure the
|
|
@@ -466,6 +597,63 @@ Exit codes (full):
|
|
|
466
597
|
beside it (cmd.exe would read a repository path's `&` as syntax);
|
|
467
598
|
the JSON has the scope and no exports; extract by source reading
|
|
468
599
|
|
|
600
|
+
Denominator mode
|
|
601
|
+
----------------
|
|
602
|
+
|
|
603
|
+
--mode denominator [--source-root <dir> | --tree-file <listing>]
|
|
604
|
+
[--brief <skill-brief.yaml>] [--include GLOB]... [--exclude GLOB]...
|
|
605
|
+
[--tier-a-include GLOB]... [--scope-type TYPE] [--language LANG]...
|
|
606
|
+
[--quick <quick.json>]... [--extraction <full.json>] [-o <out.json>]
|
|
607
|
+
< {payload_json}
|
|
608
|
+
|
|
609
|
+
The counts full mode writes, for public names no runner diff gave: those
|
|
610
|
+
a stdin payload lists,
|
|
611
|
+
|
|
612
|
+
{"public": [{"name", "file", "language",
|
|
613
|
+
"members": [{"name", "file"}, ...]}, ...],
|
|
614
|
+
"exports": [{"name", "file", "language"}, ...]}
|
|
615
|
+
|
|
616
|
+
(`file` relative to --source-root, with `/`; `members` for a submodule
|
|
617
|
+
the entry points export by name, which counts as the names its module
|
|
618
|
+
passes on), each export of a --quick output at its `defined_in` (else its
|
|
619
|
+
`source_file`), and the --extraction runner's `entry_point_diff.public`;
|
|
620
|
+
a payload name replaces a --quick or runner one of its family and name.
|
|
621
|
+
`exports_internal` counts the names of the payload's `exports` (the
|
|
622
|
+
exports extraction found; `_` names left out) and of the runner's
|
|
623
|
+
`internal` that no public name or member has. `effective_denominator`,
|
|
624
|
+
its basis, `curated_subset` and `effective_denominator_decision` follow
|
|
625
|
+
full mode's rules, over the tree under --source-root, with the scope and
|
|
626
|
+
arms full mode works out there; an --extraction that is `incomplete` is
|
|
627
|
+
never stored. A name whose `file` is not in that tree counts in
|
|
628
|
+
`exports_public_api` alone, with a warning. With no --source-root (a
|
|
629
|
+
remote source read by eye) the tree is the --tree-file listing (a file,
|
|
630
|
+
or `-` for stdin, which then holds no payload: JSON with a `tree` list, a
|
|
631
|
+
list, or one path per line; one that lists no file is an input error),
|
|
632
|
+
else the names' files; the arms are all false, there is no entry point,
|
|
633
|
+
and the decision is never to store (`no-source-tree`). An empty payload
|
|
634
|
+
(`{}`, or no stdin) gives the scope facts alone, with no recipe run:
|
|
635
|
+
`scope`, the two file lists (by full mode's Which files rule, null with
|
|
636
|
+
neither a tree nor a listing), `entry_points` (`files`, `exports_maps`)
|
|
637
|
+
and `arms`.
|
|
638
|
+
|
|
639
|
+
`entry_point_diff` replaces a compare by eye, each item {"name",
|
|
640
|
+
"language", "file"}: `public`, the names `exports_public_api` counts (a
|
|
641
|
+
namespace as its `members`); `internal`, the exports `exports_internal`
|
|
642
|
+
counts; `extraction_gaps`, the public names the payload and --quick give
|
|
643
|
+
that no export found has (the payload's `exports`, the --extraction
|
|
644
|
+
runner's names), to trace (the runner lists its own gaps).
|
|
645
|
+
|
|
646
|
+
Output JSON: `mode` "denominator", `source_root`, `scope` (as full mode
|
|
647
|
+
writes it), `files_in_scope`, `files_without_recipes`, `entry_points`,
|
|
648
|
+
`arms`, `runner_status` (the --extraction's `status`, else null),
|
|
649
|
+
`counts` (as full mode writes it), `entry_point_diff`, `warnings`, and
|
|
650
|
+
last `files_in_scope_list` and `files_without_recipes_list` (as full
|
|
651
|
+
mode writes them), to -o when given (its folder created). Exit 0, or 2
|
|
652
|
+
on an input error (a --source-root that is no folder, a brief, --quick,
|
|
653
|
+
--extraction or --tree-file that cannot be read, --tree-file beside
|
|
654
|
+
--source-root, a payload that is not this shape, a flag of another
|
|
655
|
+
mode).
|
|
656
|
+
|
|
469
657
|
CLI examples:
|
|
470
658
|
uv run skf-extract-public-api.py --mode quick < {payload_json}
|
|
471
659
|
uv run skf-extract-public-api.py --mode entries --language python \\
|
|
@@ -722,7 +910,7 @@ def parse_package_swift(content: str) -> dict:
|
|
|
722
910
|
"""Best-effort parse of a SwiftPM Package.swift manifest.
|
|
723
911
|
|
|
724
912
|
SwiftPM has no version field in the manifest (versions come from git tags),
|
|
725
|
-
so `version` is always None;
|
|
913
|
+
so `version` is always None; a caller reads it from a git tag.
|
|
726
914
|
"""
|
|
727
915
|
name_m = re.search(r"\bPackage\s*\(\s*name:\s*['\"]([^'\"]+)['\"]", content, re.DOTALL)
|
|
728
916
|
deps: list[str] = []
|
|
@@ -1332,6 +1520,35 @@ def _detect_placeholder_version(version: str | None) -> str | None:
|
|
|
1332
1520
|
return None
|
|
1333
1521
|
|
|
1334
1522
|
|
|
1523
|
+
def _literal_version(version: object, warnings: list[str]) -> str | None:
|
|
1524
|
+
"""The version the manifest writes as a literal string, else None.
|
|
1525
|
+
|
|
1526
|
+
A placeholder and a version that is not a string (Cargo's
|
|
1527
|
+
`version = { workspace = true }`, which the workspace root's Cargo.toml
|
|
1528
|
+
holds) give None with a warning, so the output's `version` is a string
|
|
1529
|
+
or null, which --resolve-version looks up."""
|
|
1530
|
+
if version is None:
|
|
1531
|
+
return None
|
|
1532
|
+
if not isinstance(version, str):
|
|
1533
|
+
if isinstance(version, dict) and version.get("workspace") is True:
|
|
1534
|
+
warnings.append(
|
|
1535
|
+
"manifest version is inherited from the workspace (version.workspace = true), "
|
|
1536
|
+
"which the workspace root's Cargo.toml holds, so the manifest states none"
|
|
1537
|
+
)
|
|
1538
|
+
else:
|
|
1539
|
+
warnings.append(f"manifest version {version!r} is not a string, so the manifest states none")
|
|
1540
|
+
return None
|
|
1541
|
+
placeholder = _detect_placeholder_version(version)
|
|
1542
|
+
if placeholder is not None:
|
|
1543
|
+
warnings.append(
|
|
1544
|
+
f"manifest version {placeholder!r} is a release-time placeholder "
|
|
1545
|
+
f"(workspace protocol or semantic-release sentinel) and is not a real version, "
|
|
1546
|
+
f"so the manifest states none"
|
|
1547
|
+
)
|
|
1548
|
+
return None
|
|
1549
|
+
return version
|
|
1550
|
+
|
|
1551
|
+
|
|
1335
1552
|
def extract(payload: dict) -> dict:
|
|
1336
1553
|
"""Orchestrate manifest parse + export scan for one logical package."""
|
|
1337
1554
|
warnings: list[str] = []
|
|
@@ -1353,14 +1570,7 @@ def extract(payload: dict) -> dict:
|
|
|
1353
1570
|
warnings.append(parsed["_parse_error"])
|
|
1354
1571
|
parsed = {}
|
|
1355
1572
|
|
|
1356
|
-
|
|
1357
|
-
if placeholder is not None:
|
|
1358
|
-
warnings.append(
|
|
1359
|
-
f"manifest version {placeholder!r} is a release-time placeholder "
|
|
1360
|
-
f"(workspace protocol or semantic-release sentinel) and is not a real version; "
|
|
1361
|
-
f"the brief will fall back to user-supplied target_version or the default"
|
|
1362
|
-
)
|
|
1363
|
-
parsed["version"] = None
|
|
1573
|
+
parsed["version"] = _literal_version(parsed.get("version"), warnings)
|
|
1364
1574
|
|
|
1365
1575
|
_, scanner = LANGUAGE_DISPATCH[language]
|
|
1366
1576
|
exports: list[dict] = []
|
|
@@ -1536,6 +1746,8 @@ JS_SOURCE_OF = {
|
|
|
1536
1746
|
JS_TARGET_SUFFIXES = (".d.mts", ".d.cts", ".d.ts", ".mjs", ".cjs", ".jsx", ".js", ".mts", ".cts", ".tsx", ".ts")
|
|
1537
1747
|
PACKAGE_ENTRY_FIELDS = ("types", "typings", "module", "main", "browser")
|
|
1538
1748
|
WORKSPACE_MANIFESTS = ("package.json", "pnpm-workspace.yaml", "lerna.json", "Cargo.toml", "rush.json", "nx.json")
|
|
1749
|
+
# The arms under which a curated count can be stored as stats.effective_denominator.
|
|
1750
|
+
DENOMINATOR_ARMS = ("monorepo", "specific_modules", "multi_subpath_exports")
|
|
1539
1751
|
|
|
1540
1752
|
|
|
1541
1753
|
class RunnerError(Exception):
|
|
@@ -1560,8 +1772,9 @@ def _sibling(filename: str):
|
|
|
1560
1772
|
"""A helper installed beside this script, loaded once as a module:
|
|
1561
1773
|
skf-source-tree.py (git without the caller's hooks and git variables,
|
|
1562
1774
|
and a tool lookup that never takes a shim planted in the current
|
|
1563
|
-
folder), skf-resolve-authoritative-files.py (the brief's
|
|
1564
|
-
skf-detect-workspaces.py (monorepo detection). Raises
|
|
1775
|
+
folder), skf-resolve-authoritative-files.py (the brief's YAML and glob
|
|
1776
|
+
rules) and skf-detect-workspaces.py (monorepo detection). Raises
|
|
1777
|
+
RunnerError."""
|
|
1565
1778
|
module = _SIBLINGS.get(filename)
|
|
1566
1779
|
if module is None:
|
|
1567
1780
|
path = Path(__file__).resolve().parent / filename
|
|
@@ -1856,8 +2069,9 @@ def _read_file_list(path: Path) -> list[str]:
|
|
|
1856
2069
|
def _read_brief(path: Path) -> dict:
|
|
1857
2070
|
"""The scope fields and language of a skill brief."""
|
|
1858
2071
|
yaml = _yaml()
|
|
2072
|
+
parse_brief_yaml = _sibling("skf-resolve-authoritative-files.py").parse_brief_yaml
|
|
1859
2073
|
try:
|
|
1860
|
-
data =
|
|
2074
|
+
data = parse_brief_yaml(path.read_text(encoding="utf-8"))
|
|
1861
2075
|
except (OSError, UnicodeDecodeError) as exc:
|
|
1862
2076
|
raise RunnerError(f"cannot read brief {path}: {exc}") from exc
|
|
1863
2077
|
except yaml.YAMLError as exc:
|
|
@@ -2003,12 +2217,17 @@ def _scan(exe: str, root: Path, folder: Path, rules: list[dict], files: list[str
|
|
|
2003
2217
|
errors: list[dict], clock: _Clock | None = None) -> list[dict]:
|
|
2004
2218
|
"""ast-grep's matches of `rules` in `files` (relative to `root`), in
|
|
2005
2219
|
batches. The rules go in a file as JSON documents (YAML reads JSON),
|
|
2006
|
-
never on the command line. A call that times out, exits
|
|
2007
|
-
rules carry no severity, so a scan that read every file
|
|
2008
|
-
or not) or prints other than JSON lines adds to
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2220
|
+
never on the command line. A call that cannot start, times out, exits
|
|
2221
|
+
non-zero (its rules carry no severity, so a scan that read every file
|
|
2222
|
+
exits 0, matches or not) or prints other than JSON lines adds to
|
|
2223
|
+
`errors` an item whose `unread` lists every file of its batch: ast-grep
|
|
2224
|
+
prints a file's matches as one block per rule, with other files' blocks
|
|
2225
|
+
between them, and a kill can stop it anywhere, so the output cannot show
|
|
2226
|
+
which files it printed in full. A call that exited keeps the matches it
|
|
2227
|
+
printed before any line that is not JSON; a call that timed out keeps
|
|
2228
|
+
none. When `clock` (the run's --timeout) runs out, a call running is
|
|
2229
|
+
stopped (its `time-limit` item lists its batch) and no other starts: a
|
|
2230
|
+
`time-limit` item lists the files no call read, if any."""
|
|
2012
2231
|
if not rules or not files:
|
|
2013
2232
|
return []
|
|
2014
2233
|
rule_file = folder / "rules.yml"
|
|
@@ -2055,10 +2274,8 @@ def _scan(exe: str, root: Path, folder: Path, rules: list[dict], files: list[str
|
|
|
2055
2274
|
if match is not None:
|
|
2056
2275
|
found.append(match)
|
|
2057
2276
|
if failure is not None:
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
error["unread"] = list(batch)
|
|
2061
|
-
errors.append(error)
|
|
2277
|
+
errors.append({"reason": failure[0], "files": len(batch), "first_file": batch[0], "detail": failure[1],
|
|
2278
|
+
"unread": list(batch)})
|
|
2062
2279
|
return found
|
|
2063
2280
|
|
|
2064
2281
|
|
|
@@ -2861,26 +3078,66 @@ def _specifier_local(text: str) -> str:
|
|
|
2861
3078
|
return re.split(r"\s+as\s+", item, maxsplit=1)[0].strip().strip("'\"")
|
|
2862
3079
|
|
|
2863
3080
|
|
|
3081
|
+
def _member(name: str, local: str | None, file: str | None, line: int | None) -> dict:
|
|
3082
|
+
"""One name a namespace's module passes on, traced to the file that
|
|
3083
|
+
defines it (`local` the name it has there when it differs)."""
|
|
3084
|
+
return {"name": name, "local": local if local and local != name else None, "file": file, "line": line}
|
|
3085
|
+
|
|
3086
|
+
|
|
3087
|
+
def _flatten(start, direct: Callable[[object], list[tuple[str, str, object]]]) -> list[dict]:
|
|
3088
|
+
"""The members of the namespace module `start`: what `direct` says it
|
|
3089
|
+
passes on, (name, "member", member) or (name, "module", the module),
|
|
3090
|
+
each nested namespace module's members spliced in its place (each
|
|
3091
|
+
module once, so a cycle is cut), each (name, file) once, and no name
|
|
3092
|
+
that starts with `_`, which is never public."""
|
|
3093
|
+
out: list[dict] = []
|
|
3094
|
+
seen: set[tuple[str, str | None]] = set()
|
|
3095
|
+
visited: set = set()
|
|
3096
|
+
|
|
3097
|
+
def walk(module) -> None:
|
|
3098
|
+
visited.add(module)
|
|
3099
|
+
for name, kind, value in direct(module):
|
|
3100
|
+
if not name or name.startswith("_"):
|
|
3101
|
+
continue
|
|
3102
|
+
if kind == "module":
|
|
3103
|
+
if value not in visited:
|
|
3104
|
+
walk(value)
|
|
3105
|
+
elif (value["name"], value["file"]) not in seen:
|
|
3106
|
+
seen.add((value["name"], value["file"]))
|
|
3107
|
+
out.append(value)
|
|
3108
|
+
|
|
3109
|
+
walk(start)
|
|
3110
|
+
return out
|
|
3111
|
+
|
|
3112
|
+
|
|
2864
3113
|
class _Surface:
|
|
2865
3114
|
"""The public names one language family's entry points declare:
|
|
2866
3115
|
{name: info}, the first entry point naming it wins; the names, and the
|
|
2867
3116
|
files, a namespace export (a module) makes reachable, which are not
|
|
2868
|
-
internal; and the re-exports the trace could not follow.
|
|
3117
|
+
internal; and the re-exports the trace could not follow. A namespace
|
|
3118
|
+
item also carries its `members`, the names its module passes on (None
|
|
3119
|
+
when the trace cannot read that module); `opaque` holds the (name,
|
|
3120
|
+
file) of each member that is itself a module the trace cannot read,
|
|
3121
|
+
which, as such a namespace, is no recipe's target."""
|
|
2869
3122
|
|
|
2870
3123
|
def __init__(self) -> None:
|
|
2871
3124
|
self.names: dict[str, dict] = {}
|
|
2872
3125
|
self.reachable: set[str] = set()
|
|
2873
3126
|
self.reachable_files: set[str] = set()
|
|
2874
3127
|
self.unresolved: list[dict] = []
|
|
3128
|
+
self.opaque: set[tuple[str, str | None]] = set()
|
|
2875
3129
|
|
|
2876
3130
|
def add(self, name: str, entry: str, via: str, file: str | None, line: int | None,
|
|
2877
|
-
source: str | None = None, local: str | None = None
|
|
3131
|
+
source: str | None = None, local: str | None = None,
|
|
3132
|
+
members: list[dict] | None = None) -> dict | None:
|
|
2878
3133
|
if not name or name.startswith("_"):
|
|
2879
3134
|
return None
|
|
2880
3135
|
if name not in self.names:
|
|
2881
3136
|
self.names[name] = {"name": name, "entry": entry, "via": via, "from": source,
|
|
2882
3137
|
"local": local if local and local != name else None,
|
|
2883
3138
|
"file": file, "line": line}
|
|
3139
|
+
if via == "namespace":
|
|
3140
|
+
self.names[name]["members"] = [dict(m) for m in members] if members is not None else None
|
|
2884
3141
|
return self.names[name]
|
|
2885
3142
|
|
|
2886
3143
|
def unresolve(self, entry: str, name: str, source: str | None, file: str) -> None:
|
|
@@ -3053,6 +3310,40 @@ def _js_surface(root: Path, entries: list[dict], tree_set: set[str], forms: dict
|
|
|
3053
3310
|
if fact is not None and fact["how"] == "namespace":
|
|
3054
3311
|
reach(target(fact["source"], where))
|
|
3055
3312
|
|
|
3313
|
+
passes: dict[str, list[tuple[str, str, object]]] = {}
|
|
3314
|
+
|
|
3315
|
+
def direct(file: str) -> list[tuple[str, str, object]]:
|
|
3316
|
+
"""What the namespace module `file` passes on, in order: (name,
|
|
3317
|
+
"member", a member traced as publish traces a name) or (name,
|
|
3318
|
+
"module", the file of a namespace among them); a namespace whose
|
|
3319
|
+
module no file here holds is a member itself, with no file, and so
|
|
3320
|
+
is a re-export from another package, which (unlike the same form in
|
|
3321
|
+
an entry point) is not unresolved."""
|
|
3322
|
+
if file not in passes:
|
|
3323
|
+
out: list[tuple[str, str, object]] = []
|
|
3324
|
+
for name, owner in passed_on(file).items():
|
|
3325
|
+
where, local, line, fact = resolve(owner, name, set())
|
|
3326
|
+
how = fact["how"] if fact is not None else None
|
|
3327
|
+
namespace = target(fact["source"], where) if how == "namespace" else None
|
|
3328
|
+
if namespace is not None:
|
|
3329
|
+
out.append((name, "module", namespace))
|
|
3330
|
+
elif how == "namespace":
|
|
3331
|
+
surface.opaque.add((name, None))
|
|
3332
|
+
out.append((name, "member", _member(name, None, None, None)))
|
|
3333
|
+
elif how == "re-export":
|
|
3334
|
+
out.append((name, "member", _member(name, local, None, None)))
|
|
3335
|
+
else:
|
|
3336
|
+
out.append((name, "member", _member(name, local, where, line)))
|
|
3337
|
+
passes[file] = out
|
|
3338
|
+
return passes[file]
|
|
3339
|
+
|
|
3340
|
+
member_lists: dict[str, list[dict]] = {}
|
|
3341
|
+
|
|
3342
|
+
def members(file: str) -> list[dict]:
|
|
3343
|
+
if file not in member_lists:
|
|
3344
|
+
member_lists[file] = _flatten(file, direct)
|
|
3345
|
+
return member_lists[file]
|
|
3346
|
+
|
|
3056
3347
|
def publish(entry: str, name: str, via: str, module: str, source: str | None) -> None:
|
|
3057
3348
|
"""Add public `name`, which `module` exports, traced to what it is."""
|
|
3058
3349
|
chain: set[str] = set()
|
|
@@ -3061,7 +3352,8 @@ def _js_surface(root: Path, entries: list[dict], tree_set: set[str], forms: dict
|
|
|
3061
3352
|
how = fact["how"] if fact is not None else None
|
|
3062
3353
|
if how == "namespace":
|
|
3063
3354
|
namespace = target(fact["source"], where)
|
|
3064
|
-
surface.add(name, entry, "namespace", namespace, None, source
|
|
3355
|
+
surface.add(name, entry, "namespace", namespace, None, source,
|
|
3356
|
+
members=members(namespace) if namespace is not None else None)
|
|
3065
3357
|
if namespace is None:
|
|
3066
3358
|
surface.unresolve(entry, name, fact["source"], where)
|
|
3067
3359
|
reach(namespace)
|
|
@@ -3371,9 +3663,16 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
|
|
|
3371
3663
|
if found is None:
|
|
3372
3664
|
return None, binding["imported"], None, "import"
|
|
3373
3665
|
if found.endswith("__init__.py"):
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
|
|
3666
|
+
# Python's own precedence: the package's explicit binding of
|
|
3667
|
+
# the name first, the submodule only when it binds none (or
|
|
3668
|
+
# binds the submodule itself, `from . import name` or
|
|
3669
|
+
# `from pkg.x import name`); a star import is not read here
|
|
3670
|
+
inner = bindings(found)[1].get(binding["imported"])
|
|
3671
|
+
if inner is None or (inner["kind"] == "import" and inner["imported"] == binding["imported"]
|
|
3672
|
+
and module_file(found, inner["level"], inner["module"], top) == found):
|
|
3673
|
+
sub = _py_submodule(posixpath.dirname(found), binding["imported"], tree_set)
|
|
3674
|
+
if sub and sub != rel:
|
|
3675
|
+
return sub, binding["imported"], None, "submodule"
|
|
3377
3676
|
rel, name = found, binding["imported"]
|
|
3378
3677
|
return rel, name, None, "missing"
|
|
3379
3678
|
|
|
@@ -3391,6 +3690,31 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
|
|
|
3391
3690
|
if what == "submodule" and where is not None:
|
|
3392
3691
|
reach(where, top)
|
|
3393
3692
|
|
|
3693
|
+
passes: dict[tuple[str, str], list[tuple[str, str, object]]] = {}
|
|
3694
|
+
|
|
3695
|
+
def direct(rel: str, top: str) -> list[tuple[str, str, object]]:
|
|
3696
|
+
"""What the submodule `rel` passes on (its star_names), in order:
|
|
3697
|
+
(name, "member", a member traced by resolve) or (name, "module",
|
|
3698
|
+
the file of a submodule among them); a name from another package
|
|
3699
|
+
has no file."""
|
|
3700
|
+
if (rel, top) not in passes:
|
|
3701
|
+
out: list[tuple[str, str, object]] = []
|
|
3702
|
+
for name, owner in star_names(rel, top).items():
|
|
3703
|
+
where, local, line, what = resolve(owner, name, top, set())
|
|
3704
|
+
if what == "submodule" and where is not None:
|
|
3705
|
+
out.append((name, "module", where))
|
|
3706
|
+
else:
|
|
3707
|
+
out.append((name, "member", _member(name, local, where, line)))
|
|
3708
|
+
passes[(rel, top)] = out
|
|
3709
|
+
return passes[(rel, top)]
|
|
3710
|
+
|
|
3711
|
+
member_lists: dict[tuple[str, str], list[dict]] = {}
|
|
3712
|
+
|
|
3713
|
+
def members(rel: str, top: str) -> list[dict]:
|
|
3714
|
+
if (rel, top) not in member_lists:
|
|
3715
|
+
member_lists[(rel, top)] = _flatten(rel, lambda module: direct(module, top))
|
|
3716
|
+
return member_lists[(rel, top)]
|
|
3717
|
+
|
|
3394
3718
|
for entry in [e["file"] for e in entries]:
|
|
3395
3719
|
top = posixpath.dirname(entry)
|
|
3396
3720
|
all_names, bound, stars = bindings(entry)
|
|
@@ -3420,7 +3744,7 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
|
|
|
3420
3744
|
where, local, line, what = resolve(entry, name, top, chain)
|
|
3421
3745
|
surface.reachable.update(chain)
|
|
3422
3746
|
if what == "submodule":
|
|
3423
|
-
surface.add(name, entry, "namespace", where, None, source, local)
|
|
3747
|
+
surface.add(name, entry, "namespace", where, None, source, local, members=members(where, top))
|
|
3424
3748
|
reach(where, top)
|
|
3425
3749
|
elif what == "import":
|
|
3426
3750
|
surface.add(name, entry, "re-export", None, None, source, local)
|
|
@@ -3674,6 +3998,41 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
|
|
|
3674
3998
|
if sub is not None:
|
|
3675
3999
|
reach(lib, sub, reached)
|
|
3676
4000
|
|
|
4001
|
+
passes: dict[tuple[str, tuple[str, ...]], list[tuple[str, str, object]]] = {}
|
|
4002
|
+
|
|
4003
|
+
def direct(lib: str, here: tuple[str, ...]) -> list[tuple[str, str, object]]:
|
|
4004
|
+
"""What the module at `here` passes on (exported), in order: (name,
|
|
4005
|
+
"member", a member traced as publish traces a name) or (name,
|
|
4006
|
+
"module", the path of a module among them that a file of its own
|
|
4007
|
+
holds); an inline or `#[path]` module is a member itself, at its
|
|
4008
|
+
`pub mod`, and a `pub use` of another crate's item one with no file,
|
|
4009
|
+
which (unlike the same form in the crate root) is not unresolved."""
|
|
4010
|
+
if (lib, here) not in passes:
|
|
4011
|
+
out: list[tuple[str, str, object]] = []
|
|
4012
|
+
for name in exported(lib, list(here)):
|
|
4013
|
+
file, local, line, sub, stop = resolve(lib, list(here), name, set())
|
|
4014
|
+
if stop is not None:
|
|
4015
|
+
out.append((name, "member", _member(name, local, None, None)))
|
|
4016
|
+
elif sub is not None and _rust_file(lib, sub, tree_set) is not None:
|
|
4017
|
+
out.append((name, "module", tuple(sub)))
|
|
4018
|
+
else:
|
|
4019
|
+
if sub is not None:
|
|
4020
|
+
surface.opaque.add((name, file))
|
|
4021
|
+
out.append((name, "member", _member(name, local, file, line)))
|
|
4022
|
+
passes[(lib, here)] = out
|
|
4023
|
+
return passes[(lib, here)]
|
|
4024
|
+
|
|
4025
|
+
member_lists: dict[tuple[str, tuple[str, ...]], list[dict]] = {}
|
|
4026
|
+
|
|
4027
|
+
def members(lib: str, here: list[str]) -> list[dict] | None:
|
|
4028
|
+
"""The members of the module at `here`; None when no file of its own holds it."""
|
|
4029
|
+
if _rust_file(lib, here, tree_set) is None:
|
|
4030
|
+
return None
|
|
4031
|
+
key = (lib, tuple(here))
|
|
4032
|
+
if key not in member_lists:
|
|
4033
|
+
member_lists[key] = _flatten(tuple(here), lambda path: direct(lib, path))
|
|
4034
|
+
return member_lists[key]
|
|
4035
|
+
|
|
3677
4036
|
def publish(lib: str, here: list[str], name: str, via: str, source: str | None,
|
|
3678
4037
|
reached: set[tuple[str, ...]]) -> None:
|
|
3679
4038
|
"""Add public `name`, which the module at `here` exports, traced to
|
|
@@ -3685,7 +4044,7 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
|
|
|
3685
4044
|
surface.add(name, lib, via, None, None, source, local)
|
|
3686
4045
|
surface.unresolve(lib, name, *stop)
|
|
3687
4046
|
elif sub is not None:
|
|
3688
|
-
surface.add(name, lib, "namespace", file, line, "::".join(sub))
|
|
4047
|
+
surface.add(name, lib, "namespace", file, line, "::".join(sub), members=members(lib, sub))
|
|
3689
4048
|
reach(lib, sub, reached)
|
|
3690
4049
|
else:
|
|
3691
4050
|
surface.add(name, lib, via, file, line, source, local)
|
|
@@ -3700,7 +4059,8 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
|
|
|
3700
4059
|
continue
|
|
3701
4060
|
sub = [name.removeprefix("r#")]
|
|
3702
4061
|
sub_file = _rust_file(lib, sub, tree_set)
|
|
3703
|
-
surface.add(name, lib, "namespace", sub_file or lib, None if sub_file else line
|
|
4062
|
+
surface.add(name, lib, "namespace", sub_file or lib, None if sub_file else line,
|
|
4063
|
+
members=members(lib, sub))
|
|
3704
4064
|
reach(lib, sub, reached)
|
|
3705
4065
|
for path, alias, glob, _ in info["uses"]:
|
|
3706
4066
|
if not glob:
|
|
@@ -3779,8 +4139,13 @@ def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries
|
|
|
3779
4139
|
reachable (a name, or any export of a reachable file), is internal. A
|
|
3780
4140
|
declared name no recipe found is a gap when a file in scope defines it
|
|
3781
4141
|
(or the trace found no file), and is outside scope when a file out of
|
|
3782
|
-
scope does; a module is no recipe's target, so neither
|
|
3783
|
-
with
|
|
4142
|
+
scope does; a module is no recipe's target, so neither: a namespace
|
|
4143
|
+
with `members` has each member diffed in its place, found when a
|
|
4144
|
+
recipe found its name, or its `local` one, in its file, and given no
|
|
4145
|
+
row of its own when a declared name has its name and file (or another
|
|
4146
|
+
namespace gave it one); one whose members are unknown (None), or a
|
|
4147
|
+
member that is such a module (`opaque`), is no gap. In a family with
|
|
4148
|
+
no entry point, or only empty ones, every export is public."""
|
|
3784
4149
|
statuses: dict[str, str] = {}
|
|
3785
4150
|
public: dict[tuple[str, str], dict] = {}
|
|
3786
4151
|
internal: list[dict] = []
|
|
@@ -3814,16 +4179,50 @@ def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries
|
|
|
3814
4179
|
if name in names or (info["local"] or name) in names or info["via"] == "namespace":
|
|
3815
4180
|
continue
|
|
3816
4181
|
(gaps if info["file"] is None or info["file"] in in_scope else outside).append((family, info))
|
|
4182
|
+
# A namespace's members, each by its name (or `local`) and file, or
|
|
4183
|
+
# by name alone when it has no file, as a declared name is: a name
|
|
4184
|
+
# at a file a declared name already gives counts once.
|
|
4185
|
+
found = {(e["export_name"], e["source_file"]) for e in records}
|
|
4186
|
+
given = {(info["name"], info["file"]) for info in declared.values() if not _has_members(info)}
|
|
4187
|
+
for info in declared.values():
|
|
4188
|
+
for member in info["members"] if _has_members(info) else []:
|
|
4189
|
+
key = (member["name"], member["file"])
|
|
4190
|
+
if (key in given or key in found or key in surface.opaque
|
|
4191
|
+
or (member["local"] and (member["local"], member["file"]) in found)
|
|
4192
|
+
or (member["file"] is None and {member["name"], member["local"]} & names)):
|
|
4193
|
+
continue
|
|
4194
|
+
given.add(key)
|
|
4195
|
+
row = {"name": member["name"], "entry": info["entry"], "file": member["file"],
|
|
4196
|
+
"line": member["line"]}
|
|
4197
|
+
(gaps if member["file"] is None or member["file"] in in_scope else outside).append((family, row))
|
|
3817
4198
|
return statuses, public, internal, gaps, outside
|
|
3818
4199
|
|
|
3819
4200
|
|
|
4201
|
+
def _has_members(info: dict) -> bool:
|
|
4202
|
+
"""A namespace item whose module's members the trace read (a list, maybe empty)."""
|
|
4203
|
+
return info.get("via") == "namespace" and isinstance(info.get("members"), list)
|
|
4204
|
+
|
|
4205
|
+
|
|
4206
|
+
def _public_keys(public: dict[tuple[str, str], dict]) -> set[tuple[str, str, str | None]]:
|
|
4207
|
+
"""The (family, name, file) the counts count: each public name but a
|
|
4208
|
+
namespace with members, in its place each member, and so a member that
|
|
4209
|
+
is also a declared name (or another namespace's member) once."""
|
|
4210
|
+
keys: set[tuple[str, str, str | None]] = set()
|
|
4211
|
+
for (family, _), info in public.items():
|
|
4212
|
+
if _has_members(info):
|
|
4213
|
+
keys.update((family, m["name"], m["file"]) for m in info["members"])
|
|
4214
|
+
else:
|
|
4215
|
+
keys.add((family, info["name"], info["file"]))
|
|
4216
|
+
return keys
|
|
4217
|
+
|
|
4218
|
+
|
|
3820
4219
|
def _head_cap(value: int | None, tier: str | None, scope_type: str | None) -> int | None:
|
|
3821
4220
|
if value is not None:
|
|
3822
4221
|
return value or None
|
|
3823
4222
|
return HEAD_CAPS.get((tier, scope_type), DEFAULT_HEAD_CAP)
|
|
3824
4223
|
|
|
3825
4224
|
|
|
3826
|
-
def
|
|
4225
|
+
def _workspaces(root: Path, tree: list[str], tree_set: set[str]) -> dict:
|
|
3827
4226
|
"""skf-detect-workspaces.py's verdict on the source root."""
|
|
3828
4227
|
manifests = {}
|
|
3829
4228
|
for name in WORKSPACE_MANIFESTS:
|
|
@@ -3831,19 +4230,111 @@ def _monorepo(root: Path, tree: list[str], tree_set: set[str]) -> tuple[bool, st
|
|
|
3831
4230
|
text = _read_text(root, name)
|
|
3832
4231
|
if text is not None:
|
|
3833
4232
|
manifests[name] = text
|
|
3834
|
-
|
|
4233
|
+
return _sibling("skf-detect-workspaces.py").detect({"tree": tree, "manifests": manifests})
|
|
4234
|
+
|
|
4235
|
+
|
|
4236
|
+
def _monorepo(root: Path, tree: list[str], tree_set: set[str]) -> tuple[bool, str | None]:
|
|
4237
|
+
"""skf-detect-workspaces.py's verdict on the source root: (is it a monorepo, its manifest kind)."""
|
|
4238
|
+
result = _workspaces(root, tree, tree_set)
|
|
3835
4239
|
return bool(result.get("is_monorepo")), result.get("manifest_kind")
|
|
3836
4240
|
|
|
3837
4241
|
|
|
4242
|
+
def _workspace_sources(result: dict, tree: list[str], wanted: set[str]) -> list[set[str]]:
|
|
4243
|
+
"""The source files (of a language --language selects) of each workspace
|
|
4244
|
+
skf-detect-workspaces.py found, one set per workspace that holds any."""
|
|
4245
|
+
sources = []
|
|
4246
|
+
for workspace in result.get("workspaces") or [] if result.get("is_monorepo") else []:
|
|
4247
|
+
folder = _norm(workspace.get("path") if isinstance(workspace, dict) else "").strip("/")
|
|
4248
|
+
files = {f for f in tree if (not folder or f.startswith(folder + "/")) and _file_language(f) in wanted}
|
|
4249
|
+
if files:
|
|
4250
|
+
sources.append(files)
|
|
4251
|
+
return sources
|
|
4252
|
+
|
|
4253
|
+
|
|
4254
|
+
def denominator_count(counted: set[tuple[str, str, str | None]], tree: list[str], includes: list[str],
|
|
4255
|
+
excludes: list[str], tier_a: list[str], glob_match: Callable[[str, str], bool],
|
|
4256
|
+
read_all: bool) -> tuple[int, str, set[str], str | None]:
|
|
4257
|
+
"""(effective_denominator, its basis, the files it counts, a warning or None)
|
|
4258
|
+
over the (family, name, file) keys `counted`: each name once whose file
|
|
4259
|
+
matches scope.include (any file with no glob), scope.tier_a_include when
|
|
4260
|
+
given (a tier A name counts only in a file scope.include matches too) and
|
|
4261
|
+
no scope.exclude glob. A tier A list that counts no name falls back to
|
|
4262
|
+
scope.include (else every file) once a name was counted and every file
|
|
4263
|
+
was read (`read_all`)."""
|
|
4264
|
+
|
|
4265
|
+
def files_matching(globs: list[str]) -> set[str]:
|
|
4266
|
+
return {f for f in tree if (not globs or any(glob_match(f, g) for g in globs))
|
|
4267
|
+
and not any(glob_match(f, g) for g in excludes)}
|
|
4268
|
+
|
|
4269
|
+
in_include = files_matching(includes)
|
|
4270
|
+
denominator_files = (files_matching(tier_a) & in_include) if tier_a else in_include
|
|
4271
|
+
basis = "tier_a_include" if tier_a else ("scope.include" if includes else "all-files")
|
|
4272
|
+
warning = None
|
|
4273
|
+
if tier_a and counted and read_all and not any(file in denominator_files for _, _, file in counted):
|
|
4274
|
+
basis = "scope.include" if includes else "all-files"
|
|
4275
|
+
denominator_files = in_include
|
|
4276
|
+
counts_what = "scope.include" if includes else "every file"
|
|
4277
|
+
warning = f"scope.tier_a_include counts no public name: effective_denominator counts {counts_what}"
|
|
4278
|
+
# Each name once, as Test Skill's sets of names count it: a name the entry points pass on
|
|
4279
|
+
# from two files, or in two languages, is two keys but one name.
|
|
4280
|
+
count = len({name for _, name, file in counted if file in denominator_files})
|
|
4281
|
+
return count, basis, denominator_files, warning
|
|
4282
|
+
|
|
4283
|
+
|
|
4284
|
+
def curated_subset(counted: set[tuple[str, str, str | None]], basis: str, tree: list[str], includes: list[str],
|
|
4285
|
+
excludes: list[str], tier_a: list[str], glob_match: Callable[[str, str], bool],
|
|
4286
|
+
arms: dict, workspace_sources: list[set[str]]) -> bool:
|
|
4287
|
+
"""Whether the files the count read are a curated subset rather than the
|
|
4288
|
+
whole workspace (in a single-package repository, the whole package): the
|
|
4289
|
+
files the basis globs list (a tier A list's, among those scope.include
|
|
4290
|
+
lists; under arm (b), less those scope.exclude drops) leave out a file
|
|
4291
|
+
that defines a counted public name or, in a monorepo, every source file
|
|
4292
|
+
of a workspace. A basis of all-files lists none."""
|
|
4293
|
+
if basis == "all-files":
|
|
4294
|
+
return False
|
|
4295
|
+
files = {f for f in tree if not includes or any(glob_match(f, g) for g in includes)}
|
|
4296
|
+
if basis == "tier_a_include":
|
|
4297
|
+
files = {f for f in files if any(glob_match(f, g) for g in tier_a)}
|
|
4298
|
+
if arms.get("specific_modules"):
|
|
4299
|
+
files = {f for f in files if not any(glob_match(f, g) for g in excludes)}
|
|
4300
|
+
tree_set = set(tree)
|
|
4301
|
+
defining = {file for _, _, file in counted if file in tree_set}
|
|
4302
|
+
return bool(defining - files) or any(not (sources & files) for sources in workspace_sources)
|
|
4303
|
+
|
|
4304
|
+
|
|
4305
|
+
def effective_denominator_decision(arms: dict, scope_type: str | None, basis: str, curated: bool, count: int,
|
|
4306
|
+
incomplete: bool) -> dict:
|
|
4307
|
+
"""Whether create-skill and update-skill store the count as
|
|
4308
|
+
stats.effective_denominator, the brief's scope.notes aside: {store, reason},
|
|
4309
|
+
the reason naming the first condition that fails."""
|
|
4310
|
+
if scope_type == "reference-app":
|
|
4311
|
+
reason = "reference-app"
|
|
4312
|
+
elif incomplete:
|
|
4313
|
+
reason = "incomplete-run"
|
|
4314
|
+
elif not any(arms.get(arm) for arm in DENOMINATOR_ARMS):
|
|
4315
|
+
reason = "no-arm"
|
|
4316
|
+
elif scope_type == "full-library":
|
|
4317
|
+
reason = "full-library"
|
|
4318
|
+
elif basis == "all-files":
|
|
4319
|
+
reason = "all-files"
|
|
4320
|
+
elif not curated:
|
|
4321
|
+
reason = "not-curated"
|
|
4322
|
+
elif count == 0:
|
|
4323
|
+
reason = "zero-count"
|
|
4324
|
+
else:
|
|
4325
|
+
return {"store": True, "reason": "curated-subset"}
|
|
4326
|
+
return {"store": False, "reason": reason}
|
|
4327
|
+
|
|
4328
|
+
|
|
3838
4329
|
def _select_files(root: Path, candidates: list[str], listed: bool, includes: list[str], excludes: list[str],
|
|
3839
4330
|
wanted: set[str], glob_match: Callable[[str, str], bool], issues: list[dict],
|
|
3840
|
-
warnings: list[str]) -> tuple[list[str],
|
|
3841
|
-
"""(the files in scope,
|
|
3842
|
-
|
|
3843
|
-
recipe reads, is an issue; an include glob
|
|
3844
|
-
warning."""
|
|
4331
|
+
warnings: list[str]) -> tuple[list[str], list[str], list[str]]:
|
|
4332
|
+
"""(the files in scope, the in-scope source files no recipe reads, the
|
|
4333
|
+
include globs no candidate matches). A file a --files-from list names
|
|
4334
|
+
that is missing, or that no recipe reads, is an issue; an include glob
|
|
4335
|
+
no candidate matches, a warning too."""
|
|
3845
4336
|
in_scope: list[str] = []
|
|
3846
|
-
without_recipes:
|
|
4337
|
+
without_recipes: list[str] = []
|
|
3847
4338
|
used = dict.fromkeys(includes, False)
|
|
3848
4339
|
for rel in candidates:
|
|
3849
4340
|
hits = [g for g in includes if glob_match(rel, g)]
|
|
@@ -3859,51 +4350,66 @@ def _select_files(root: Path, candidates: list[str], listed: bool, includes: lis
|
|
|
3859
4350
|
if listed:
|
|
3860
4351
|
issues.append({"file": rel, "issue": "no-recipes"})
|
|
3861
4352
|
elif extension in NO_RECIPE_EXTENSIONS:
|
|
3862
|
-
without_recipes
|
|
4353
|
+
without_recipes.append(rel)
|
|
3863
4354
|
continue
|
|
3864
4355
|
in_scope.append(rel)
|
|
3865
|
-
|
|
3866
|
-
|
|
4356
|
+
unmatched = [g for g, hit in used.items() if not hit]
|
|
4357
|
+
warnings += [f"scope.include pattern {g!r} matches no file" for g in unmatched]
|
|
4358
|
+
return in_scope, without_recipes, unmatched
|
|
3867
4359
|
|
|
3868
4360
|
|
|
3869
|
-
def
|
|
3870
|
-
"""
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
|
|
3875
|
-
|
|
3876
|
-
|
|
4361
|
+
def _by_extension(files: list[str]) -> dict[str, int]:
|
|
4362
|
+
"""`files_without_recipes`: how many of `files` each extension has."""
|
|
4363
|
+
return dict(sorted(Counter(posixpath.splitext(f)[1].lower() for f in files).items()))
|
|
4364
|
+
|
|
4365
|
+
|
|
4366
|
+
def _unmatched_tier_a(tier_a: list[str], includes: list[str], tree: list[str],
|
|
4367
|
+
glob_match: Callable[[str, str], bool], warnings: list[str]) -> list[str]:
|
|
4368
|
+
"""The tier A globs that match no file of `tree` (the whole tree, regardless
|
|
4369
|
+
of --files-from and before the excludes), with a warning for each, and one
|
|
4370
|
+
for each that matches only files outside scope.include."""
|
|
4371
|
+
unmatched = [g for g in dict.fromkeys(tier_a) if not any(glob_match(f, g) for f in tree)]
|
|
4372
|
+
warnings += [f"scope.tier_a_include pattern {g!r} matches no file" for g in unmatched]
|
|
4373
|
+
warnings += [f"scope.tier_a_include pattern {g!r} matches only files outside scope.include"
|
|
4374
|
+
for g in dict.fromkeys(tier_a) if includes and g not in unmatched
|
|
4375
|
+
and not any(glob_match(f, g) and any(glob_match(f, i) for i in includes) for f in tree)]
|
|
4376
|
+
return unmatched
|
|
4377
|
+
|
|
4378
|
+
|
|
4379
|
+
def _brief_scope(args: argparse.Namespace) -> dict:
|
|
4380
|
+
"""The scope globs, scope type and languages: the flags', else the brief's."""
|
|
3877
4381
|
brief = _read_brief(Path(args.brief)) if args.brief else {}
|
|
3878
|
-
includes = [g for g in (_norm(v) for v in (args.include or brief.get("include") or [])) if g]
|
|
3879
|
-
excludes = [g for g in (_norm(v) for v in (args.exclude or brief.get("exclude") or [])) if g]
|
|
3880
|
-
tier_a = [g for g in (_norm(v) for v in (args.tier_a_include or brief.get("tier_a_include") or [])) if g]
|
|
3881
|
-
scope_type = args.scope_type or brief.get("type")
|
|
3882
4382
|
language_values = args.language or ([brief["language"]] if brief.get("language") else [])
|
|
3883
4383
|
wanted, unknown_languages = _wanted_languages(language_values)
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
4384
|
+
return {
|
|
4385
|
+
"includes": [g for g in (_norm(v) for v in (args.include or brief.get("include") or [])) if g],
|
|
4386
|
+
"excludes": [g for g in (_norm(v) for v in (args.exclude or brief.get("exclude") or [])) if g],
|
|
4387
|
+
"tier_a": [g for g in (_norm(v) for v in (args.tier_a_include or brief.get("tier_a_include") or [])) if g],
|
|
4388
|
+
"scope_type": args.scope_type or brief.get("type"),
|
|
4389
|
+
"wanted": wanted,
|
|
4390
|
+
"glob_match": _sibling("skf-resolve-authoritative-files.py").glob_match,
|
|
4391
|
+
"warnings": [f"no recipe reads {value} files" for value in unknown_languages],
|
|
4392
|
+
}
|
|
3887
4393
|
|
|
3888
|
-
|
|
3889
|
-
|
|
4394
|
+
|
|
4395
|
+
def _scope_facts(args: argparse.Namespace, root: Path) -> dict:
|
|
4396
|
+
"""What full mode and denominator mode know before any recipe runs: the
|
|
4397
|
+
brief's scope, the files in scope, the entry points and the arms."""
|
|
4398
|
+
scope = _brief_scope(args)
|
|
4399
|
+
includes, excludes, tier_a = scope["includes"], scope["excludes"], scope["tier_a"]
|
|
4400
|
+
scope_type, wanted, glob_match = scope["scope_type"], scope["wanted"], scope["glob_match"]
|
|
4401
|
+
warnings: list[str] = scope["warnings"]
|
|
3890
4402
|
issues: list[dict] = []
|
|
3891
4403
|
tree = list_tree(root)
|
|
3892
4404
|
tree_set = set(tree)
|
|
3893
|
-
|
|
3894
|
-
|
|
3895
|
-
|
|
3896
|
-
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
3901
|
-
issues.append({"file": rel, "issue": "unreadable"})
|
|
3902
|
-
continue
|
|
3903
|
-
except UnicodeDecodeError:
|
|
3904
|
-
issues.append({"file": rel, "issue": "not-utf8"})
|
|
3905
|
-
continue
|
|
3906
|
-
readable.append(rel)
|
|
4405
|
+
files_from = getattr(args, "files_from", None)
|
|
4406
|
+
listed = _read_file_list(Path(files_from)) if files_from else None
|
|
4407
|
+
in_scope, without_recipes, unmatched_include = _select_files(
|
|
4408
|
+
root, tree if listed is None else listed, listed is not None, includes, excludes, wanted, glob_match,
|
|
4409
|
+
issues, warnings)
|
|
4410
|
+
# A tier A glob is matched against the whole tree under --source-root (the
|
|
4411
|
+
# tree effective_denominator reads).
|
|
4412
|
+
unmatched_tier_a = _unmatched_tier_a(tier_a, includes, tree, glob_match, warnings)
|
|
3907
4413
|
|
|
3908
4414
|
# The entry points and the arms need no ast-grep.
|
|
3909
4415
|
in_scope_set = set(in_scope)
|
|
@@ -3922,13 +4428,85 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
3922
4428
|
f"{entry['package']}) is outside the files in scope: widen scope.include to document "
|
|
3923
4429
|
"the names it exports (entry_point_diff.outside_scope), which effective_denominator "
|
|
3924
4430
|
"leaves out")
|
|
3925
|
-
|
|
4431
|
+
workspaces = _workspaces(root, tree, tree_set)
|
|
4432
|
+
monorepo = bool(workspaces.get("is_monorepo"))
|
|
3926
4433
|
arms = {
|
|
3927
4434
|
"monorepo": monorepo,
|
|
3928
|
-
"monorepo_kind":
|
|
4435
|
+
"monorepo_kind": workspaces.get("manifest_kind"),
|
|
3929
4436
|
"specific_modules": scope_type == "specific-modules" and not monorepo,
|
|
3930
4437
|
"multi_subpath_exports": not monorepo and any(m["non_root_subpaths"] > 1 for m in maps),
|
|
3931
4438
|
}
|
|
4439
|
+
return {
|
|
4440
|
+
"includes": includes, "excludes": excludes, "tier_a": tier_a, "scope_type": scope_type, "wanted": wanted,
|
|
4441
|
+
"glob_match": glob_match, "warnings": warnings, "issues": issues, "tree": tree, "tree_set": tree_set,
|
|
4442
|
+
"in_scope": in_scope, "without_recipes": without_recipes, "unmatched_include": unmatched_include,
|
|
4443
|
+
"unmatched_tier_a": unmatched_tier_a, "surfaces": surfaces, "js_entries": js_entries, "maps": maps,
|
|
4444
|
+
"unresolved_subpaths": unresolved_subpaths, "py_entries": py_entries, "rust_entries": rust_entries,
|
|
4445
|
+
"entries": entries, "arms": arms, "workspace_sources": _workspace_sources(workspaces, tree, wanted),
|
|
4446
|
+
}
|
|
4447
|
+
|
|
4448
|
+
|
|
4449
|
+
def _scope_record(args: argparse.Namespace, facts: dict) -> dict:
|
|
4450
|
+
"""The `scope` object full mode and denominator mode write."""
|
|
4451
|
+
return {"include": facts["includes"], "exclude": facts["excludes"], "tier_a_include": facts["tier_a"] or None,
|
|
4452
|
+
"type": facts["scope_type"], "languages": sorted(facts["wanted"]),
|
|
4453
|
+
"files_from": getattr(args, "files_from", None), "unmatched_include": facts["unmatched_include"],
|
|
4454
|
+
"unmatched_tier_a_include": facts["unmatched_tier_a"]}
|
|
4455
|
+
|
|
4456
|
+
|
|
4457
|
+
def _denominator_counts(facts: dict, counted: set[tuple[str, str, str | None]], public_api: int, internal: int,
|
|
4458
|
+
incomplete: bool, warnings: list[str]) -> dict:
|
|
4459
|
+
"""The `counts` object: the two API counts, effective_denominator with its
|
|
4460
|
+
basis and files, whether those files are a curated subset, and the
|
|
4461
|
+
decision whether to store the count."""
|
|
4462
|
+
count, basis, denominator_files, warning = denominator_count(
|
|
4463
|
+
counted, facts["tree"], facts["includes"], facts["excludes"], facts["tier_a"], facts["glob_match"],
|
|
4464
|
+
not incomplete)
|
|
4465
|
+
if warning:
|
|
4466
|
+
warnings.append(warning)
|
|
4467
|
+
curated = curated_subset(counted, basis, facts["tree"], facts["includes"], facts["excludes"], facts["tier_a"],
|
|
4468
|
+
facts["glob_match"], facts["arms"], facts["workspace_sources"])
|
|
4469
|
+
return {
|
|
4470
|
+
"exports_public_api": public_api,
|
|
4471
|
+
"exports_internal": internal,
|
|
4472
|
+
"effective_denominator": count,
|
|
4473
|
+
"effective_denominator_basis": basis,
|
|
4474
|
+
"denominator_files": len(denominator_files),
|
|
4475
|
+
"curated_subset": curated,
|
|
4476
|
+
"effective_denominator_decision": effective_denominator_decision(
|
|
4477
|
+
facts["arms"], facts["scope_type"], basis, curated, count, incomplete),
|
|
4478
|
+
}
|
|
4479
|
+
|
|
4480
|
+
|
|
4481
|
+
def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
4482
|
+
"""Run the recipes over the source tree; (result, exit code)."""
|
|
4483
|
+
clock = _Clock(DEFAULT_TIMEOUT_SEC if args.timeout is None else args.timeout)
|
|
4484
|
+
root = Path(args.source_root)
|
|
4485
|
+
if not root.is_dir():
|
|
4486
|
+
raise RunnerError(f"source root not found: {args.source_root}")
|
|
4487
|
+
recipes_path = Path(args.recipes) if args.recipes else RECIPES_FILE
|
|
4488
|
+
recipes = load_recipe_file(recipes_path)
|
|
4489
|
+
facts = _scope_facts(args, root)
|
|
4490
|
+
scope_type, warnings, issues = facts["scope_type"], facts["warnings"], facts["issues"]
|
|
4491
|
+
tree_set, in_scope = facts["tree_set"], facts["in_scope"]
|
|
4492
|
+
in_scope_set = set(in_scope)
|
|
4493
|
+
surfaces, js_entries, maps = facts["surfaces"], facts["js_entries"], facts["maps"]
|
|
4494
|
+
py_entries, rust_entries, entries = facts["py_entries"], facts["rust_entries"], facts["entries"]
|
|
4495
|
+
unresolved_subpaths, arms = facts["unresolved_subpaths"], facts["arms"]
|
|
4496
|
+
recipe_set = args.recipe_set or ("component-library" if scope_type == "component-library" else "standard")
|
|
4497
|
+
head_cap = _head_cap(args.head_cap, args.tier, scope_type)
|
|
4498
|
+
errors: list[dict] = []
|
|
4499
|
+
readable = []
|
|
4500
|
+
for rel in in_scope:
|
|
4501
|
+
try:
|
|
4502
|
+
(root / rel).read_bytes().decode("utf-8")
|
|
4503
|
+
except OSError:
|
|
4504
|
+
issues.append({"file": rel, "issue": "unreadable"})
|
|
4505
|
+
continue
|
|
4506
|
+
except UnicodeDecodeError:
|
|
4507
|
+
issues.append({"file": rel, "issue": "not-utf8"})
|
|
4508
|
+
continue
|
|
4509
|
+
readable.append(rel)
|
|
3932
4510
|
|
|
3933
4511
|
exe, version, shim = _ast_grep()
|
|
3934
4512
|
result: dict = {
|
|
@@ -3938,12 +4516,10 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
3938
4516
|
"recipes_file": recipes_path.as_posix(),
|
|
3939
4517
|
"recipe_set": recipe_set,
|
|
3940
4518
|
"ast_grep": {"path": exe, "version": version},
|
|
3941
|
-
"scope":
|
|
3942
|
-
"type": scope_type, "languages": sorted(wanted),
|
|
3943
|
-
"files_from": args.files_from},
|
|
4519
|
+
"scope": _scope_record(args, facts),
|
|
3944
4520
|
"files_in_scope": len(in_scope),
|
|
3945
4521
|
"files_by_language": dict(sorted(Counter(_file_language(f) for f in in_scope).items())),
|
|
3946
|
-
"files_without_recipes": without_recipes,
|
|
4522
|
+
"files_without_recipes": _by_extension(facts["without_recipes"]),
|
|
3947
4523
|
"file_issues": issues,
|
|
3948
4524
|
"head_cap": head_cap,
|
|
3949
4525
|
"truncated": False,
|
|
@@ -3958,6 +4534,9 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
3958
4534
|
"arms": arms,
|
|
3959
4535
|
"errors": errors,
|
|
3960
4536
|
"warnings": warnings,
|
|
4537
|
+
# Last: only a run without the recipes reads them, and a large tree lists thousands.
|
|
4538
|
+
"files_in_scope_list": in_scope,
|
|
4539
|
+
"files_without_recipes_list": facts["without_recipes"],
|
|
3961
4540
|
}
|
|
3962
4541
|
if exe is None:
|
|
3963
4542
|
result["status"] = "no-ast-grep"
|
|
@@ -4013,10 +4592,12 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
4013
4592
|
else:
|
|
4014
4593
|
status = "empty-barrel" if "empty-barrel" in statuses.values() else "no-entry-point"
|
|
4015
4594
|
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
4019
|
-
|
|
4595
|
+
# A tier A list that counts no public name (each glob matches no file, or
|
|
4596
|
+
# only files an exclude drops, that scope.include leaves out or that
|
|
4597
|
+
# define no public name) narrows nothing, once the run counted a name
|
|
4598
|
+
# and read every file (no error).
|
|
4599
|
+
counted = _public_keys(public)
|
|
4600
|
+
counts = _denominator_counts(facts, counted, len(counted), len(internal), bool(errors), warnings)
|
|
4020
4601
|
|
|
4021
4602
|
by_type: Counter = Counter(e["export_type"] for e in exports)
|
|
4022
4603
|
ordered = sorted(public.items(), key=lambda item: (item[0][1], item[0][0]))
|
|
@@ -4043,13 +4624,7 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
4043
4624
|
{"name": info["name"], "language": family, "local": info["local"], "from": info["from"],
|
|
4044
4625
|
"entry": info["entry"], "file": info["file"], "line": info["line"], "via": info["via"]}
|
|
4045
4626
|
for (family, _), info in ordered if info["via"] in ("re-export", "star", "namespace")],
|
|
4046
|
-
"counts":
|
|
4047
|
-
"exports_public_api": len(public),
|
|
4048
|
-
"exports_internal": len(internal),
|
|
4049
|
-
"effective_denominator": sum(1 for info in public.values() if info["file"] in denominator_files),
|
|
4050
|
-
"effective_denominator_basis": basis,
|
|
4051
|
-
"denominator_files": len(denominator_files),
|
|
4052
|
-
},
|
|
4627
|
+
"counts": counts,
|
|
4053
4628
|
})
|
|
4054
4629
|
result["file_issues"] = sorted(issues, key=lambda i: (i["file"], i["issue"]))
|
|
4055
4630
|
if errors:
|
|
@@ -4058,37 +4633,251 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
4058
4633
|
return result, 0
|
|
4059
4634
|
|
|
4060
4635
|
|
|
4636
|
+
def _full_exit_2(message: str) -> int:
|
|
4637
|
+
"""Exit 2 of full mode (an input error, a failed -o write or a crash):
|
|
4638
|
+
`message` on one stderr line, each line break and the whitespace around
|
|
4639
|
+
it joined into one space (a YAML error spans several lines) and every
|
|
4640
|
+
other space kept, so a path is quoted as written. Callers quote the line
|
|
4641
|
+
as the reason they stop or fall back."""
|
|
4642
|
+
sys.stderr.write("error: " + re.sub(r"\s*[\r\n]\s*", " ", message.strip("\r\n")) + "\n")
|
|
4643
|
+
return 2
|
|
4644
|
+
|
|
4645
|
+
|
|
4061
4646
|
def _main_full(args: argparse.Namespace) -> int:
|
|
4062
4647
|
if not args.source_root:
|
|
4063
|
-
|
|
4064
|
-
return 2
|
|
4648
|
+
return _full_exit_2("--mode full needs --source-root")
|
|
4065
4649
|
if args.head_cap is not None and args.head_cap < 0:
|
|
4066
|
-
|
|
4067
|
-
return 2
|
|
4650
|
+
return _full_exit_2("--head-cap must be 0 (no cap) or more")
|
|
4068
4651
|
if args.timeout is not None and not args.timeout > 0:
|
|
4069
|
-
|
|
4070
|
-
return 2
|
|
4652
|
+
return _full_exit_2("--timeout must be more than 0 seconds")
|
|
4071
4653
|
try:
|
|
4072
4654
|
result, code = run_full(args)
|
|
4073
4655
|
except RunnerError as exc:
|
|
4074
|
-
|
|
4075
|
-
return 2
|
|
4656
|
+
return _full_exit_2(str(exc))
|
|
4076
4657
|
except Exception as exc: # noqa: BLE001 - one stderr line and exit 2, never a traceback
|
|
4077
|
-
|
|
4078
|
-
sys.stderr.write(f"error: the run failed: {type(exc).__name__}: {detail}\n")
|
|
4079
|
-
return 2
|
|
4658
|
+
return _full_exit_2(f"the run failed: {type(exc).__name__}: {exc}")
|
|
4080
4659
|
text = json.dumps(result, indent=2) + "\n"
|
|
4081
4660
|
if args.output:
|
|
4082
4661
|
try:
|
|
4083
4662
|
Path(args.output).write_text(text, encoding="utf-8")
|
|
4084
4663
|
except OSError as exc:
|
|
4085
|
-
|
|
4086
|
-
return 2
|
|
4664
|
+
return _full_exit_2(f"cannot write {args.output}: {exc}")
|
|
4087
4665
|
else:
|
|
4088
4666
|
sys.stdout.write(text)
|
|
4089
4667
|
return code
|
|
4090
4668
|
|
|
4091
4669
|
|
|
4670
|
+
# --------------------------------------------------------------------------
|
|
4671
|
+
# Denominator mode: the counts and the effective_denominator decision of
|
|
4672
|
+
# names counted without the runner's diff
|
|
4673
|
+
# --------------------------------------------------------------------------
|
|
4674
|
+
|
|
4675
|
+
NO_ARMS = {"monorepo": False, "monorepo_kind": None, "specific_modules": False, "multi_subpath_exports": False}
|
|
4676
|
+
|
|
4677
|
+
|
|
4678
|
+
class DenominatorInputError(Exception):
|
|
4679
|
+
"""Exit 2 of denominator mode."""
|
|
4680
|
+
|
|
4681
|
+
|
|
4682
|
+
def _name_family(language: object, file: object) -> str:
|
|
4683
|
+
"""The language family a name counts in: its file's, else its language's."""
|
|
4684
|
+
family = FAMILY_OF.get(_file_language(file)) if isinstance(file, str) and file else None
|
|
4685
|
+
if family is not None:
|
|
4686
|
+
return family
|
|
4687
|
+
value = str(language or "").strip().lower()
|
|
4688
|
+
selected = LANGUAGE_FAMILIES.get(value)
|
|
4689
|
+
return FAMILY_OF[selected[0]] if selected else value
|
|
4690
|
+
|
|
4691
|
+
|
|
4692
|
+
def _payload_names(items: object, where: str) -> list[dict]:
|
|
4693
|
+
"""The {name, file, language, members} items of a payload list, checked."""
|
|
4694
|
+
if items is None:
|
|
4695
|
+
return []
|
|
4696
|
+
if not isinstance(items, list):
|
|
4697
|
+
raise DenominatorInputError(f"{where} is not a list")
|
|
4698
|
+
names = []
|
|
4699
|
+
for i, item in enumerate(items):
|
|
4700
|
+
if not isinstance(item, dict) or not isinstance(item.get("name"), str) or not item["name"].strip():
|
|
4701
|
+
raise DenominatorInputError(f"{where}[{i}] is not an object with a name")
|
|
4702
|
+
if item.get("file") is not None and not isinstance(item["file"], str):
|
|
4703
|
+
raise DenominatorInputError(f"{where}[{i}].file is not a path")
|
|
4704
|
+
members = item.get("members")
|
|
4705
|
+
if members is not None:
|
|
4706
|
+
if not isinstance(members, list):
|
|
4707
|
+
raise DenominatorInputError(f"{where}[{i}].members is not a list")
|
|
4708
|
+
members = _payload_names(members, f"{where}[{i}].members")
|
|
4709
|
+
file = _norm(item["file"]) if item.get("file") else None
|
|
4710
|
+
names.append({"name": item["name"].strip(), "file": file, "language": item.get("language"),
|
|
4711
|
+
"members": members})
|
|
4712
|
+
return names
|
|
4713
|
+
|
|
4714
|
+
|
|
4715
|
+
def _read_json_file(path: str, flag: str) -> dict:
|
|
4716
|
+
try:
|
|
4717
|
+
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
4718
|
+
except (OSError, UnicodeDecodeError, ValueError) as exc:
|
|
4719
|
+
raise DenominatorInputError(f"cannot read {flag} {path}: {exc}") from exc
|
|
4720
|
+
if not isinstance(data, dict):
|
|
4721
|
+
raise DenominatorInputError(f"{flag} {path} is not a JSON object")
|
|
4722
|
+
return data
|
|
4723
|
+
|
|
4724
|
+
|
|
4725
|
+
def run_denominator(args: argparse.Namespace, payload: dict) -> dict:
|
|
4726
|
+
"""The counts of the public names `payload`, --quick and --extraction
|
|
4727
|
+
give, with the effective_denominator decision full mode makes."""
|
|
4728
|
+
public: dict[tuple[str, str], dict] = {}
|
|
4729
|
+
runner_public: dict[tuple[str, str], dict] = {}
|
|
4730
|
+
# The exports found, each key at its file: the payload's and the runner's.
|
|
4731
|
+
found: dict[tuple[str, str], str | None] = {}
|
|
4732
|
+
internal: set[tuple[str, str]] = set()
|
|
4733
|
+
incomplete = False
|
|
4734
|
+
runner_status = None
|
|
4735
|
+
if args.extraction:
|
|
4736
|
+
runner = _read_json_file(args.extraction, "--extraction")
|
|
4737
|
+
if runner.get("mode") != "full":
|
|
4738
|
+
raise DenominatorInputError(f"--extraction {args.extraction} is not --mode full output")
|
|
4739
|
+
runner_status = runner.get("status")
|
|
4740
|
+
incomplete = runner_status == "incomplete" or bool(runner.get("errors"))
|
|
4741
|
+
diff = runner.get("entry_point_diff") if isinstance(runner.get("entry_point_diff"), dict) else {}
|
|
4742
|
+
for info in diff.get("public") or []:
|
|
4743
|
+
if isinstance(info, dict) and isinstance(info.get("name"), str):
|
|
4744
|
+
key = (str(info.get("language") or ""), info["name"])
|
|
4745
|
+
public[key] = runner_public[key] = info
|
|
4746
|
+
found[key] = info.get("file")
|
|
4747
|
+
for item in diff.get("internal") or []:
|
|
4748
|
+
if isinstance(item, dict) and isinstance(item.get("name"), str):
|
|
4749
|
+
internal.add((str(item.get("language") or ""), item["name"]))
|
|
4750
|
+
found[(str(item.get("language") or ""), item["name"])] = item.get("source_file")
|
|
4751
|
+
for path in args.quick or []:
|
|
4752
|
+
quick = _read_json_file(path, "--quick")
|
|
4753
|
+
if not isinstance(quick.get("exports"), list):
|
|
4754
|
+
raise DenominatorInputError(f"--quick {path} is not --mode quick output (no exports list)")
|
|
4755
|
+
for item in quick["exports"]:
|
|
4756
|
+
if isinstance(item, dict) and isinstance(item.get("name"), str) and item["name"]:
|
|
4757
|
+
file = item.get("defined_in") or item.get("source_file")
|
|
4758
|
+
file = _norm(file) if isinstance(file, str) and file else None
|
|
4759
|
+
public[(_name_family(quick.get("language"), file), item["name"])] = {
|
|
4760
|
+
"name": item["name"], "file": file, "via": "declaration"}
|
|
4761
|
+
for item in _payload_names(payload.get("public"), "public"):
|
|
4762
|
+
family = _name_family(item["language"], item["file"])
|
|
4763
|
+
info = {"name": item["name"], "file": item["file"], "via": "declaration"}
|
|
4764
|
+
if item["members"] is not None:
|
|
4765
|
+
info.update(via="namespace", members=[{"name": m["name"], "file": m["file"]} for m in item["members"]])
|
|
4766
|
+
public[(family, item["name"])] = info
|
|
4767
|
+
for item in _payload_names(payload.get("exports"), "exports"):
|
|
4768
|
+
key = (_name_family(item["language"], item["file"]), item["name"])
|
|
4769
|
+
found.setdefault(key, item["file"])
|
|
4770
|
+
if not item["name"].startswith("_"):
|
|
4771
|
+
internal.add(key)
|
|
4772
|
+
named = set(public) | {(family, m["name"]) for (family, _), info in public.items() if _has_members(info)
|
|
4773
|
+
for m in info["members"]}
|
|
4774
|
+
counted = _public_keys(public)
|
|
4775
|
+
|
|
4776
|
+
if args.source_root:
|
|
4777
|
+
root = Path(args.source_root)
|
|
4778
|
+
if not root.is_dir():
|
|
4779
|
+
raise RunnerError(f"source root not found: {args.source_root}")
|
|
4780
|
+
facts = _scope_facts(args, root)
|
|
4781
|
+
else:
|
|
4782
|
+
# No local tree (a remote source read by eye): the repository's listing, else the names' files.
|
|
4783
|
+
facts = _brief_scope(args)
|
|
4784
|
+
in_scope = without_recipes = None
|
|
4785
|
+
unmatched_include: list[str] = []
|
|
4786
|
+
unmatched_tier_a: list[str] = []
|
|
4787
|
+
if args.tree_file is not None:
|
|
4788
|
+
try:
|
|
4789
|
+
listed, cut = _read_listing(args.tree_file)
|
|
4790
|
+
except QuickInputError as exc:
|
|
4791
|
+
raise DenominatorInputError(str(exc)) from exc
|
|
4792
|
+
tree = sorted(set(listed))
|
|
4793
|
+
if cut:
|
|
4794
|
+
facts["warnings"].append("the listing was cut short: the file lists can miss files")
|
|
4795
|
+
in_scope, without_recipes, unmatched_include = _select_files(
|
|
4796
|
+
Path("."), tree, False, facts["includes"], facts["excludes"], facts["wanted"], facts["glob_match"],
|
|
4797
|
+
[], facts["warnings"])
|
|
4798
|
+
unmatched_tier_a = _unmatched_tier_a(facts["tier_a"], facts["includes"], tree, facts["glob_match"],
|
|
4799
|
+
facts["warnings"])
|
|
4800
|
+
else:
|
|
4801
|
+
tree = sorted({file for _, _, file in counted if file})
|
|
4802
|
+
facts.update(tree=tree, in_scope=in_scope, without_recipes=without_recipes,
|
|
4803
|
+
unmatched_include=unmatched_include, unmatched_tier_a=unmatched_tier_a, arms=dict(NO_ARMS),
|
|
4804
|
+
workspace_sources=[], entries=[], maps=[])
|
|
4805
|
+
warnings = facts["warnings"]
|
|
4806
|
+
tree_set = set(facts["tree"])
|
|
4807
|
+
outside_tree = sorted((name, file) for _, name, file in counted if file and file not in tree_set)
|
|
4808
|
+
warnings += [f"public name {name} is defined in {file}, which is not in the tree: effective_denominator "
|
|
4809
|
+
"and curated_subset leave it out" for name, file in outside_tree]
|
|
4810
|
+
counts = _denominator_counts(facts, counted, len(counted), len(internal - named), incomplete, warnings)
|
|
4811
|
+
if not args.source_root and counts["effective_denominator_decision"]["reason"] not in ("reference-app",
|
|
4812
|
+
"incomplete-run"):
|
|
4813
|
+
counts["effective_denominator_decision"] = {"store": False, "reason": "no-source-tree"}
|
|
4814
|
+
without_recipes = facts["without_recipes"]
|
|
4815
|
+
|
|
4816
|
+
def row(key: tuple[str, str], file: str | None) -> dict:
|
|
4817
|
+
return {"name": key[1], "language": key[0], "file": file}
|
|
4818
|
+
|
|
4819
|
+
def by_name(keys):
|
|
4820
|
+
return sorted(keys, key=lambda k: (k[1], k[0]))
|
|
4821
|
+
|
|
4822
|
+
# The names the payload and --quick gave (the runner diffed its own), a namespace as its members.
|
|
4823
|
+
given: dict[tuple[str, str], str | None] = {}
|
|
4824
|
+
for key, info in public.items():
|
|
4825
|
+
if info is runner_public.get(key):
|
|
4826
|
+
continue
|
|
4827
|
+
for member in info["members"] if _has_members(info) else [info]:
|
|
4828
|
+
given.setdefault((key[0], member["name"]), member["file"])
|
|
4829
|
+
return {
|
|
4830
|
+
"mode": "denominator",
|
|
4831
|
+
"source_root": args.source_root,
|
|
4832
|
+
"scope": _scope_record(args, facts),
|
|
4833
|
+
"files_in_scope": None if facts["in_scope"] is None else len(facts["in_scope"]),
|
|
4834
|
+
"files_without_recipes": None if without_recipes is None else _by_extension(without_recipes),
|
|
4835
|
+
"entry_points": {"files": facts["entries"], "exports_maps": facts["maps"]},
|
|
4836
|
+
"arms": facts["arms"],
|
|
4837
|
+
"runner_status": runner_status,
|
|
4838
|
+
"counts": counts,
|
|
4839
|
+
"entry_point_diff": {
|
|
4840
|
+
"public": [row((family, name), file)
|
|
4841
|
+
for family, name, file in sorted(counted, key=lambda k: (k[1], k[0], k[2] or ""))],
|
|
4842
|
+
"internal": [row(key, found.get(key)) for key in by_name(internal - named)],
|
|
4843
|
+
"extraction_gaps": [row(key, given[key]) for key in by_name(given) if key not in found],
|
|
4844
|
+
},
|
|
4845
|
+
"warnings": warnings,
|
|
4846
|
+
"files_in_scope_list": facts["in_scope"],
|
|
4847
|
+
"files_without_recipes_list": without_recipes,
|
|
4848
|
+
}
|
|
4849
|
+
|
|
4850
|
+
|
|
4851
|
+
def _main_denominator(args: argparse.Namespace) -> int:
|
|
4852
|
+
if args.source_root and args.tree_file is not None:
|
|
4853
|
+
return _full_exit_2("--tree-file stands for a remote source with no --source-root: pass one of them")
|
|
4854
|
+
# With `--tree-file -` stdin holds the listing, so there is no payload.
|
|
4855
|
+
reads_stdin = args.tree_file != "-" and sys.stdin is not None and not sys.stdin.isatty()
|
|
4856
|
+
raw = sys.stdin.read() if reads_stdin else ""
|
|
4857
|
+
try:
|
|
4858
|
+
payload = json.loads(raw) if raw.strip() else {}
|
|
4859
|
+
except ValueError as exc:
|
|
4860
|
+
return _full_exit_2(f"stdin is not JSON: {exc}")
|
|
4861
|
+
if not isinstance(payload, dict):
|
|
4862
|
+
return _full_exit_2("stdin must be a JSON object: {\"public\": [...], \"exports\": [...]}")
|
|
4863
|
+
try:
|
|
4864
|
+
result = run_denominator(args, payload)
|
|
4865
|
+
except (RunnerError, DenominatorInputError) as exc:
|
|
4866
|
+
return _full_exit_2(str(exc))
|
|
4867
|
+
except Exception as exc: # noqa: BLE001 - one stderr line and exit 2, never a traceback
|
|
4868
|
+
return _full_exit_2(f"the run failed: {type(exc).__name__}: {exc}")
|
|
4869
|
+
text = json.dumps(result, indent=2) + "\n"
|
|
4870
|
+
if args.output:
|
|
4871
|
+
try:
|
|
4872
|
+
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
|
4873
|
+
Path(args.output).write_text(text, encoding="utf-8")
|
|
4874
|
+
except OSError as exc:
|
|
4875
|
+
return _full_exit_2(f"cannot write {args.output}: {exc}")
|
|
4876
|
+
else:
|
|
4877
|
+
sys.stdout.write(text)
|
|
4878
|
+
return 0
|
|
4879
|
+
|
|
4880
|
+
|
|
4092
4881
|
# --------------------------------------------------------------------------
|
|
4093
4882
|
# Entries mode: the files quick mode reads, found in the listing
|
|
4094
4883
|
# --------------------------------------------------------------------------
|
|
@@ -4281,10 +5070,11 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
4281
5070
|
parser.add_argument(
|
|
4282
5071
|
"--mode",
|
|
4283
5072
|
default="quick",
|
|
4284
|
-
choices=("quick", "full", "entries"),
|
|
5073
|
+
choices=("quick", "full", "entries", "denominator"),
|
|
4285
5074
|
help="quick (default): parse the stdin payload, or the --manifest-file and --entry-file files; "
|
|
4286
5075
|
"full: run the recipes over --source-root; entries: list one package's entry files from "
|
|
4287
|
-
"--tree-file
|
|
5076
|
+
"--tree-file; denominator: count the public names a stdin payload, --quick and --extraction give, "
|
|
5077
|
+
"and decide whether to store effective_denominator, as full mode does.",
|
|
4288
5078
|
)
|
|
4289
5079
|
quick = parser.add_argument_group("quick mode, files read from disk")
|
|
4290
5080
|
quick.add_argument("--manifest-file", metavar="PATH",
|
|
@@ -4298,16 +5088,23 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
4298
5088
|
help=f"read each module an unlisted[] record names as one more --follow-file, round "
|
|
4299
5089
|
f"after round (at most {FOLLOW_ROUNDS}), until none is left; needs --tree-file and "
|
|
4300
5090
|
f"--source-root")
|
|
5091
|
+
quick.add_argument("--resolve-version", action="store_true",
|
|
5092
|
+
help="look up a version the manifest does not state: a version file, a Cargo workspace "
|
|
5093
|
+
"root, then a release tag (of --repo at --ref, else git describe in --source-root); "
|
|
5094
|
+
"needs --source-root")
|
|
4301
5095
|
quick.add_argument("--repo", metavar="OWNER/REPO",
|
|
4302
5096
|
help="with --follow: fetch each module from this GitHub repository into --source-root "
|
|
4303
|
-
"first (skf-github-fetch.py)"
|
|
5097
|
+
"first (skf-github-fetch.py); with --resolve-version: the repository whose tags "
|
|
5098
|
+
"it reads, and whose files it fetches with --tree-file")
|
|
4304
5099
|
quick.add_argument("--ref", metavar="REF",
|
|
4305
|
-
help="with --repo: the branch, tag or commit to fetch
|
|
5100
|
+
help="with --repo: the branch, tag or commit to fetch and read the version at "
|
|
5101
|
+
"(default: the default branch)")
|
|
4306
5102
|
listing = parser.add_argument_group("entries mode, and the listing quick mode follows chains in")
|
|
4307
5103
|
listing.add_argument("--tree-file", metavar="PATH",
|
|
4308
5104
|
help="the repository's file listing (skf-github-probe.py tree output, JSON, or one "
|
|
4309
|
-
"path per line): entries mode finds the entry files in it,
|
|
4310
|
-
"resolves each unlisted[] statement's module_file against it"
|
|
5105
|
+
"path per line): entries mode finds the entry files in it, quick mode "
|
|
5106
|
+
"resolves each unlisted[] statement's module_file against it, and denominator "
|
|
5107
|
+
"mode, with no --source-root, takes it as the tree (- reads it from stdin)")
|
|
4311
5108
|
listing.add_argument("--scope", metavar="FOLDER",
|
|
4312
5109
|
help="entries mode: the package folder to read (default: the repository root)")
|
|
4313
5110
|
listing.add_argument("--fetch-list", metavar="FILE",
|
|
@@ -4322,7 +5119,11 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
4322
5119
|
full.add_argument("--exclude", action="append", metavar="GLOB",
|
|
4323
5120
|
help="an exclude glob, repeatable (replaces the brief's scope.exclude)")
|
|
4324
5121
|
full.add_argument("--tier-a-include", action="append", metavar="GLOB",
|
|
4325
|
-
help="a tier-A glob for effective_denominator, repeatable (replaces scope.tier_a_include)"
|
|
5122
|
+
help="a tier-A glob for effective_denominator, repeatable (replaces scope.tier_a_include), "
|
|
5123
|
+
"that counts a name only in a file scope.include matches too; one that matches no "
|
|
5124
|
+
"file is listed in scope.unmatched_tier_a_include, one that matches only files "
|
|
5125
|
+
"scope.include leaves out is named in warnings, and a list that counts no public name "
|
|
5126
|
+
"gives way to scope.include (else every file)")
|
|
4326
5127
|
full.add_argument("--scope-type", help="the scope type (replaces the brief's scope.type)")
|
|
4327
5128
|
full.add_argument("--language", action="append",
|
|
4328
5129
|
help="a language family to extract, repeatable (replaces the brief's language); quick "
|
|
@@ -4339,6 +5140,14 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
4339
5140
|
help=f"stop the ast-grep runs after this many seconds and still write the JSON, "
|
|
4340
5141
|
f"status incomplete (default {DEFAULT_TIMEOUT_SEC})")
|
|
4341
5142
|
full.add_argument("-o", "--output", metavar="FILE", help="write the JSON here instead of stdout")
|
|
5143
|
+
denominator = parser.add_argument_group("denominator mode (with --source-root, --brief, the scope flags, "
|
|
5144
|
+
"--language and -o)")
|
|
5145
|
+
denominator.add_argument("--quick", action="append", metavar="FILE",
|
|
5146
|
+
help="a --mode quick output whose exports are public names, each at its defined_in "
|
|
5147
|
+
"(else its source_file), repeatable")
|
|
5148
|
+
denominator.add_argument("--extraction", metavar="FILE",
|
|
5149
|
+
help="a --mode full output whose entry_point_diff public and internal names count "
|
|
5150
|
+
"too; an incomplete one is never stored")
|
|
4342
5151
|
return parser
|
|
4343
5152
|
|
|
4344
5153
|
|
|
@@ -4346,10 +5155,14 @@ FULL_MODE_FLAGS = ("source_root", "brief", "include", "exclude", "tier_a_include
|
|
|
4346
5155
|
"files_from", "recipe_set", "recipes", "tier", "head_cap", "timeout", "output")
|
|
4347
5156
|
# The full mode flags quick mode takes with its file inputs, and entries mode always.
|
|
4348
5157
|
QUICK_FILE_FLAGS = ("source_root", "language")
|
|
5158
|
+
# The full mode flags denominator mode takes, and its own.
|
|
5159
|
+
DENOMINATOR_FLAGS = ("source_root", "brief", "include", "exclude", "tier_a_include", "scope_type", "language",
|
|
5160
|
+
"output")
|
|
5161
|
+
DENOMINATOR_OWN_FLAGS = ("quick", "extraction")
|
|
4349
5162
|
# The flags of entries mode; quick mode takes --tree-file and --fetch-list with its file inputs.
|
|
4350
5163
|
LISTING_FLAGS = ("tree_file", "scope", "fetch_list")
|
|
4351
5164
|
# Quick mode's own flags with its file inputs: the chains it follows itself.
|
|
4352
|
-
FOLLOW_FLAGS = ("follow", "repo", "ref")
|
|
5165
|
+
FOLLOW_FLAGS = ("follow", "repo", "ref", "resolve_version")
|
|
4353
5166
|
FOLLOW_ROUNDS = 5
|
|
4354
5167
|
|
|
4355
5168
|
|
|
@@ -4479,6 +5292,327 @@ def _follow(args: argparse.Namespace, result: dict) -> dict:
|
|
|
4479
5292
|
return result
|
|
4480
5293
|
|
|
4481
5294
|
|
|
5295
|
+
# --resolve-version: a version the manifest does not state
|
|
5296
|
+
_PY_VERSION_RE = re.compile(r"""^__version__[ \t]*(?::[^=\n]*)?=[ \t]*(['"])([^'"\n]+)\1""", re.M)
|
|
5297
|
+
_JS_PRERELEASE_RE = re.compile(r"-(?:alpha|beta|rc|next|canary)", re.I)
|
|
5298
|
+
JS_FAMILY = ("js", "ts", "javascript", "typescript")
|
|
5299
|
+
|
|
5300
|
+
|
|
5301
|
+
def _version_reader(args: argparse.Namespace, tree: list[str] | None) -> Callable[[str], str | None]:
|
|
5302
|
+
"""path -> the text of that file under --source-root, else None. With
|
|
5303
|
+
--repo and --tree-file, a file the listing holds and the folder does not
|
|
5304
|
+
is fetched there first, as --follow fetches a module. Raises
|
|
5305
|
+
QuickInputError."""
|
|
5306
|
+
root = Path(args.source_root)
|
|
5307
|
+
listed = set(tree or [])
|
|
5308
|
+
fetch: list = []
|
|
5309
|
+
|
|
5310
|
+
def read(path: str) -> str | None:
|
|
5311
|
+
target = root / path
|
|
5312
|
+
if not target.is_file() and args.repo is not None and args.tree_file is not None and path in listed:
|
|
5313
|
+
if not fetch:
|
|
5314
|
+
fetch.append(_follow_reader(args))
|
|
5315
|
+
fetch[0]([path])
|
|
5316
|
+
try:
|
|
5317
|
+
if target.is_file() and target.resolve().is_relative_to(root.resolve()):
|
|
5318
|
+
return target.read_bytes().decode("utf-8-sig", errors="replace")
|
|
5319
|
+
except OSError:
|
|
5320
|
+
pass
|
|
5321
|
+
return None
|
|
5322
|
+
return read
|
|
5323
|
+
|
|
5324
|
+
|
|
5325
|
+
def _version_value(value: object) -> str | None:
|
|
5326
|
+
"""A version a version file writes: a literal that starts with a digit, not a placeholder."""
|
|
5327
|
+
if not isinstance(value, str) or not re.match(r"\d", value.strip()):
|
|
5328
|
+
return None
|
|
5329
|
+
return None if _detect_placeholder_version(value) else value.strip()
|
|
5330
|
+
|
|
5331
|
+
|
|
5332
|
+
def _python_version_file(manifest: str | None, entries: list[str],
|
|
5333
|
+
read: Callable[[str], str | None]) -> tuple[str, str] | None:
|
|
5334
|
+
"""(version, path): the `__version__` literal of an entry __init__.py,
|
|
5335
|
+
then of the _version.py beside it, then setup.py's `version=` literal
|
|
5336
|
+
beside the manifest."""
|
|
5337
|
+
inits = [e for e in entries if e.rsplit("/", 1)[-1] == "__init__.py"]
|
|
5338
|
+
folder = posixpath.dirname(manifest or "")
|
|
5339
|
+
candidates = [(path, "py") for path in inits]
|
|
5340
|
+
candidates += [(posixpath.join(posixpath.dirname(path), "_version.py"), "py") for path in inits]
|
|
5341
|
+
if posixpath.basename(manifest or "") != "setup.py":
|
|
5342
|
+
candidates.append((posixpath.join(folder, "setup.py"), "setup.py"))
|
|
5343
|
+
for path, kind in dict.fromkeys(candidates):
|
|
5344
|
+
text = read(path)
|
|
5345
|
+
if text is None:
|
|
5346
|
+
continue
|
|
5347
|
+
if kind == "py":
|
|
5348
|
+
m = _PY_VERSION_RE.search(text)
|
|
5349
|
+
version = _version_value(m.group(2)) if m else None
|
|
5350
|
+
else:
|
|
5351
|
+
version = _version_value(parse_setup_py(text).get("version"))
|
|
5352
|
+
if version:
|
|
5353
|
+
return version, path
|
|
5354
|
+
return None
|
|
5355
|
+
|
|
5356
|
+
|
|
5357
|
+
def _cargo_workspace_version(manifest: str | None, read: Callable[[str], str | None]) -> tuple[str, str] | None:
|
|
5358
|
+
"""(version, path): the `[workspace.package] version` of the workspace
|
|
5359
|
+
root, the nearest Cargo.toml at or above the manifest's folder that
|
|
5360
|
+
holds a `[workspace]` table."""
|
|
5361
|
+
if not manifest:
|
|
5362
|
+
return None
|
|
5363
|
+
folder = posixpath.dirname(manifest)
|
|
5364
|
+
while True:
|
|
5365
|
+
path = posixpath.join(folder, "Cargo.toml")
|
|
5366
|
+
text = read(path)
|
|
5367
|
+
try:
|
|
5368
|
+
data = tomllib.loads(text) if text is not None else None
|
|
5369
|
+
except tomllib.TOMLDecodeError:
|
|
5370
|
+
data = None
|
|
5371
|
+
workspace = data.get("workspace") if isinstance(data, dict) else None
|
|
5372
|
+
if isinstance(workspace, dict):
|
|
5373
|
+
package = workspace.get("package")
|
|
5374
|
+
version = _version_value(package.get("version")) if isinstance(package, dict) else None
|
|
5375
|
+
return (version, path) if version else None
|
|
5376
|
+
if not folder:
|
|
5377
|
+
return None
|
|
5378
|
+
folder = posixpath.dirname(folder)
|
|
5379
|
+
|
|
5380
|
+
|
|
5381
|
+
def _release_tag(args: argparse.Namespace, warnings: list[str]) -> tuple[str, str] | None:
|
|
5382
|
+
"""(version, tag) of the tag a version is read from: with --repo, the
|
|
5383
|
+
`release` skf-github-probe.py's tag listing gives at --ref (the newest
|
|
5384
|
+
release tag at the default branch, the pinned tag itself at a tag);
|
|
5385
|
+
without, the tag `git describe --tags --abbrev=0` names in
|
|
5386
|
+
--source-root, when its name is a version. The version drops a
|
|
5387
|
+
leading v."""
|
|
5388
|
+
try:
|
|
5389
|
+
probe = _sibling("skf-github-probe.py")
|
|
5390
|
+
except RunnerError as exc:
|
|
5391
|
+
warnings.append(f"--resolve-version could not read tags: {exc}")
|
|
5392
|
+
return None
|
|
5393
|
+
if args.repo is not None:
|
|
5394
|
+
parsed = probe.parse_repo(args.repo)
|
|
5395
|
+
if parsed is None:
|
|
5396
|
+
warnings.append(f"--resolve-version read no tags: {args.repo} is not a github.com repository")
|
|
5397
|
+
return None
|
|
5398
|
+
probe._start_clock(probe.DEFAULT_TIMEOUT_SEC)
|
|
5399
|
+
out = probe.probe_tags(*parsed, [], 1, ref=args.ref)
|
|
5400
|
+
if out["status"] != "ok":
|
|
5401
|
+
warnings.append(f"the tags of {args.repo} could not be listed ({out['message']}), so no release "
|
|
5402
|
+
f"tag gave a version")
|
|
5403
|
+
return None
|
|
5404
|
+
release = out["release"]
|
|
5405
|
+
return (release["version"], release["tag"]) if release else None
|
|
5406
|
+
try:
|
|
5407
|
+
res = _sibling("skf-source-tree.py")._git(Path(args.source_root), "describe", "--tags", "--abbrev=0")
|
|
5408
|
+
except RunnerError:
|
|
5409
|
+
return None
|
|
5410
|
+
if res is None or res[0] != 0:
|
|
5411
|
+
return None
|
|
5412
|
+
tag = res[1].decode("utf-8", errors="replace").strip()
|
|
5413
|
+
version = probe.tag_version(tag)
|
|
5414
|
+
return (version, tag) if version else None
|
|
5415
|
+
|
|
5416
|
+
|
|
5417
|
+
def resolve_version(args: argparse.Namespace, result: dict, tree: list[str] | None) -> None:
|
|
5418
|
+
"""--resolve-version: when the manifest states no version, look it up
|
|
5419
|
+
(Python: _python_version_file; Rust: _cargo_workspace_version; then any
|
|
5420
|
+
language: _release_tag), and for JavaScript or TypeScript read from
|
|
5421
|
+
--repo at the default branch, let the newest release replace a manifest
|
|
5422
|
+
pre-release (-alpha, -beta, -rc, -next, -canary). Sets `version` and
|
|
5423
|
+
`version_source`: "manifest", "file:<path>", "tag:<tag>", or null with
|
|
5424
|
+
a warning when nothing gave one. Raises QuickInputError."""
|
|
5425
|
+
warnings = result["warnings"]
|
|
5426
|
+
version = result["version"]
|
|
5427
|
+
source = "manifest" if version is not None else None
|
|
5428
|
+
manifest = Path(args.manifest_file).as_posix() if args.manifest_file is not None else None
|
|
5429
|
+
if version is None:
|
|
5430
|
+
read = _version_reader(args, tree)
|
|
5431
|
+
found = None
|
|
5432
|
+
if result["language"] == "python":
|
|
5433
|
+
found = _python_version_file(manifest, [Path(e).as_posix() for e in args.entry_file or []], read)
|
|
5434
|
+
elif result["language"] == "rust":
|
|
5435
|
+
found = _cargo_workspace_version(manifest, read)
|
|
5436
|
+
if found:
|
|
5437
|
+
version, source = found[0], f"file:{found[1]}"
|
|
5438
|
+
prerelease = (version is not None and source == "manifest" and result["language"] in JS_FAMILY
|
|
5439
|
+
and args.repo is not None and (args.ref or "").strip().lower() in ("", "null", "none", "head")
|
|
5440
|
+
and _JS_PRERELEASE_RE.search(version) is not None)
|
|
5441
|
+
if version is None or prerelease:
|
|
5442
|
+
tag = _release_tag(args, warnings)
|
|
5443
|
+
if tag and prerelease:
|
|
5444
|
+
warnings.append(f"manifest version {version!r} is a pre-release on the default branch; version is "
|
|
5445
|
+
f"the newest release, {tag[0]} (tag {tag[1]})")
|
|
5446
|
+
if tag:
|
|
5447
|
+
version, source = tag[0], f"tag:{tag[1]}"
|
|
5448
|
+
if version is None:
|
|
5449
|
+
warnings.append("no version found in the manifest, a version file or a release tag; version is null")
|
|
5450
|
+
result["version"] = version
|
|
5451
|
+
result["version_source"] = source
|
|
5452
|
+
|
|
5453
|
+
|
|
5454
|
+
_JS_LIST_SOURCE_RE = re.compile(r"\s*from\s*(['\"])")
|
|
5455
|
+
_JS_IMPORT_RE = re.compile(r"^import\s+(?:type\s+)?([A-Za-z_$*{][^;'\"]*?)\s*from\s*(['\"])", re.M)
|
|
5456
|
+
_JS_BRACES_RE = re.compile(r"\{([^}]*)\}")
|
|
5457
|
+
DEFINED_IN_FAMILIES = {"python": "python", "js": "js", "ts": "js", "javascript": "js", "typescript": "js"}
|
|
5458
|
+
|
|
5459
|
+
|
|
5460
|
+
def _quick_js_facts(content: str) -> dict:
|
|
5461
|
+
"""What one JS/TS module exports, read from its text as quick mode reads
|
|
5462
|
+
an entry file, in _js_facts's shape: {"names": {exposed name: {"how",
|
|
5463
|
+
"local", "source"}}, "stars": [source], "default": the local name of
|
|
5464
|
+
its default export, or None}. A declaration beats a list item, and an
|
|
5465
|
+
exported binding an import brings in is that import's re-export (or
|
|
5466
|
+
namespace, for `import * as`)."""
|
|
5467
|
+
code = _blank(content, "js")
|
|
5468
|
+
imports: dict[str, tuple[str, str]] = {}
|
|
5469
|
+
for m in _JS_IMPORT_RE.finditer(code):
|
|
5470
|
+
source = _js_string(content, code, m.end(2) - 1)
|
|
5471
|
+
braces = _JS_BRACES_RE.search(m.group(1))
|
|
5472
|
+
for piece in braces.group(1).split(",") if braces else []:
|
|
5473
|
+
words = piece.split()
|
|
5474
|
+
if words and _JS_NAME_RE.fullmatch(words[-1]):
|
|
5475
|
+
imports.setdefault(words[-1], (_specifier_local(piece), source))
|
|
5476
|
+
for part in _JS_BRACES_RE.sub("", m.group(1)).split(","):
|
|
5477
|
+
words = part.split()
|
|
5478
|
+
if len(words) == 3 and words[:2] == ["*", "as"]:
|
|
5479
|
+
imports.setdefault(words[2], ("*", source))
|
|
5480
|
+
elif len(words) == 1 and _JS_NAME_RE.fullmatch(words[0]):
|
|
5481
|
+
imports.setdefault(words[0], ("default", source))
|
|
5482
|
+
names: dict[str, dict] = {}
|
|
5483
|
+
|
|
5484
|
+
def put(name: str, how: str, local: str | None, source: str | None) -> None:
|
|
5485
|
+
names.setdefault(name, {"how": how, "local": local, "source": source})
|
|
5486
|
+
|
|
5487
|
+
def bound(name: str, local: str) -> None:
|
|
5488
|
+
imported, source = imports.get(local, (None, None))
|
|
5489
|
+
if imported == "*":
|
|
5490
|
+
put(name, "namespace", None, source)
|
|
5491
|
+
elif imported is not None:
|
|
5492
|
+
put(name, "re-export", imported, source)
|
|
5493
|
+
else:
|
|
5494
|
+
put(name, "declaration", local, None)
|
|
5495
|
+
|
|
5496
|
+
for export in scan_exports_js(content, "", None, None, defaults=False):
|
|
5497
|
+
if export["type"] not in ("re-export", "namespace"):
|
|
5498
|
+
put(export["name"], "declaration", export["name"], None)
|
|
5499
|
+
default = None
|
|
5500
|
+
for m in _JS_DECL_RE.finditer(code):
|
|
5501
|
+
if m.group(1) and "default" in m.group(1).split():
|
|
5502
|
+
default = default or m.group(3)
|
|
5503
|
+
put("default", "declaration", m.group(3), None)
|
|
5504
|
+
elif m.group(1) and _js_type(m.group(2)) == "namespace":
|
|
5505
|
+
put(m.group(3), "declaration", m.group(3), None) # `export namespace X`, `export module X`
|
|
5506
|
+
stars: list[str] = []
|
|
5507
|
+
for m in _JS_STAR_RE.finditer(code):
|
|
5508
|
+
source = _js_string(content, code, m.end(2) - 1)
|
|
5509
|
+
if m.group(1):
|
|
5510
|
+
put(m.group(1), "namespace", None, source)
|
|
5511
|
+
else:
|
|
5512
|
+
stars.append(source)
|
|
5513
|
+
for m in _JS_LIST_RE.finditer(code):
|
|
5514
|
+
after = _JS_LIST_SOURCE_RE.match(code, m.end())
|
|
5515
|
+
source = _js_string(content, code, after.end(1) - 1) if after else None
|
|
5516
|
+
for piece in m.group(1).split(","):
|
|
5517
|
+
words = piece.split()
|
|
5518
|
+
if not words or not _JS_NAME_RE.fullmatch(words[-1]):
|
|
5519
|
+
continue
|
|
5520
|
+
if source is not None:
|
|
5521
|
+
put(words[-1], "re-export", _specifier_local(piece), source)
|
|
5522
|
+
else:
|
|
5523
|
+
bound(words[-1], _specifier_local(piece))
|
|
5524
|
+
for m in _JS_DEFAULT_RE.finditer(code):
|
|
5525
|
+
if m.group(1) not in _JS_NOT_NAMES:
|
|
5526
|
+
default = default or m.group(1)
|
|
5527
|
+
bound("default", m.group(1))
|
|
5528
|
+
return {"names": names, "stars": stars, "default": default}
|
|
5529
|
+
|
|
5530
|
+
|
|
5531
|
+
def _js_defined_in(root: Path, tree: set[str]) -> Callable[[str, str], str | None]:
|
|
5532
|
+
"""(module, name it exports) -> the file that defines that name, as
|
|
5533
|
+
_js_surface's resolve traces it: through each re-export, imported
|
|
5534
|
+
binding and `export *` (a cycle ends the trace at the module it
|
|
5535
|
+
reached), a namespace export giving its module's file; None for a
|
|
5536
|
+
re-export or a namespace whose module no file under `root` holds."""
|
|
5537
|
+
cache: dict[str, dict] = {}
|
|
5538
|
+
|
|
5539
|
+
def facts(file: str) -> dict:
|
|
5540
|
+
if file not in cache:
|
|
5541
|
+
cache[file] = _quick_js_facts(_read_text(root, file) or "")
|
|
5542
|
+
return cache[file]
|
|
5543
|
+
|
|
5544
|
+
def owner(file: str, name: str) -> str | None:
|
|
5545
|
+
"""The first module the `export *` chain of `file` passes `name` on from."""
|
|
5546
|
+
stack, visited = [file], set()
|
|
5547
|
+
while stack:
|
|
5548
|
+
module = stack.pop()
|
|
5549
|
+
if module in visited:
|
|
5550
|
+
continue
|
|
5551
|
+
visited.add(module)
|
|
5552
|
+
if module != file and name in facts(module)["names"]:
|
|
5553
|
+
return module
|
|
5554
|
+
stack += reversed([t for t in (resolve_js_module(s, module, tree) for s in facts(module)["stars"]) if t])
|
|
5555
|
+
return None
|
|
5556
|
+
|
|
5557
|
+
def resolve(file: str, name: str) -> str | None:
|
|
5558
|
+
if name not in facts(file)["names"] and name == facts(file)["default"]:
|
|
5559
|
+
name = "default" # quick mode names a default export by what it declares
|
|
5560
|
+
visited: set[tuple[str, str]] = set()
|
|
5561
|
+
while (file, name) not in visited:
|
|
5562
|
+
visited.add((file, name))
|
|
5563
|
+
fact = facts(file)["names"].get(name)
|
|
5564
|
+
if fact is None:
|
|
5565
|
+
found = owner(file, name) if name != "default" else None
|
|
5566
|
+
if found is None:
|
|
5567
|
+
return file
|
|
5568
|
+
file = found
|
|
5569
|
+
continue
|
|
5570
|
+
if fact["how"] == "declaration":
|
|
5571
|
+
return file
|
|
5572
|
+
target = resolve_js_module(fact["source"], file, tree)
|
|
5573
|
+
if fact["how"] == "namespace" or target is None:
|
|
5574
|
+
return target
|
|
5575
|
+
file, name = target, fact["local"]
|
|
5576
|
+
return file
|
|
5577
|
+
|
|
5578
|
+
return resolve
|
|
5579
|
+
|
|
5580
|
+
|
|
5581
|
+
def _py_defined_in(root: Path, tree: set[str]) -> Callable[[str, str], str | None]:
|
|
5582
|
+
"""(module, name it exports) -> the file full mode's _py_surface traces
|
|
5583
|
+
that name to from that module (None when it gives none: a name another
|
|
5584
|
+
package passes on, or one the trace does not list)."""
|
|
5585
|
+
traced: dict[str, dict[str, str | None]] = {}
|
|
5586
|
+
|
|
5587
|
+
def resolve(file: str, name: str) -> str | None:
|
|
5588
|
+
if file not in traced:
|
|
5589
|
+
surface = _Surface()
|
|
5590
|
+
_py_surface(root, [{"file": file}], tree, surface, [])
|
|
5591
|
+
traced[file] = {n: info["file"] for n, info in surface.names.items()}
|
|
5592
|
+
return traced[file].get(name)
|
|
5593
|
+
|
|
5594
|
+
return resolve
|
|
5595
|
+
|
|
5596
|
+
|
|
5597
|
+
def _defined_in(root: Path, result: dict) -> None:
|
|
5598
|
+
"""Give each export of a Python, JavaScript or TypeScript quick result
|
|
5599
|
+
its `defined_in` (see "Quick mode" above): the file the export is
|
|
5600
|
+
declared in for a name its file binds itself, else the file the trace
|
|
5601
|
+
reaches. The tree under `root` is listed only when a name is traced."""
|
|
5602
|
+
family = DEFINED_IN_FAMILIES.get(result.get("language") or "")
|
|
5603
|
+
if family is None:
|
|
5604
|
+
return
|
|
5605
|
+
resolve: Callable[[str, str], str | None] | None = None
|
|
5606
|
+
for export in result["exports"]:
|
|
5607
|
+
if export["type"] not in ("re-export", "namespace", "module"):
|
|
5608
|
+
export["defined_in"] = export["source_file"]
|
|
5609
|
+
continue
|
|
5610
|
+
if resolve is None:
|
|
5611
|
+
tree = set(list_tree(root))
|
|
5612
|
+
resolve = (_py_defined_in if family == "python" else _js_defined_in)(root, tree)
|
|
5613
|
+
export["defined_in"] = resolve(export["source_file"], export["name"])
|
|
5614
|
+
|
|
5615
|
+
|
|
4482
5616
|
def _main_entries(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
4483
5617
|
"""`--mode entries`: the entry files of one package, as JSON on stdout."""
|
|
4484
5618
|
if args.manifest_file is not None or args.entry_file or args.follow_file:
|
|
@@ -4517,9 +5651,20 @@ def main(argv: list[str]) -> int:
|
|
|
4517
5651
|
args = parser.parse_args(argv)
|
|
4518
5652
|
files = args.manifest_file is not None or bool(args.entry_file) or bool(args.follow_file)
|
|
4519
5653
|
listing = [f"--{flag.replace('_', '-')}" for flag in LISTING_FLAGS if getattr(args, flag) is not None]
|
|
4520
|
-
following = [f"--{flag}" for flag in FOLLOW_FLAGS if getattr(args, flag) not in (None, False)]
|
|
5654
|
+
following = [f"--{flag.replace('_', '-')}" for flag in FOLLOW_FLAGS if getattr(args, flag) not in (None, False)]
|
|
4521
5655
|
if following and (args.mode != "quick" or not files):
|
|
4522
5656
|
parser.error(f"{', '.join(following)}: quick mode only, with --manifest-file or --entry-file")
|
|
5657
|
+
own = [f"--{flag}" for flag in DENOMINATOR_OWN_FLAGS if getattr(args, flag) is not None]
|
|
5658
|
+
if own and args.mode != "denominator":
|
|
5659
|
+
parser.error(f"{', '.join(own)}: denominator mode only (--mode denominator)")
|
|
5660
|
+
if args.mode == "denominator":
|
|
5661
|
+
other = [f"--{flag.replace('_', '-')}" for flag in FULL_MODE_FLAGS
|
|
5662
|
+
if getattr(args, flag) is not None and flag not in DENOMINATOR_FLAGS]
|
|
5663
|
+
listing = [flag for flag in listing if flag != "--tree-file"]
|
|
5664
|
+
if files or listing or other:
|
|
5665
|
+
parser.error(f"{', '.join(other + listing) or '--manifest-file, --entry-file, --follow-file'}: "
|
|
5666
|
+
"not denominator mode's")
|
|
5667
|
+
return _main_denominator(args)
|
|
4523
5668
|
if args.mode == "full":
|
|
4524
5669
|
if files:
|
|
4525
5670
|
parser.error("--manifest-file, --entry-file, --follow-file: quick mode only (--mode quick)")
|
|
@@ -4543,17 +5688,24 @@ def main(argv: list[str]) -> int:
|
|
|
4543
5688
|
if files:
|
|
4544
5689
|
if not args.language or len(args.language) != 1:
|
|
4545
5690
|
parser.error("--manifest-file and --entry-file take one --language")
|
|
4546
|
-
if (args.repo is not None or args.ref is not None) and not args.follow:
|
|
4547
|
-
parser.error("--repo and --ref fetch the modules --follow reads
|
|
5691
|
+
if (args.repo is not None or args.ref is not None) and not (args.follow or args.resolve_version):
|
|
5692
|
+
parser.error("--repo and --ref fetch the modules --follow reads and name the tags --resolve-version "
|
|
5693
|
+
"reads, so pass --follow or --resolve-version")
|
|
4548
5694
|
if args.ref is not None and args.repo is None:
|
|
4549
5695
|
parser.error("--ref is the ref of --repo, so pass --repo")
|
|
4550
5696
|
if args.follow and (args.tree_file is None or args.source_root is None):
|
|
4551
5697
|
parser.error("--follow finds each module in --tree-file and reads it under --source-root: pass both")
|
|
5698
|
+
if args.resolve_version and args.source_root is None:
|
|
5699
|
+
parser.error("--resolve-version reads the version files under --source-root: pass it")
|
|
4552
5700
|
try:
|
|
4553
5701
|
payload = _quick_files_payload(args)
|
|
4554
5702
|
result = extract(payload)
|
|
4555
5703
|
if args.follow and "_error" not in result:
|
|
4556
5704
|
result = _follow(args, result)
|
|
5705
|
+
if args.resolve_version and "_error" not in result:
|
|
5706
|
+
resolve_version(args, result, payload.get("tree"))
|
|
5707
|
+
if args.source_root is not None and "_error" not in result:
|
|
5708
|
+
_defined_in(Path(args.source_root), result)
|
|
4557
5709
|
if args.fetch_list is not None and "_error" not in result:
|
|
4558
5710
|
follow = [item["module_file"] for item in result["unlisted"] if item["module_file"]]
|
|
4559
5711
|
_write_fetch_list(args.fetch_list, list(dict.fromkeys(follow)))
|