bmad-module-skill-forge 3.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +3 -3
- package/docs/_data/pinned.yaml +19 -13
- package/docs/architecture.md +2 -2
- package/docs/concepts.md +4 -4
- package/docs/examples.md +33 -26
- package/docs/getting-started.md +3 -3
- package/docs/how-it-works.md +2 -2
- package/docs/index.md +1 -1
- package/docs/skill-model.md +41 -39
- package/docs/troubleshooting.md +8 -0
- package/docs/verifying-a-skill.md +46 -21
- package/docs/why-skf.md +1 -1
- package/docs/workflows.md +5 -5
- package/package.json +1 -1
- package/src/knowledge/qmd-registry.md +1 -1
- package/src/knowledge/version-paths.md +1 -1
- package/src/shared/health-check.md +2 -50
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +12 -4
- package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +212 -59
- package/src/shared/scripts/skf-build-change-manifest.py +1437 -178
- package/src/shared/scripts/skf-classify-changed-files.py +78 -0
- package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
- package/src/shared/scripts/skf-count-tokens.py +54 -10
- package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
- package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
- package/src/shared/scripts/skf-detect-workspaces.py +67 -0
- package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
- package/src/shared/scripts/skf-extract-public-api.py +1673 -189
- package/src/shared/scripts/skf-extraction-inventory.py +584 -46
- package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
- package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
- package/src/shared/scripts/skf-github-probe.py +47 -8
- package/src/shared/scripts/skf-hash-content.py +14 -8
- package/src/shared/scripts/skf-load-provenance.py +3 -1
- package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
- package/src/shared/scripts/skf-parse-gaps.py +135 -26
- package/src/shared/scripts/skf-preflight.py +15 -1
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
- package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
- package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
- package/src/shared/scripts/skf-resolve-customization.py +197 -0
- package/src/shared/scripts/skf-shard-body.py +239 -41
- package/src/shared/scripts/skf-structural-diff.py +133 -18
- package/src/shared/scripts/skf-update-active-symlink.py +115 -62
- package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
- package/src/shared/scripts/skf-validate-output.py +12 -3
- package/src/shared/scripts/skf-verify-provenance-completeness.py +1439 -79
- package/src/shared/scripts/skf-write-skill-brief.py +41 -8
- package/src/shared/tool-requirements.yaml +8 -2
- package/src/skf-analyze-source/SKILL.md +2 -2
- package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
- package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
- package/src/skf-audit-skill/SKILL.md +2 -2
- package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
- package/src/skf-audit-skill/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +2 -1
- package/src/skf-audit-skill/references/re-index.md +14 -9
- package/src/skf-audit-skill/references/report.md +2 -2
- package/src/skf-audit-skill/references/semantic-diff.md +15 -15
- package/src/skf-audit-skill/references/structural-diff.md +10 -5
- package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
- package/src/skf-brief-skill/SKILL.md +3 -3
- package/src/skf-brief-skill/assets/scope-templates.md +1 -1
- package/src/skf-brief-skill/assets/skill-brief-schema.md +4 -11
- package/src/skf-brief-skill/references/analyze-target.md +6 -6
- package/src/skf-brief-skill/references/confirm-brief.md +1 -1
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
- package/src/skf-brief-skill/references/version-resolution.md +5 -9
- package/src/skf-campaign/SKILL.md +2 -2
- package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
- package/src/skf-create-skill/SKILL.md +2 -2
- package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
- package/src/skf-create-skill/assets/skill-sections.md +7 -7
- package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
- package/src/skf-create-skill/references/compile.md +31 -60
- package/src/skf-create-skill/references/component-extraction.md +3 -3
- package/src/skf-create-skill/references/enrich.md +2 -2
- package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
- package/src/skf-create-skill/references/extract.md +23 -19
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
- package/src/skf-create-skill/references/extraction-patterns.md +10 -11
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +21 -5
- package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
- package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
- package/src/skf-create-stack-skill/SKILL.md +2 -2
- package/src/skf-create-stack-skill/references/validate.md +1 -1
- package/src/skf-drop-skill/SKILL.md +2 -2
- package/src/skf-drop-skill/references/execute.md +5 -1
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +2 -2
- package/src/skf-drop-skill/references/select.md +7 -3
- package/src/skf-export-skill/SKILL.md +2 -2
- package/src/skf-export-skill/assets/snippet-format.md +5 -5
- package/src/skf-export-skill/references/generate-snippet.md +22 -15
- package/src/skf-export-skill/references/load-skill.md +1 -1
- package/src/skf-export-skill/references/token-report.md +29 -9
- package/src/skf-forger/SKILL.md +2 -2
- package/src/skf-quick-skill/SKILL.md +2 -2
- package/src/skf-refine-architecture/SKILL.md +6 -8
- package/src/skf-rename-skill/SKILL.md +2 -2
- package/src/skf-setup/SKILL.md +2 -2
- package/src/skf-test-skill/SKILL.md +2 -2
- package/src/skf-test-skill/assets/output-section-formats.md +6 -4
- package/src/skf-test-skill/references/coherence-check.md +5 -4
- package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
- package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
- package/src/skf-test-skill/references/coverage-check.md +69 -111
- package/src/skf-test-skill/references/init.md +5 -6
- package/src/skf-test-skill/references/migration-section-rules.md +26 -14
- package/src/skf-test-skill/references/report.md +11 -9
- package/src/skf-test-skill/references/scoring-rules.md +8 -6
- package/src/skf-test-skill/references/source-access-protocol.md +15 -18
- package/src/skf-test-skill/scripts/build-result-context.py +114 -21
- package/src/skf-test-skill/scripts/gap-ledger.py +580 -80
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +646 -107
- package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
- package/src/skf-test-skill/scripts/reconcile-coverage.py +450 -23
- package/src/skf-test-skill/scripts/score-signatures.py +356 -61
- package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
- package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
- package/src/skf-test-skill/templates/test-report-template.md +1 -0
- package/src/skf-update-skill/SKILL.md +2 -2
- package/src/skf-update-skill/references/detect-changes.md +32 -13
- package/src/skf-update-skill/references/gap-driven.md +37 -22
- package/src/skf-update-skill/references/init.md +6 -2
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +17 -12
- package/src/skf-update-skill/references/re-extract.md +4 -2
- package/src/skf-update-skill/references/report.md +7 -4
- package/src/skf-update-skill/references/write.md +77 -36
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
- package/src/skf-verify-stack/SKILL.md +2 -2
- package/tools/cli/lib/tool-check.js +158 -2
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
# ///
|
|
5
5
|
"""SKF Extract Public API: one package's public surface, by parser or by recipe.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Four modes:
|
|
8
8
|
|
|
9
9
|
--mode quick (the default) A pure parser: takes one logical package (one
|
|
10
10
|
manifest, one or more entry-point files) and emits its
|
|
@@ -16,9 +16,10 @@ Three modes:
|
|
|
16
16
|
raw fetches into a folder, or a local checkout), so a file's
|
|
17
17
|
text never passes through the model or a shell string.
|
|
18
18
|
--follow also reads the modules an `export *` chain passes
|
|
19
|
-
names on from, fetching them first with --repo
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
names on from, fetching them first with --repo, and
|
|
20
|
+
--resolve-version looks up a version the manifest does
|
|
21
|
+
not state. Multi-module monorepos are caller-orchestrated:
|
|
22
|
+
invoke once per module, aggregate.
|
|
22
23
|
--mode full The ast-grep recipe runner (skf-create-skill and the
|
|
23
24
|
workflows that extract as it does): runs the recipes in
|
|
24
25
|
src/shared/data/ast-grep-recipes.yaml over a source tree
|
|
@@ -30,6 +31,11 @@ Three modes:
|
|
|
30
31
|
rules, so no caller derives an entry file from a
|
|
31
32
|
distribution name or reads a manifest by eye. See "Entries
|
|
32
33
|
mode" below.
|
|
34
|
+
--mode denominator
|
|
35
|
+
Full mode's counts and effective_denominator decision for
|
|
36
|
+
public names counted without its diff (by quick mode or by
|
|
37
|
+
eye), and its scope facts with no recipe run. See
|
|
38
|
+
"Denominator mode" below.
|
|
33
39
|
|
|
34
40
|
Quick mode
|
|
35
41
|
----------
|
|
@@ -120,7 +126,7 @@ or the files, read from disk (stdin is then never read):
|
|
|
120
126
|
--mode quick --language <language> [--source-root <dir>]
|
|
121
127
|
[--manifest-file <path>] [--entry-file <path>]...
|
|
122
128
|
[--follow-file <path>]... [--tree-file <listing>] [--fetch-list <file>]
|
|
123
|
-
[--follow [--repo <owner/repo> [--ref <ref>]]
|
|
129
|
+
[--follow] [--resolve-version] [--repo <owner/repo> [--ref <ref>]]
|
|
124
130
|
|
|
125
131
|
An entry with `followed` true (a --follow-file) is a module an `unlisted[]`
|
|
126
132
|
record named, read as an entry is, less a JS/TS `export default`, which
|
|
@@ -138,6 +144,41 @@ A module that cannot be fetched or read stays in `unlisted[]`, and a
|
|
|
138
144
|
warning names it and why. The output then also lists `followed`, the
|
|
139
145
|
modules read this way, in the order they were read.
|
|
140
146
|
|
|
147
|
+
--resolve-version (with --source-root) looks up a version the manifest does
|
|
148
|
+
not state, in this order, and names where it came from in `version_source`:
|
|
149
|
+
|
|
150
|
+
python the `__version__` literal of an entry `__init__.py`, then of the
|
|
151
|
+
`_version.py` beside it, then the `version=` literal of the
|
|
152
|
+
setup.py beside the manifest (a dynamic version, `attr:` or
|
|
153
|
+
`file:`, or a version setup.py computes)
|
|
154
|
+
rust the `[workspace.package] version` of the workspace root, the
|
|
155
|
+
nearest Cargo.toml at or above the manifest's folder that holds
|
|
156
|
+
a `[workspace]` table (`version.workspace = true`)
|
|
157
|
+
any the release tag: with --repo, skf-github-probe.py's tag listing
|
|
158
|
+
at --ref (the newest tag whose name is a release version, X.Y.Z,
|
|
159
|
+
at the default branch; the pinned tag itself at a tag; none at a
|
|
160
|
+
branch or a commit); without, the tag `git describe --tags
|
|
161
|
+
--abbrev=0` names in --source-root, when its name is a version
|
|
162
|
+
|
|
163
|
+
A file the lookups read is read under --source-root; with --repo and
|
|
164
|
+
--tree-file, one the listing holds is fetched there first, at --ref. A
|
|
165
|
+
version a file gives must start with a digit and be no placeholder, and
|
|
166
|
+
one a tag gives drops a leading `v`. For JavaScript and TypeScript read
|
|
167
|
+
with --repo at the default branch, a manifest pre-release (`-alpha`,
|
|
168
|
+
`-beta`, `-rc`, `-next`, `-canary`) gives way to the newest release, with
|
|
169
|
+
a warning. When nothing gives a version, a warning says so.
|
|
170
|
+
|
|
171
|
+
With --source-root, each Python, JavaScript and TypeScript export also
|
|
172
|
+
carries `defined_in`: the file that defines it, traced as full mode traces
|
|
173
|
+
an entry point (through each import, re-export, `export *` and star import
|
|
174
|
+
to the end of its chain, a cycle cut), so a name an entry file only passes
|
|
175
|
+
on names its own module. It reads only files under --source-root (the
|
|
176
|
+
files full mode lists there, never the --tree-file listing) and needs no
|
|
177
|
+
ast-grep: Python with full mode's module reader, JavaScript and TypeScript
|
|
178
|
+
from each module's text as quick mode reads an entry file. It is null when
|
|
179
|
+
the trace finds no file, as for a name passed on from another package or a
|
|
180
|
+
module --source-root does not hold. `source_file` stays the file read.
|
|
181
|
+
|
|
141
182
|
Each path is relative to --source-root, the folder the files were staged in
|
|
142
183
|
laid out like the repository (or a local checkout), else to the current
|
|
143
184
|
folder, and is the `path` the output names (`source_file`), written with
|
|
@@ -155,9 +196,20 @@ Output JSON shape (stdout):
|
|
|
155
196
|
{
|
|
156
197
|
"language": "python",
|
|
157
198
|
"package_name": "foo",
|
|
158
|
-
"version": "1.2.3",
|
|
199
|
+
"version": "1.2.3", (the manifest's literal
|
|
200
|
+
version, else null: a
|
|
201
|
+
go.mod, a dynamic or
|
|
202
|
+
workspace-inherited one,
|
|
203
|
+
a placeholder; with
|
|
204
|
+
--resolve-version, the
|
|
205
|
+
one it looked up)
|
|
206
|
+
"version_source": "manifest" | "file:<path>" | "tag:<tag>" | null,
|
|
207
|
+
(--resolve-version only)
|
|
159
208
|
"description": "...",
|
|
160
|
-
"exports": [{"name": "Bar", "type": "class", "source_file": "..."
|
|
209
|
+
"exports": [{"name": "Bar", "type": "class", "source_file": "...",
|
|
210
|
+
"defined_in": "..."}, ...], (defined_in: Python,
|
|
211
|
+
JS and TS, with
|
|
212
|
+
--source-root)
|
|
161
213
|
"dependencies": ["requests", "pydantic", ...],
|
|
162
214
|
"modules": ["server", "client"], (Maven/Gradle only)
|
|
163
215
|
"module_folders": ["server", "client"], (each module's folder in
|
|
@@ -193,10 +245,12 @@ Exit codes (quick):
|
|
|
193
245
|
or given twice with the file inputs, a --manifest-file, --entry-file
|
|
194
246
|
or --follow-file that cannot be read or leads outside --source-root,
|
|
195
247
|
a --tree-file that cannot be read or a --fetch-list that cannot be
|
|
196
|
-
written, --follow without --tree-file and --source-root,
|
|
197
|
-
--
|
|
198
|
-
|
|
199
|
-
|
|
248
|
+
written, --follow without --tree-file and --source-root,
|
|
249
|
+
--resolve-version without --source-root, --repo or --ref without
|
|
250
|
+
--follow or --resolve-version (--ref without --repo), or a --repo or
|
|
251
|
+
--ref skf-github-fetch.py does not accept. A module --follow could
|
|
252
|
+
not read, and a version --resolve-version could not find, change no
|
|
253
|
+
exit code.
|
|
200
254
|
|
|
201
255
|
Entries mode
|
|
202
256
|
------------
|
|
@@ -276,7 +330,15 @@ is one the recipes read (python, typescript, tsx, javascript, rust, go and
|
|
|
276
330
|
.vue files) and --language allows it. Globs follow
|
|
277
331
|
skf-resolve-authoritative-files.py's rules: `**` spans any number of path
|
|
278
332
|
segments, none included, and `*` and `?` stay inside one, so `src/**/*.ts`
|
|
279
|
-
matches `src/index.ts` and `**/test_*` matches a top-level `test_x.py`.
|
|
333
|
+
matches `src/index.ts` and `**/test_*` matches a top-level `test_x.py`. An
|
|
334
|
+
include glob that matches no file (a brief written for an older layout,
|
|
335
|
+
say) is named in `warnings` and listed in `scope.unmatched_include`. A
|
|
336
|
+
scope.tier_a_include glob is matched against the whole tree under
|
|
337
|
+
--source-root, regardless of --files-from and before the excludes: one
|
|
338
|
+
that matches no file is named in `warnings` too and listed in
|
|
339
|
+
`scope.unmatched_tier_a_include`, and, when scope.include has a glob, one
|
|
340
|
+
that matches only files no scope.include glob matches is named in
|
|
341
|
+
`warnings`.
|
|
280
342
|
|
|
281
343
|
--brief supplies scope.include, scope.exclude, scope.tier_a_include,
|
|
282
344
|
scope.type and language; --include, --exclude, --tier-a-include,
|
|
@@ -288,7 +350,11 @@ scope.type and language; --include, --exclude, --tier-a-include,
|
|
|
288
350
|
Which recipes. --recipe-set picks the recipes whose metadata.sets lists it:
|
|
289
351
|
`component-library` when the scope type is component-library, else
|
|
290
352
|
`standard`. Each recipe runs in the languages its metadata.languages
|
|
291
|
-
lists. A match whose name starts with `_` is never an
|
|
353
|
+
lists. A Python or Rust match whose name starts with `_` is never an
|
|
354
|
+
export, nor public anywhere below: Python's by the Source API Surface
|
|
355
|
+
Definition's rule, Rust's because a `_` item is hidden or unnamed (a
|
|
356
|
+
`#[doc(hidden)] pub mod __private`, which the runner cannot read, or a
|
|
357
|
+
`pub const _`); a TS/JS `_` name is an export like any other. --head-cap
|
|
292
358
|
keeps at most N matches of each recipe, in file and line order (0: no
|
|
293
359
|
cap); without it the cap is 200, or at Forge+ and Deep tiers 500 for a
|
|
294
360
|
full-library scope and 300 for a component-library one. `truncated` is
|
|
@@ -337,34 +403,125 @@ JS/TS one that is not). Chains are followed to the end, a cycle cut: an
|
|
|
337
403
|
`export *`, a star import or a glob passes on what its module exports,
|
|
338
404
|
that module's own chains included; a re-exported name is traced through
|
|
339
405
|
each re-export, import and glob between to the file that defines it (a
|
|
340
|
-
default export to the name it declares). A
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
406
|
+
default export to the name it declares). A Python package's explicit
|
|
407
|
+
binding of a name (an import, or a def or class in its __init__.py)
|
|
408
|
+
comes before its submodule: `from .x import y` imports the y
|
|
409
|
+
x/__init__.py binds, and the submodule x/y.py only when x/__init__.py
|
|
410
|
+
binds no y or binds the submodule itself (`from . import y`, `from
|
|
411
|
+
pkg.x import y`); a y x/__init__.py binds only through a star import
|
|
412
|
+
(`from .impl import *`) is still read as the submodule x/y.py. A
|
|
413
|
+
namespace (`export * as ns`, a Python submodule, a Rust `pub mod`)
|
|
414
|
+
lists in `members` the names its module passes on, which users reach as
|
|
415
|
+
`pkg.ns.name`, each traced to the file that defines it as a public name
|
|
416
|
+
is, and in place of a namespace among them its members (each module once,
|
|
417
|
+
so a cycle is cut); `members` is null when the trace cannot read the
|
|
418
|
+
module (a JavaScript target no file here holds, a Rust inline or
|
|
419
|
+
`#[path]` module), and a nested one the trace cannot read is a member
|
|
420
|
+
itself. A member re-exported from another package has no file, is no
|
|
421
|
+
gap when a recipe found its name, marks its name public, and, unlike the
|
|
422
|
+
same form in an entry point, adds no `unresolved` item. What a namespace
|
|
423
|
+
makes reachable is not internal: the names its module passes on, the
|
|
424
|
+
modules among them in turn, and for Rust every recipe match in the crate
|
|
425
|
+
root and in each module file a `pub mod` or `pub use` of a reachable
|
|
426
|
+
module names.
|
|
345
427
|
A chain the trace cannot follow (a module no file here holds, a `#[path]`
|
|
346
428
|
module) is listed in `unresolved`. The diff against the recipe inventory,
|
|
347
429
|
one language family at a time (JavaScript/TypeScript, Python, Rust, Go):
|
|
348
|
-
`public` (the entry points' names
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
430
|
+
`public` (the entry points' names, each at the file and line the first
|
|
431
|
+
entry point naming it gives, in path order, a JS/TS entry point's own and
|
|
432
|
+
named exports all coming before any `export *`, unless that file is out
|
|
433
|
+
of scope, or it gives none, and another entry point gives a file in
|
|
434
|
+
scope, other than through a star or glob its own export of the name
|
|
435
|
+
shadows),
|
|
436
|
+
`other_definitions` (each other definition of a public name in a file in
|
|
437
|
+
scope, one row per file and `local` name: the file and line an entry
|
|
438
|
+
point gives, never a namespace nor one beside a public row that is a
|
|
439
|
+
namespace, nor a star or glob through which an entry point passes on a
|
|
440
|
+
name it already exports (its own export shadows it), while two exports
|
|
441
|
+
of the name by one entry point, such as Rust `pub use` items a `cfg`
|
|
442
|
+
gates, are both kept; in every language family, so a name each
|
|
443
|
+
framework package of a monorepo defines keeps
|
|
444
|
+
each package's file, while a gap and an outside_scope row still go by
|
|
445
|
+
name, from `public` alone),
|
|
446
|
+
`internal` (names the recipes found that no entry point exports or makes
|
|
447
|
+
reachable), `extraction_gaps` (names an entry point exports that no
|
|
448
|
+
recipe found, defined in a file in scope or in none the trace reached:
|
|
449
|
+
read them by eye) and `outside_scope` (names an entry point exports that
|
|
450
|
+
a file out of scope defines, even when a recipe found the name in another
|
|
451
|
+
file, which is another definition: widen scope.include to document
|
|
452
|
+
them). A namespace with `members` is diffed as its members: each is found
|
|
453
|
+
when a recipe found its name, or its `local` one, in its file (one with
|
|
454
|
+
no file by its name alone, as a public name is), and has no row of its
|
|
455
|
+
own when a public name (or another namespace's member) has its name and
|
|
456
|
+
file; one with null `members`, and a member that is such a module, is no
|
|
457
|
+
gap. In a family with no entry point, or only empty ones, every name the
|
|
458
|
+
recipes found is public. A JS/TS module exports, besides the names the
|
|
459
|
+
recipes match, those its column-0 `export` statements declare in a form
|
|
460
|
+
no recipe matches, read from its text: `export let`, `export declare
|
|
461
|
+
const`, `export namespace`, and each name a destructuring pattern that
|
|
462
|
+
opens an `export const`, `let` or `var` statement binds (`export const
|
|
463
|
+
{ a, b: c } = obj` exports a and c), at the line the name is on. Such a
|
|
464
|
+
name that an entry point exports, or passes on through `export *`, is
|
|
465
|
+
public and, no recipe having found it, an `extraction_gaps` row
|
|
466
|
+
(`outside_scope`, in a file out of scope) at that line. A pattern after
|
|
467
|
+
the statement's first declarator (`export const a = 1, { b } = o`), or
|
|
468
|
+
one the reader cannot read (one that does not close, or a default
|
|
469
|
+
holding a template literal over several lines), binds nothing here and
|
|
470
|
+
no warning names it: read it by eye.
|
|
355
471
|
|
|
356
472
|
Counts. `exports_public_api` counts the public names (the package's whole
|
|
357
|
-
public API, `outside_scope` names included) and
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
473
|
+
public API, `outside_scope` names included), each distinct name and file
|
|
474
|
+
`public` gives once (`other_definitions` adds none): a namespace with
|
|
475
|
+
`members` counts as its members (none for an empty list, and a member
|
|
476
|
+
that is also a public name, or another namespace's member, once), one
|
|
477
|
+
with null `members` as one. `exports_internal` counts the internal names.
|
|
478
|
+
`effective_denominator` counts those public names with a definition file
|
|
479
|
+
(the `public` row's, or one of its `other_definitions`') that matches
|
|
480
|
+
scope.include (any file when it has no glob), scope.tier_a_include when
|
|
481
|
+
given, and no scope.exclude glob: each name once, however many in-scope
|
|
482
|
+
files define it, so a type counts once whatever its methods, a barrel
|
|
483
|
+
that only re-exports adds nothing of its own, and a name the entry points
|
|
484
|
+
pass on from two files (two packages' entry points that each define it,
|
|
485
|
+
or a namespace member named like another public name, or like another
|
|
486
|
+
namespace's member, at another file) or in two languages counts once,
|
|
487
|
+
when any one of its files matches, as Test Skill's sets count it. A tier
|
|
488
|
+
A list that counts no public name (each glob matches no file, or only
|
|
489
|
+
files scope.exclude drops, that scope.include leaves out or that define
|
|
490
|
+
no public name) narrows nothing on a run that counted a public name and
|
|
491
|
+
read every file (no `errors` item): the count falls back to scope.include
|
|
492
|
+
(else every file), with a warning.
|
|
493
|
+
`arms` gives compile's effective_denominator arms:
|
|
494
|
+
(a) `monorepo` from skf-detect-workspaces.py over the tree the brief
|
|
495
|
+
keeps (its scoped_tree: a member counts only through the files no
|
|
496
|
+
scope.exclude glob matches, so excluding its sources drops it, excluding
|
|
497
|
+
only its manifest does not, root manifests always stay, and a workspace
|
|
498
|
+
with no member left is none), (b) `specific_modules`
|
|
363
499
|
(scope type specific-modules on a repository that is not a monorepo) and
|
|
364
500
|
(c) `multi_subpath_exports` (an in-scope package.json whose `exports` map
|
|
365
501
|
has more than one non-root subpath without a `*`, on a repository that is
|
|
366
502
|
not a monorepo).
|
|
367
503
|
|
|
504
|
+
Store or not. `curated_subset` is true when the files the count's basis
|
|
505
|
+
globs list (scope.include's, else every file; a tier A count's, those
|
|
506
|
+
among them a tier A glob matches too; under arm (b), less those a
|
|
507
|
+
scope.exclude glob matches) leave out a file that defines a counted public
|
|
508
|
+
name (an `outside_scope` name's file and an `other_definitions` file
|
|
509
|
+
included) or every source file of a
|
|
510
|
+
workspace skf-detect-workspaces.py finds over the whole tree
|
|
511
|
+
(scope.exclude aside, unlike arm (a): a member the brief drops is still
|
|
512
|
+
part of the repository it chose a subset of); a basis of `all-files` lists
|
|
513
|
+
none. `effective_denominator_decision` says whether create-skill and
|
|
514
|
+
update-skill store `effective_denominator` as stats.effective_denominator,
|
|
515
|
+
the brief's scope.notes aside (documenting the stratification is the one
|
|
516
|
+
judgment left to them): `store` is true only when none of these holds, and
|
|
517
|
+
`reason` names the first that does, else `curated-subset`: `reference-app`
|
|
518
|
+
(scope type reference-app: pattern_surfaces_documented is its basis),
|
|
519
|
+
`incomplete-run` (an `errors` item: a file a failed call left unread can
|
|
520
|
+
hide a public name, so the count can be 0 or partial), `no-arm` (no arm is
|
|
521
|
+
true), `full-library` (scope type full-library), `all-files` (the basis),
|
|
522
|
+
`not-curated` (`curated_subset` false) and `zero-count` (a stored 0 is no
|
|
523
|
+
denominator: Test Skill reads it as deflation).
|
|
524
|
+
|
|
368
525
|
Output JSON (stdout, or -o):
|
|
369
526
|
|
|
370
527
|
{
|
|
@@ -376,7 +533,9 @@ Output JSON (stdout, or -o):
|
|
|
376
533
|
"ast_grep": {"path": "<exe>" | null, "version": "<x.y.z>" | null},
|
|
377
534
|
"scope": {"include": [...], "exclude": [...],
|
|
378
535
|
"tier_a_include": [...] | null, "type": "<type>" | null,
|
|
379
|
-
"languages": [...], "files_from": "<file>" | null
|
|
536
|
+
"languages": [...], "files_from": "<file>" | null,
|
|
537
|
+
"unmatched_include": [...], # include globs no file matches
|
|
538
|
+
"unmatched_tier_a_include": [...]}, # tier A globs no file matches
|
|
380
539
|
"files_in_scope": N,
|
|
381
540
|
"files_by_language": {"<language>": N, ...},
|
|
382
541
|
"files_without_recipes": {".java": N, ...}, # in scope, no recipe reads them
|
|
@@ -409,35 +568,46 @@ Output JSON (stdout, or -o):
|
|
|
409
568
|
| {"entry", "name", "from", "file"}, ...]},
|
|
410
569
|
"entry_point_diff": {
|
|
411
570
|
"public": [{"name", "language", "entry", "via", "from", "local",
|
|
412
|
-
"file", "line"
|
|
571
|
+
"file", "line",
|
|
572
|
+
"members": [{"name", "local", "file", "line"}, ...]
|
|
573
|
+
| null}, ...], # members: via namespace only
|
|
574
|
+
"other_definitions": [{"name", "language", "entry", "via", "from",
|
|
575
|
+
"local", "file", "line"}, ...], # by name, then file
|
|
413
576
|
"internal": [{"name", "language", "source_file", "source_line"}, ...],
|
|
414
577
|
"extraction_gaps": [{"name", "language", "entry", "file", "line"}, ...],
|
|
415
578
|
"outside_scope": [{"name", "language", "entry", "file", "line"}, ...]},
|
|
416
579
|
"reexport_targets": [{"name", "language", "local", "from", "entry",
|
|
417
580
|
"file", "line", "via"}, ...],
|
|
418
|
-
"counts": {"exports_public_api": N,
|
|
419
|
-
"
|
|
581
|
+
"counts": {"exports_public_api": N, # (family, name, file) keys
|
|
582
|
+
"exports_internal": N,
|
|
583
|
+
"effective_denominator": N, # names, each once
|
|
420
584
|
"effective_denominator_basis": "tier_a_include" | "scope.include"
|
|
421
|
-
| "all-files",
|
|
422
|
-
"denominator_files": N
|
|
585
|
+
| "all-files", # not tier_a_include when it counts no scope.include name
|
|
586
|
+
"denominator_files": N,
|
|
587
|
+
"curated_subset": bool,
|
|
588
|
+
"effective_denominator_decision": {"store": bool, "reason": "<reason>"}},
|
|
423
589
|
"arms": {"monorepo": bool, "monorepo_kind": "<kind>" | null,
|
|
424
590
|
"specific_modules": bool, "multi_subpath_exports": bool},
|
|
425
591
|
"errors": [{"reason": "ast-grep-error" | "ast-grep-timeout" | "time-limit",
|
|
426
592
|
"files": N, "first_file", "detail",
|
|
427
|
-
"unread": [...]}, ...], # unread:
|
|
428
|
-
"warnings": ["..."]
|
|
593
|
+
"unread": [...]}, ...], # unread: the files its call(s) may have left unread
|
|
594
|
+
"warnings": ["..."],
|
|
595
|
+
"files_in_scope_list": ["<file>", ...], # the files files_in_scope counts, in tree order
|
|
596
|
+
"files_without_recipes_list": ["<file>", ...] # the files files_without_recipes counts
|
|
429
597
|
}
|
|
430
598
|
|
|
431
599
|
`via` says how an entry point exports a name: `declaration` (defined in the
|
|
432
600
|
entry point), `re-export` (from another file, which `file` names, `local`
|
|
433
601
|
giving the name it has there when it differs), `star` (through `export *`,
|
|
434
602
|
`from .x import *` or `pub use x::*`) or `namespace` (a module: `export *
|
|
435
|
-
as ns`, a Python submodule, a Rust module). `line`
|
|
436
|
-
defines the name (a module, a name from another
|
|
437
|
-
|
|
438
|
-
`
|
|
439
|
-
|
|
440
|
-
|
|
603
|
+
as ns`, a Python submodule, a Rust module, its names in `members`). `line`
|
|
604
|
+
is null where no line defines the name (a module, a name from another
|
|
605
|
+
package), and a member's `file` is null for a name from another package.
|
|
606
|
+
An `unresolved` item names a subpath whose targets name no file, or a
|
|
607
|
+
chain met tracing `entry` that stops at `file`, the module `from` naming
|
|
608
|
+
no file here.
|
|
609
|
+
Without ast-grep the JSON still has the files in scope, the entry points'
|
|
610
|
+
`files`, the `exports_maps`, the unresolved subpaths and the `arms`.
|
|
441
611
|
`file_issues` lists the files to read by eye: a file ast-grep skips (not
|
|
442
612
|
UTF-8), and a file where the parser met code it could not read (an ERROR
|
|
443
613
|
node), where a recipe can miss an export. `t1_low` is 0: every export here
|
|
@@ -449,14 +619,18 @@ under the 10-minute limit an agent's shell call often has) have passed
|
|
|
449
619
|
since the run started: the run going on is stopped, no other starts, and
|
|
450
620
|
an `errors[]` item with the reason `time-limit` lists in `unread` the
|
|
451
621
|
files left unread (each ast-grep call also stops after 600 seconds,
|
|
452
|
-
`ast-grep-timeout`).
|
|
622
|
+
`ast-grep-timeout`). A call that times out keeps no match; one that exited
|
|
623
|
+
keeps the matches it printed before any line that is not JSON, but a kill
|
|
624
|
+
can cut a file's matches anywhere, so the item of every failed call lists
|
|
625
|
+
all its files in `unread`. The JSON is still written, status "incomplete".
|
|
453
626
|
|
|
454
627
|
Exit codes (full):
|
|
455
628
|
|
|
456
629
|
0 the recipes ran on every file in scope ("ok")
|
|
457
|
-
1 an ast-grep
|
|
458
|
-
|
|
459
|
-
incomplete ("incomplete")
|
|
630
|
+
1 an ast-grep call could not start, exited non-zero, printed other
|
|
631
|
+
than JSON lines or timed out, or the --timeout ran out: the result
|
|
632
|
+
is incomplete ("incomplete"), and each `errors` item lists in
|
|
633
|
+
`unread` the files it may have left unread
|
|
460
634
|
2 input error (no source root, an unreadable brief, file list or
|
|
461
635
|
recipe file, a recipe without its metadata, no PyYAML, an -o file
|
|
462
636
|
that cannot be written, a --timeout not above 0), or a failure the
|
|
@@ -466,6 +640,72 @@ Exit codes (full):
|
|
|
466
640
|
beside it (cmd.exe would read a repository path's `&` as syntax);
|
|
467
641
|
the JSON has the scope and no exports; extract by source reading
|
|
468
642
|
|
|
643
|
+
Denominator mode
|
|
644
|
+
----------------
|
|
645
|
+
|
|
646
|
+
--mode denominator [--source-root <dir> | --tree-file <listing>]
|
|
647
|
+
[--brief <skill-brief.yaml>] [--include GLOB]... [--exclude GLOB]...
|
|
648
|
+
[--tier-a-include GLOB]... [--scope-type TYPE] [--language LANG]...
|
|
649
|
+
[--quick <quick.json>]... [--extraction <full.json>] [-o <out.json>]
|
|
650
|
+
< {payload_json}
|
|
651
|
+
|
|
652
|
+
The counts full mode writes, for public names no runner diff gave: those
|
|
653
|
+
a stdin payload lists,
|
|
654
|
+
|
|
655
|
+
{"public": [{"name", "file", "language",
|
|
656
|
+
"members": [{"name", "file"}, ...]}, ...],
|
|
657
|
+
"exports": [{"name", "file", "language"}, ...]}
|
|
658
|
+
|
|
659
|
+
(`file` relative to --source-root, with `/`; `members` for a submodule
|
|
660
|
+
the entry points export by name, which counts as the names its module
|
|
661
|
+
passes on), each export of a --quick output at its `defined_in` (else its
|
|
662
|
+
`source_file`), and the --extraction runner's `entry_point_diff.public`;
|
|
663
|
+
a payload name replaces a --quick or runner one of its family and name.
|
|
664
|
+
The runner's `other_definitions` count in `effective_denominator` and
|
|
665
|
+
`curated_subset` as full mode counts them, never in `exports_public_api`,
|
|
666
|
+
each only while the record kept for its name names the file of the
|
|
667
|
+
runner's `public` row (a row whose name has no runner row, or one a
|
|
668
|
+
payload or --quick name moves to another file, adds nothing), and none
|
|
669
|
+
beside a namespace with members, as in full mode.
|
|
670
|
+
`exports_internal` counts the names of the payload's `exports` (the
|
|
671
|
+
exports extraction found; Python, Rust and Dart `_` names left out, those
|
|
672
|
+
of every other language counted) and of the runner's `internal` that no
|
|
673
|
+
public name or member has. `effective_denominator`,
|
|
674
|
+
its basis, `curated_subset` and `effective_denominator_decision` follow
|
|
675
|
+
full mode's rules, over the tree under --source-root, with the scope and
|
|
676
|
+
arms full mode works out there; an --extraction that is `incomplete` is
|
|
677
|
+
never stored. A name whose `file` is not in that tree counts in
|
|
678
|
+
`exports_public_api` alone, with a warning. With no --source-root (a
|
|
679
|
+
remote source read by eye) the tree is the --tree-file listing (a file,
|
|
680
|
+
or `-` for stdin, which then holds no payload: JSON with a `tree` list, a
|
|
681
|
+
list, or one path per line; one that lists no file is an input error),
|
|
682
|
+
else the names' files; the arms are all false, there is no entry point,
|
|
683
|
+
and the decision is never to store (`no-source-tree`). An empty payload
|
|
684
|
+
(`{}`, or no stdin) gives the scope facts alone, with no recipe run:
|
|
685
|
+
`scope`, the two file lists (by full mode's Which files rule, null with
|
|
686
|
+
neither a tree nor a listing), `entry_points` (`files`, `exports_maps`)
|
|
687
|
+
and `arms`.
|
|
688
|
+
|
|
689
|
+
`entry_point_diff` replaces a compare by eye, each item {"name",
|
|
690
|
+
"language", "file"}: `public`, the names `exports_public_api` counts (a
|
|
691
|
+
namespace as its `members`); `other_definitions`, the runner's other
|
|
692
|
+
definitions the counts read, by name then file; `internal`, the exports
|
|
693
|
+
`exports_internal` counts; `extraction_gaps`, the public names the
|
|
694
|
+
payload and --quick give that no export found has (the payload's
|
|
695
|
+
`exports`, the --extraction runner's names), to trace (the runner lists
|
|
696
|
+
its own gaps).
|
|
697
|
+
|
|
698
|
+
Output JSON: `mode` "denominator", `source_root`, `scope` (as full mode
|
|
699
|
+
writes it), `files_in_scope`, `files_without_recipes`, `entry_points`,
|
|
700
|
+
`arms`, `runner_status` (the --extraction's `status`, else null),
|
|
701
|
+
`counts` (as full mode writes it), `entry_point_diff`, `warnings`, and
|
|
702
|
+
last `files_in_scope_list` and `files_without_recipes_list` (as full
|
|
703
|
+
mode writes them), to -o when given (its folder created). Exit 0, or 2
|
|
704
|
+
on an input error (a --source-root that is no folder, a brief, --quick,
|
|
705
|
+
--extraction or --tree-file that cannot be read, --tree-file beside
|
|
706
|
+
--source-root, a payload that is not this shape, a flag of another
|
|
707
|
+
mode).
|
|
708
|
+
|
|
469
709
|
CLI examples:
|
|
470
710
|
uv run skf-extract-public-api.py --mode quick < {payload_json}
|
|
471
711
|
uv run skf-extract-public-api.py --mode entries --language python \\
|
|
@@ -722,7 +962,7 @@ def parse_package_swift(content: str) -> dict:
|
|
|
722
962
|
"""Best-effort parse of a SwiftPM Package.swift manifest.
|
|
723
963
|
|
|
724
964
|
SwiftPM has no version field in the manifest (versions come from git tags),
|
|
725
|
-
so `version` is always None;
|
|
965
|
+
so `version` is always None; a caller reads it from a git tag.
|
|
726
966
|
"""
|
|
727
967
|
name_m = re.search(r"\bPackage\s*\(\s*name:\s*['\"]([^'\"]+)['\"]", content, re.DOTALL)
|
|
728
968
|
deps: list[str] = []
|
|
@@ -1332,6 +1572,35 @@ def _detect_placeholder_version(version: str | None) -> str | None:
|
|
|
1332
1572
|
return None
|
|
1333
1573
|
|
|
1334
1574
|
|
|
1575
|
+
def _literal_version(version: object, warnings: list[str]) -> str | None:
|
|
1576
|
+
"""The version the manifest writes as a literal string, else None.
|
|
1577
|
+
|
|
1578
|
+
A placeholder and a version that is not a string (Cargo's
|
|
1579
|
+
`version = { workspace = true }`, which the workspace root's Cargo.toml
|
|
1580
|
+
holds) give None with a warning, so the output's `version` is a string
|
|
1581
|
+
or null, which --resolve-version looks up."""
|
|
1582
|
+
if version is None:
|
|
1583
|
+
return None
|
|
1584
|
+
if not isinstance(version, str):
|
|
1585
|
+
if isinstance(version, dict) and version.get("workspace") is True:
|
|
1586
|
+
warnings.append(
|
|
1587
|
+
"manifest version is inherited from the workspace (version.workspace = true), "
|
|
1588
|
+
"which the workspace root's Cargo.toml holds, so the manifest states none"
|
|
1589
|
+
)
|
|
1590
|
+
else:
|
|
1591
|
+
warnings.append(f"manifest version {version!r} is not a string, so the manifest states none")
|
|
1592
|
+
return None
|
|
1593
|
+
placeholder = _detect_placeholder_version(version)
|
|
1594
|
+
if placeholder is not None:
|
|
1595
|
+
warnings.append(
|
|
1596
|
+
f"manifest version {placeholder!r} is a release-time placeholder "
|
|
1597
|
+
f"(workspace protocol or semantic-release sentinel) and is not a real version, "
|
|
1598
|
+
f"so the manifest states none"
|
|
1599
|
+
)
|
|
1600
|
+
return None
|
|
1601
|
+
return version
|
|
1602
|
+
|
|
1603
|
+
|
|
1335
1604
|
def extract(payload: dict) -> dict:
|
|
1336
1605
|
"""Orchestrate manifest parse + export scan for one logical package."""
|
|
1337
1606
|
warnings: list[str] = []
|
|
@@ -1353,14 +1622,7 @@ def extract(payload: dict) -> dict:
|
|
|
1353
1622
|
warnings.append(parsed["_parse_error"])
|
|
1354
1623
|
parsed = {}
|
|
1355
1624
|
|
|
1356
|
-
|
|
1357
|
-
if placeholder is not None:
|
|
1358
|
-
warnings.append(
|
|
1359
|
-
f"manifest version {placeholder!r} is a release-time placeholder "
|
|
1360
|
-
f"(workspace protocol or semantic-release sentinel) and is not a real version; "
|
|
1361
|
-
f"the brief will fall back to user-supplied target_version or the default"
|
|
1362
|
-
)
|
|
1363
|
-
parsed["version"] = None
|
|
1625
|
+
parsed["version"] = _literal_version(parsed.get("version"), warnings)
|
|
1364
1626
|
|
|
1365
1627
|
_, scanner = LANGUAGE_DISPATCH[language]
|
|
1366
1628
|
exports: list[dict] = []
|
|
@@ -1536,6 +1798,8 @@ JS_SOURCE_OF = {
|
|
|
1536
1798
|
JS_TARGET_SUFFIXES = (".d.mts", ".d.cts", ".d.ts", ".mjs", ".cjs", ".jsx", ".js", ".mts", ".cts", ".tsx", ".ts")
|
|
1537
1799
|
PACKAGE_ENTRY_FIELDS = ("types", "typings", "module", "main", "browser")
|
|
1538
1800
|
WORKSPACE_MANIFESTS = ("package.json", "pnpm-workspace.yaml", "lerna.json", "Cargo.toml", "rush.json", "nx.json")
|
|
1801
|
+
# The arms under which a curated count can be stored as stats.effective_denominator.
|
|
1802
|
+
DENOMINATOR_ARMS = ("monorepo", "specific_modules", "multi_subpath_exports")
|
|
1539
1803
|
|
|
1540
1804
|
|
|
1541
1805
|
class RunnerError(Exception):
|
|
@@ -1560,8 +1824,9 @@ def _sibling(filename: str):
|
|
|
1560
1824
|
"""A helper installed beside this script, loaded once as a module:
|
|
1561
1825
|
skf-source-tree.py (git without the caller's hooks and git variables,
|
|
1562
1826
|
and a tool lookup that never takes a shim planted in the current
|
|
1563
|
-
folder), skf-resolve-authoritative-files.py (the brief's
|
|
1564
|
-
skf-detect-workspaces.py (monorepo detection). Raises
|
|
1827
|
+
folder), skf-resolve-authoritative-files.py (the brief's YAML and glob
|
|
1828
|
+
rules) and skf-detect-workspaces.py (monorepo detection). Raises
|
|
1829
|
+
RunnerError."""
|
|
1565
1830
|
module = _SIBLINGS.get(filename)
|
|
1566
1831
|
if module is None:
|
|
1567
1832
|
path = Path(__file__).resolve().parent / filename
|
|
@@ -1856,8 +2121,9 @@ def _read_file_list(path: Path) -> list[str]:
|
|
|
1856
2121
|
def _read_brief(path: Path) -> dict:
|
|
1857
2122
|
"""The scope fields and language of a skill brief."""
|
|
1858
2123
|
yaml = _yaml()
|
|
2124
|
+
parse_brief_yaml = _sibling("skf-resolve-authoritative-files.py").parse_brief_yaml
|
|
1859
2125
|
try:
|
|
1860
|
-
data =
|
|
2126
|
+
data = parse_brief_yaml(path.read_text(encoding="utf-8"))
|
|
1861
2127
|
except (OSError, UnicodeDecodeError) as exc:
|
|
1862
2128
|
raise RunnerError(f"cannot read brief {path}: {exc}") from exc
|
|
1863
2129
|
except yaml.YAMLError as exc:
|
|
@@ -2003,12 +2269,17 @@ def _scan(exe: str, root: Path, folder: Path, rules: list[dict], files: list[str
|
|
|
2003
2269
|
errors: list[dict], clock: _Clock | None = None) -> list[dict]:
|
|
2004
2270
|
"""ast-grep's matches of `rules` in `files` (relative to `root`), in
|
|
2005
2271
|
batches. The rules go in a file as JSON documents (YAML reads JSON),
|
|
2006
|
-
never on the command line. A call that times out, exits
|
|
2007
|
-
rules carry no severity, so a scan that read every file
|
|
2008
|
-
or not) or prints other than JSON lines adds to
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2272
|
+
never on the command line. A call that cannot start, times out, exits
|
|
2273
|
+
non-zero (its rules carry no severity, so a scan that read every file
|
|
2274
|
+
exits 0, matches or not) or prints other than JSON lines adds to
|
|
2275
|
+
`errors` an item whose `unread` lists every file of its batch: ast-grep
|
|
2276
|
+
prints a file's matches as one block per rule, with other files' blocks
|
|
2277
|
+
between them, and a kill can stop it anywhere, so the output cannot show
|
|
2278
|
+
which files it printed in full. A call that exited keeps the matches it
|
|
2279
|
+
printed before any line that is not JSON; a call that timed out keeps
|
|
2280
|
+
none. When `clock` (the run's --timeout) runs out, a call running is
|
|
2281
|
+
stopped (its `time-limit` item lists its batch) and no other starts: a
|
|
2282
|
+
`time-limit` item lists the files no call read, if any."""
|
|
2012
2283
|
if not rules or not files:
|
|
2013
2284
|
return []
|
|
2014
2285
|
rule_file = folder / "rules.yml"
|
|
@@ -2055,10 +2326,8 @@ def _scan(exe: str, root: Path, folder: Path, rules: list[dict], files: list[str
|
|
|
2055
2326
|
if match is not None:
|
|
2056
2327
|
found.append(match)
|
|
2057
2328
|
if failure is not None:
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
error["unread"] = list(batch)
|
|
2061
|
-
errors.append(error)
|
|
2329
|
+
errors.append({"reason": failure[0], "files": len(batch), "first_file": batch[0], "detail": failure[1],
|
|
2330
|
+
"unread": list(batch)})
|
|
2062
2331
|
return found
|
|
2063
2332
|
|
|
2064
2333
|
|
|
@@ -2410,11 +2679,22 @@ def _preferred(at_line: list[dict], forms: dict, order: dict[str, int]) -> dict:
|
|
|
2410
2679
|
return min(left, key=lambda m: (order[m["rule"]], m["language"]))
|
|
2411
2680
|
|
|
2412
2681
|
|
|
2682
|
+
def _private(family: str | None, name: str) -> bool:
|
|
2683
|
+
"""A name full mode and denominator mode never count: one that starts
|
|
2684
|
+
with `_` in Python (the Source API Surface Definition leaves it out),
|
|
2685
|
+
Rust (a `#[doc(hidden)] pub mod __private`, which the runner cannot
|
|
2686
|
+
read, or a `pub const _`) or Dart (private to its library). A TS/JS `_`
|
|
2687
|
+
name is exported like any other, and Go's recipe reads only a
|
|
2688
|
+
capitalized name. Quick mode drops only Python's `_` names."""
|
|
2689
|
+
return family in ("python", "rust", "dart") and name.startswith("_")
|
|
2690
|
+
|
|
2691
|
+
|
|
2413
2692
|
def merge_exports(matches: list[dict], recipes: list[dict], recipe_set: str,
|
|
2414
2693
|
head_cap: int | None) -> tuple[list[dict], list[dict]]:
|
|
2415
2694
|
"""(exports, per-recipe stats) from the matches of one recipe set: at
|
|
2416
2695
|
most `head_cap` matches of each recipe, in file and line order, merged
|
|
2417
|
-
by (file, name) at the first line that names it.
|
|
2696
|
+
by (file, name) at the first line that names it. A Python or Rust `_`
|
|
2697
|
+
name is no export."""
|
|
2418
2698
|
forms = _forms(recipes)
|
|
2419
2699
|
order: dict[str, int] = {}
|
|
2420
2700
|
languages: dict[str, list[str]] = {}
|
|
@@ -2425,7 +2705,8 @@ def merge_exports(matches: list[dict], recipes: list[dict], recipe_set: str,
|
|
|
2425
2705
|
langs.extend(lang for lang in recipe["metadata"]["languages"] if lang not in langs)
|
|
2426
2706
|
per_recipe: dict[str, list[dict]] = {rid: [] for rid in order}
|
|
2427
2707
|
for m in matches:
|
|
2428
|
-
if m["rule"] in per_recipe and m["name"] and not
|
|
2708
|
+
if (m["rule"] in per_recipe and m["name"] and not _private(FAMILY_OF.get(m["language"]), m["name"])
|
|
2709
|
+
and (m["rule"], m["language"]) in forms):
|
|
2429
2710
|
per_recipe[m["rule"]].append(m)
|
|
2430
2711
|
stats, kept = [], []
|
|
2431
2712
|
for rid, found in per_recipe.items():
|
|
@@ -2622,14 +2903,190 @@ _JS_DEFAULT_NAME_RE = re.compile(r"^export\s+default\s+([A-Za-z_$][\w$]*)\s*;?\s
|
|
|
2622
2903
|
_JS_DEFAULT_EXPORT_RE = re.compile(r"export\s+default\b")
|
|
2623
2904
|
|
|
2624
2905
|
|
|
2906
|
+
# Keep identical to _js_pattern_bindings in skf-verify-provenance-completeness.py.
|
|
2907
|
+
def _js_pattern_bindings(
|
|
2908
|
+
text: str, start: int
|
|
2909
|
+
) -> tuple[list[tuple[str, int]], int | None]:
|
|
2910
|
+
"""The names the destructuring pattern whose `{` or `[` is at
|
|
2911
|
+
`text[start]` binds, each with its offset in `text`, and the offset
|
|
2912
|
+
just past the pattern's closing bracket.
|
|
2913
|
+
|
|
2914
|
+
An object pattern's property is `...target`, a shorthand key or
|
|
2915
|
+
`key: target` (a key is a name, a string, a number or a computed
|
|
2916
|
+
`[...]`, and only a shorthand key binds itself); an array pattern's
|
|
2917
|
+
element is `...target`, a target or a hole; a target is a name or a
|
|
2918
|
+
nested pattern. A `= default` is skipped to the next comma at the
|
|
2919
|
+
pattern's depth, and so is a property or element of any other shape,
|
|
2920
|
+
which binds nothing. '...', "..." and `...` strings are skipped
|
|
2921
|
+
whole, so no bracket or comma inside one counts. Whitespace, newlines
|
|
2922
|
+
included, may sit anywhere between the parts.
|
|
2923
|
+
|
|
2924
|
+
Returns ([], None) when `text[start]` opens no pattern, the pattern
|
|
2925
|
+
does not close in `text`, a string in it does not end, or a bracket in
|
|
2926
|
+
it closes one it did not open. A pure function of its arguments (no
|
|
2927
|
+
module state), so another script can carry a copy of it.
|
|
2928
|
+
"""
|
|
2929
|
+
n = len(text)
|
|
2930
|
+
names: list[tuple[str, int]] = []
|
|
2931
|
+
|
|
2932
|
+
def space(i: int) -> int:
|
|
2933
|
+
while i < n and text[i].isspace():
|
|
2934
|
+
i += 1
|
|
2935
|
+
return i
|
|
2936
|
+
|
|
2937
|
+
def name_end(i: int) -> int:
|
|
2938
|
+
"""The end of the identifier at `i`, `i` itself when none starts there."""
|
|
2939
|
+
if i < n and (text[i].isalpha() or text[i] in "_$"):
|
|
2940
|
+
i += 1
|
|
2941
|
+
while i < n and (text[i].isalnum() or text[i] in "_$"):
|
|
2942
|
+
i += 1
|
|
2943
|
+
return i
|
|
2944
|
+
|
|
2945
|
+
def string_end(i: int) -> int | None:
|
|
2946
|
+
"""The offset past the string whose quote is at `i`."""
|
|
2947
|
+
quote = text[i]
|
|
2948
|
+
i += 1
|
|
2949
|
+
while i < n:
|
|
2950
|
+
if text[i] == "\\":
|
|
2951
|
+
i += 2
|
|
2952
|
+
elif text[i] == quote:
|
|
2953
|
+
return i + 1
|
|
2954
|
+
else:
|
|
2955
|
+
i += 1
|
|
2956
|
+
return None
|
|
2957
|
+
|
|
2958
|
+
def skip(i: int, to_comma: bool = True) -> int | None:
|
|
2959
|
+
"""From `i`, the offset of the next comma (unless not `to_comma`)
|
|
2960
|
+
or closing bracket at depth 0; with `to_comma` False, `text[i]`
|
|
2961
|
+
opens a bracket and the offset past the one closing it."""
|
|
2962
|
+
depth = 0
|
|
2963
|
+
while i < n:
|
|
2964
|
+
ch = text[i]
|
|
2965
|
+
if ch in "'\"`":
|
|
2966
|
+
end = string_end(i)
|
|
2967
|
+
if end is None:
|
|
2968
|
+
return None
|
|
2969
|
+
i = end
|
|
2970
|
+
continue
|
|
2971
|
+
if ch in "([{":
|
|
2972
|
+
depth += 1
|
|
2973
|
+
elif ch in ")]}":
|
|
2974
|
+
if depth == 0:
|
|
2975
|
+
return i
|
|
2976
|
+
depth -= 1
|
|
2977
|
+
if depth == 0 and not to_comma:
|
|
2978
|
+
return i + 1
|
|
2979
|
+
elif ch == "," and depth == 0 and to_comma:
|
|
2980
|
+
return i
|
|
2981
|
+
i += 1
|
|
2982
|
+
return None
|
|
2983
|
+
|
|
2984
|
+
def default(i: int) -> int | None:
|
|
2985
|
+
"""After a target: past its `= default`, when it has one."""
|
|
2986
|
+
i = space(i)
|
|
2987
|
+
if i < n and text[i] == "=" and text[i + 1 : i + 2] not in ("=", ">"):
|
|
2988
|
+
return skip(i + 1)
|
|
2989
|
+
return i
|
|
2990
|
+
|
|
2991
|
+
def bind(i: int, end: int, close: str) -> int | None:
|
|
2992
|
+
"""The name `text[i:end]` is bound when its property or element
|
|
2993
|
+
ends after it or takes a default (any other text: another shape)."""
|
|
2994
|
+
after = space(end)
|
|
2995
|
+
if after < n and (
|
|
2996
|
+
text[after] in (",", close)
|
|
2997
|
+
or text[after] == "=" and text[after + 1 : after + 2] not in ("=", ">")
|
|
2998
|
+
):
|
|
2999
|
+
names.append((text[i:end], i))
|
|
3000
|
+
return default(after)
|
|
3001
|
+
return after
|
|
3002
|
+
|
|
3003
|
+
def target(i: int, close: str) -> int | None:
|
|
3004
|
+
if i < n and text[i] in "{[":
|
|
3005
|
+
end = pattern(i)
|
|
3006
|
+
return None if end is None else default(end)
|
|
3007
|
+
end = name_end(i)
|
|
3008
|
+
return i if end == i else bind(i, end, close)
|
|
3009
|
+
|
|
3010
|
+
def element(i: int, close: str) -> int | None:
|
|
3011
|
+
"""One property or element at `i`: the offset after it."""
|
|
3012
|
+
if text.startswith("...", i):
|
|
3013
|
+
return target(space(i + 3), close)
|
|
3014
|
+
if close == "]":
|
|
3015
|
+
return target(i, close)
|
|
3016
|
+
if text[i] in "'\"":
|
|
3017
|
+
end = string_end(i)
|
|
3018
|
+
elif text[i] == "[":
|
|
3019
|
+
end = skip(i, to_comma=False) # a computed key
|
|
3020
|
+
else:
|
|
3021
|
+
end = name_end(i)
|
|
3022
|
+
if end > i:
|
|
3023
|
+
after = space(end)
|
|
3024
|
+
if after < n and text[after] == ":":
|
|
3025
|
+
return target(space(after + 1), close)
|
|
3026
|
+
return bind(i, end, close) # a shorthand key
|
|
3027
|
+
while end < n and (text[end].isalnum() or text[end] in "._"):
|
|
3028
|
+
end += 1 # a number key
|
|
3029
|
+
if end is None:
|
|
3030
|
+
return None
|
|
3031
|
+
after = space(end)
|
|
3032
|
+
if end > i and after < n and text[after] == ":":
|
|
3033
|
+
return target(space(after + 1), close)
|
|
3034
|
+
return after
|
|
3035
|
+
|
|
3036
|
+
def pattern(i: int) -> int | None:
|
|
3037
|
+
"""The pattern opening at `i`: the offset past its closing bracket."""
|
|
3038
|
+
close = "}" if text[i] == "{" else "]"
|
|
3039
|
+
i += 1
|
|
3040
|
+
while True:
|
|
3041
|
+
i = space(i)
|
|
3042
|
+
if i >= n:
|
|
3043
|
+
return None
|
|
3044
|
+
if text[i] == close:
|
|
3045
|
+
return i + 1
|
|
3046
|
+
if text[i] in ")]}":
|
|
3047
|
+
return None
|
|
3048
|
+
if text[i] == ",":
|
|
3049
|
+
i += 1 # a hole, or a stray comma
|
|
3050
|
+
continue
|
|
3051
|
+
end = element(i, close)
|
|
3052
|
+
if end is None:
|
|
3053
|
+
return None
|
|
3054
|
+
end = space(end)
|
|
3055
|
+
if end < n and text[end] not in (",", close):
|
|
3056
|
+
end = skip(end) # any other shape binds nothing
|
|
3057
|
+
if end is None:
|
|
3058
|
+
return None
|
|
3059
|
+
i = end + 1 if end < n and text[end] == "," else end
|
|
3060
|
+
|
|
3061
|
+
if not (0 <= start < n and text[start] in "{["):
|
|
3062
|
+
return [], None
|
|
3063
|
+
end = pattern(start)
|
|
3064
|
+
return (names, end) if end is not None else ([], None)
|
|
3065
|
+
|
|
3066
|
+
|
|
2625
3067
|
def _js_text_names(text: str) -> tuple[list[tuple[str, int, bool]], list[tuple[str, int]]]:
|
|
2626
3068
|
"""The names a JS/TS file's column-0 `export` declarations bind, read
|
|
2627
3069
|
from its text for the forms no recipe matches (`export let`, `export
|
|
2628
|
-
declare const`, `export namespace
|
|
2629
|
-
|
|
3070
|
+
declare const`, `export namespace`, and each name a destructuring
|
|
3071
|
+
pattern that opens an `export const`, `let` or `var` statement binds:
|
|
3072
|
+
`export const { a, b: c, ...d } = obj` binds a, c and d), as (name,
|
|
3073
|
+
line, is the default export), and its `export default <name>;`
|
|
3074
|
+
statements, as (name, line).
|
|
3075
|
+
|
|
3076
|
+
A pattern is read from its `{` or `[` on (`_JS_DESTRUCTURE_RE`) with
|
|
3077
|
+
`_js_pattern_bindings`, so it may span lines, and each name it binds
|
|
3078
|
+
takes the line the name is on. A pattern after the first declarator
|
|
3079
|
+
is not read, and one that does not close, or that nests deeper than
|
|
3080
|
+
the interpreter's recursion limit, binds nothing."""
|
|
2630
3081
|
code = _blank(text, "js")
|
|
2631
3082
|
declared = [(m.group(2), _line_of(code, m.start(2)), bool(m.group(1)))
|
|
2632
3083
|
for rx in (_JS_DECLARATION_RE, _JS_IMPORT_EQUALS_RE) for m in rx.finditer(code)]
|
|
3084
|
+
for m in _JS_DESTRUCTURE_RE.finditer(code):
|
|
3085
|
+
try:
|
|
3086
|
+
names, _ = _js_pattern_bindings(code, m.end() - 1)
|
|
3087
|
+
except RecursionError:
|
|
3088
|
+
continue # patterns nested past the interpreter's limit: none bound
|
|
3089
|
+
declared += [(name, _line_of(code, offset), False) for name, offset in names]
|
|
2633
3090
|
defaults = [(m.group(1), _line_of(code, m.start(1))) for m in _JS_DEFAULT_NAME_RE.finditer(code)]
|
|
2634
3091
|
return declared, defaults
|
|
2635
3092
|
|
|
@@ -2861,26 +3318,96 @@ def _specifier_local(text: str) -> str:
|
|
|
2861
3318
|
return re.split(r"\s+as\s+", item, maxsplit=1)[0].strip().strip("'\"")
|
|
2862
3319
|
|
|
2863
3320
|
|
|
2864
|
-
|
|
2865
|
-
"""
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
3321
|
+
def _member(name: str, local: str | None, file: str | None, line: int | None) -> dict:
|
|
3322
|
+
"""One name a namespace's module passes on, traced to the file that
|
|
3323
|
+
defines it (`local` the name it has there when it differs)."""
|
|
3324
|
+
return {"name": name, "local": local if local and local != name else None, "file": file, "line": line}
|
|
3325
|
+
|
|
3326
|
+
|
|
3327
|
+
def _flatten(start, direct: Callable[[object], list[tuple[str, str, object]]], family: str) -> list[dict]:
|
|
3328
|
+
"""The members of the namespace module `start`, of language family
|
|
3329
|
+
`family`: what `direct` says it passes on, (name, "member", member) or
|
|
3330
|
+
(name, "module", the module), each nested namespace module's members
|
|
3331
|
+
spliced in its place (each module once, so a cycle is cut), each
|
|
3332
|
+
(name, file) once, and no Python or Rust name that starts with `_`,
|
|
3333
|
+
which is never public."""
|
|
3334
|
+
out: list[dict] = []
|
|
3335
|
+
seen: set[tuple[str, str | None]] = set()
|
|
3336
|
+
visited: set = set()
|
|
3337
|
+
|
|
3338
|
+
def walk(module) -> None:
|
|
3339
|
+
visited.add(module)
|
|
3340
|
+
for name, kind, value in direct(module):
|
|
3341
|
+
if not name or _private(family, name):
|
|
3342
|
+
continue
|
|
3343
|
+
if kind == "module":
|
|
3344
|
+
if value not in visited:
|
|
3345
|
+
walk(value)
|
|
3346
|
+
elif (value["name"], value["file"]) not in seen:
|
|
3347
|
+
seen.add((value["name"], value["file"]))
|
|
3348
|
+
out.append(value)
|
|
3349
|
+
|
|
3350
|
+
walk(start)
|
|
3351
|
+
return out
|
|
3352
|
+
|
|
2869
3353
|
|
|
2870
|
-
|
|
3354
|
+
class _Surface:
|
|
3355
|
+
"""The public names the entry points of language family `family`
|
|
3356
|
+
declare: {name: info}, the first `add` of a name wins (the JS/TS trace
|
|
3357
|
+
adds every entry point's own and named exports before any `export *`),
|
|
3358
|
+
unless the file it gives is out of scope (or it gives none) and another
|
|
3359
|
+
entry point gives a file in `in_scope`, which then replaces it (one
|
|
3360
|
+
entry point never replaces its own, and a star or glob that passes on a
|
|
3361
|
+
name its entry point already exports replaces none: an explicit export
|
|
3362
|
+
shadows it; with no `in_scope`, the first always wins); `others`,
|
|
3363
|
+
{name: [info]}, the other definitions of a held name in files in
|
|
3364
|
+
`in_scope` (#830), each an add that does not become the held record,
|
|
3365
|
+
is no such shadowed star or glob, is no namespace, beside a held record
|
|
3366
|
+
that is none, and gives a file in scope whose (file, `local` or name)
|
|
3367
|
+
neither the held record nor an earlier one of `others` gives; the
|
|
3368
|
+
names, and the files, a namespace export (a module) makes reachable,
|
|
3369
|
+
which are not internal; and the re-exports the trace could not follow.
|
|
3370
|
+
A namespace item also carries its `members`, the names its module
|
|
3371
|
+
passes on (None when the trace cannot read that module); `opaque` holds
|
|
3372
|
+
the (name, file) of each member that is itself a module the trace
|
|
3373
|
+
cannot read, which, as such a namespace, is no recipe's target. A
|
|
3374
|
+
Python or Rust `_` name is never added."""
|
|
3375
|
+
|
|
3376
|
+
def __init__(self, family: str, in_scope: set[str] | None = None) -> None:
|
|
3377
|
+
self.family = family
|
|
3378
|
+
self.in_scope = in_scope
|
|
2871
3379
|
self.names: dict[str, dict] = {}
|
|
3380
|
+
self.others: dict[str, list[dict]] = {}
|
|
3381
|
+
self._other_at: dict[str, set[tuple[str, str]]] = {} # name: the (file, local or name) of its others
|
|
3382
|
+
# (entry, name) of each add: a star or glob an entry point passes an exported name on through is shadowed
|
|
3383
|
+
self._added: set[tuple[str, str]] = set()
|
|
2872
3384
|
self.reachable: set[str] = set()
|
|
2873
3385
|
self.reachable_files: set[str] = set()
|
|
2874
3386
|
self.unresolved: list[dict] = []
|
|
3387
|
+
self.opaque: set[tuple[str, str | None]] = set()
|
|
2875
3388
|
|
|
2876
3389
|
def add(self, name: str, entry: str, via: str, file: str | None, line: int | None,
|
|
2877
|
-
source: str | None = None, local: str | None = None
|
|
2878
|
-
|
|
3390
|
+
source: str | None = None, local: str | None = None,
|
|
3391
|
+
members: list[dict] | None = None) -> dict | None:
|
|
3392
|
+
if not name or _private(self.family, name):
|
|
2879
3393
|
return None
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
|
|
2883
|
-
|
|
3394
|
+
held = self.names.get(name)
|
|
3395
|
+
shadowed = via == "star" and (entry, name) in self._added
|
|
3396
|
+
self._added.add((entry, name))
|
|
3397
|
+
info = {"name": name, "entry": entry, "via": via, "from": source,
|
|
3398
|
+
"local": local if local and local != name else None, "file": file, "line": line}
|
|
3399
|
+
at = (file, info["local"] or name)
|
|
3400
|
+
if held is None or (not shadowed and self.in_scope is not None and file in self.in_scope
|
|
3401
|
+
and entry != held["entry"]
|
|
3402
|
+
and (held["file"] is None or held["file"] not in self.in_scope)):
|
|
3403
|
+
self.names[name] = info
|
|
3404
|
+
if via == "namespace":
|
|
3405
|
+
self.names[name]["members"] = [dict(m) for m in members] if members is not None else None
|
|
3406
|
+
elif (not shadowed and via != "namespace" and held["via"] != "namespace" and self.in_scope is not None
|
|
3407
|
+
and file in self.in_scope and at != (held["file"], held["local"] or name)
|
|
3408
|
+
and at not in self._other_at.get(name, ())):
|
|
3409
|
+
self.others.setdefault(name, []).append(info)
|
|
3410
|
+
self._other_at.setdefault(name, set()).add(at)
|
|
2884
3411
|
return self.names[name]
|
|
2885
3412
|
|
|
2886
3413
|
def unresolve(self, entry: str, name: str, source: str | None, file: str) -> None:
|
|
@@ -3053,6 +3580,40 @@ def _js_surface(root: Path, entries: list[dict], tree_set: set[str], forms: dict
|
|
|
3053
3580
|
if fact is not None and fact["how"] == "namespace":
|
|
3054
3581
|
reach(target(fact["source"], where))
|
|
3055
3582
|
|
|
3583
|
+
passes: dict[str, list[tuple[str, str, object]]] = {}
|
|
3584
|
+
|
|
3585
|
+
def direct(file: str) -> list[tuple[str, str, object]]:
|
|
3586
|
+
"""What the namespace module `file` passes on, in order: (name,
|
|
3587
|
+
"member", a member traced as publish traces a name) or (name,
|
|
3588
|
+
"module", the file of a namespace among them); a namespace whose
|
|
3589
|
+
module no file here holds is a member itself, with no file, and so
|
|
3590
|
+
is a re-export from another package, which (unlike the same form in
|
|
3591
|
+
an entry point) is not unresolved."""
|
|
3592
|
+
if file not in passes:
|
|
3593
|
+
out: list[tuple[str, str, object]] = []
|
|
3594
|
+
for name, owner in passed_on(file).items():
|
|
3595
|
+
where, local, line, fact = resolve(owner, name, set())
|
|
3596
|
+
how = fact["how"] if fact is not None else None
|
|
3597
|
+
namespace = target(fact["source"], where) if how == "namespace" else None
|
|
3598
|
+
if namespace is not None:
|
|
3599
|
+
out.append((name, "module", namespace))
|
|
3600
|
+
elif how == "namespace":
|
|
3601
|
+
surface.opaque.add((name, None))
|
|
3602
|
+
out.append((name, "member", _member(name, None, None, None)))
|
|
3603
|
+
elif how == "re-export":
|
|
3604
|
+
out.append((name, "member", _member(name, local, None, None)))
|
|
3605
|
+
else:
|
|
3606
|
+
out.append((name, "member", _member(name, local, where, line)))
|
|
3607
|
+
passes[file] = out
|
|
3608
|
+
return passes[file]
|
|
3609
|
+
|
|
3610
|
+
member_lists: dict[str, list[dict]] = {}
|
|
3611
|
+
|
|
3612
|
+
def members(file: str) -> list[dict]:
|
|
3613
|
+
if file not in member_lists:
|
|
3614
|
+
member_lists[file] = _flatten(file, direct, "javascript")
|
|
3615
|
+
return member_lists[file]
|
|
3616
|
+
|
|
3056
3617
|
def publish(entry: str, name: str, via: str, module: str, source: str | None) -> None:
|
|
3057
3618
|
"""Add public `name`, which `module` exports, traced to what it is."""
|
|
3058
3619
|
chain: set[str] = set()
|
|
@@ -3061,7 +3622,8 @@ def _js_surface(root: Path, entries: list[dict], tree_set: set[str], forms: dict
|
|
|
3061
3622
|
how = fact["how"] if fact is not None else None
|
|
3062
3623
|
if how == "namespace":
|
|
3063
3624
|
namespace = target(fact["source"], where)
|
|
3064
|
-
surface.add(name, entry, "namespace", namespace, None, source
|
|
3625
|
+
surface.add(name, entry, "namespace", namespace, None, source,
|
|
3626
|
+
members=members(namespace) if namespace is not None else None)
|
|
3065
3627
|
if namespace is None:
|
|
3066
3628
|
surface.unresolve(entry, name, fact["source"], where)
|
|
3067
3629
|
reach(namespace)
|
|
@@ -3371,9 +3933,16 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
|
|
|
3371
3933
|
if found is None:
|
|
3372
3934
|
return None, binding["imported"], None, "import"
|
|
3373
3935
|
if found.endswith("__init__.py"):
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
|
|
3936
|
+
# Python's own precedence: the package's explicit binding of
|
|
3937
|
+
# the name first, the submodule only when it binds none (or
|
|
3938
|
+
# binds the submodule itself, `from . import name` or
|
|
3939
|
+
# `from pkg.x import name`); a star import is not read here
|
|
3940
|
+
inner = bindings(found)[1].get(binding["imported"])
|
|
3941
|
+
if inner is None or (inner["kind"] == "import" and inner["imported"] == binding["imported"]
|
|
3942
|
+
and module_file(found, inner["level"], inner["module"], top) == found):
|
|
3943
|
+
sub = _py_submodule(posixpath.dirname(found), binding["imported"], tree_set)
|
|
3944
|
+
if sub and sub != rel:
|
|
3945
|
+
return sub, binding["imported"], None, "submodule"
|
|
3377
3946
|
rel, name = found, binding["imported"]
|
|
3378
3947
|
return rel, name, None, "missing"
|
|
3379
3948
|
|
|
@@ -3391,6 +3960,31 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
|
|
|
3391
3960
|
if what == "submodule" and where is not None:
|
|
3392
3961
|
reach(where, top)
|
|
3393
3962
|
|
|
3963
|
+
passes: dict[tuple[str, str], list[tuple[str, str, object]]] = {}
|
|
3964
|
+
|
|
3965
|
+
def direct(rel: str, top: str) -> list[tuple[str, str, object]]:
|
|
3966
|
+
"""What the submodule `rel` passes on (its star_names), in order:
|
|
3967
|
+
(name, "member", a member traced by resolve) or (name, "module",
|
|
3968
|
+
the file of a submodule among them); a name from another package
|
|
3969
|
+
has no file."""
|
|
3970
|
+
if (rel, top) not in passes:
|
|
3971
|
+
out: list[tuple[str, str, object]] = []
|
|
3972
|
+
for name, owner in star_names(rel, top).items():
|
|
3973
|
+
where, local, line, what = resolve(owner, name, top, set())
|
|
3974
|
+
if what == "submodule" and where is not None:
|
|
3975
|
+
out.append((name, "module", where))
|
|
3976
|
+
else:
|
|
3977
|
+
out.append((name, "member", _member(name, local, where, line)))
|
|
3978
|
+
passes[(rel, top)] = out
|
|
3979
|
+
return passes[(rel, top)]
|
|
3980
|
+
|
|
3981
|
+
member_lists: dict[tuple[str, str], list[dict]] = {}
|
|
3982
|
+
|
|
3983
|
+
def members(rel: str, top: str) -> list[dict]:
|
|
3984
|
+
if (rel, top) not in member_lists:
|
|
3985
|
+
member_lists[(rel, top)] = _flatten(rel, lambda module: direct(module, top), "python")
|
|
3986
|
+
return member_lists[(rel, top)]
|
|
3987
|
+
|
|
3394
3988
|
for entry in [e["file"] for e in entries]:
|
|
3395
3989
|
top = posixpath.dirname(entry)
|
|
3396
3990
|
all_names, bound, stars = bindings(entry)
|
|
@@ -3420,7 +4014,7 @@ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _S
|
|
|
3420
4014
|
where, local, line, what = resolve(entry, name, top, chain)
|
|
3421
4015
|
surface.reachable.update(chain)
|
|
3422
4016
|
if what == "submodule":
|
|
3423
|
-
surface.add(name, entry, "namespace", where, None, source, local)
|
|
4017
|
+
surface.add(name, entry, "namespace", where, None, source, local, members=members(where, top))
|
|
3424
4018
|
reach(where, top)
|
|
3425
4019
|
elif what == "import":
|
|
3426
4020
|
surface.add(name, entry, "re-export", None, None, source, local)
|
|
@@ -3674,6 +4268,41 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
|
|
|
3674
4268
|
if sub is not None:
|
|
3675
4269
|
reach(lib, sub, reached)
|
|
3676
4270
|
|
|
4271
|
+
passes: dict[tuple[str, tuple[str, ...]], list[tuple[str, str, object]]] = {}
|
|
4272
|
+
|
|
4273
|
+
def direct(lib: str, here: tuple[str, ...]) -> list[tuple[str, str, object]]:
|
|
4274
|
+
"""What the module at `here` passes on (exported), in order: (name,
|
|
4275
|
+
"member", a member traced as publish traces a name) or (name,
|
|
4276
|
+
"module", the path of a module among them that a file of its own
|
|
4277
|
+
holds); an inline or `#[path]` module is a member itself, at its
|
|
4278
|
+
`pub mod`, and a `pub use` of another crate's item one with no file,
|
|
4279
|
+
which (unlike the same form in the crate root) is not unresolved."""
|
|
4280
|
+
if (lib, here) not in passes:
|
|
4281
|
+
out: list[tuple[str, str, object]] = []
|
|
4282
|
+
for name in exported(lib, list(here)):
|
|
4283
|
+
file, local, line, sub, stop = resolve(lib, list(here), name, set())
|
|
4284
|
+
if stop is not None:
|
|
4285
|
+
out.append((name, "member", _member(name, local, None, None)))
|
|
4286
|
+
elif sub is not None and _rust_file(lib, sub, tree_set) is not None:
|
|
4287
|
+
out.append((name, "module", tuple(sub)))
|
|
4288
|
+
else:
|
|
4289
|
+
if sub is not None:
|
|
4290
|
+
surface.opaque.add((name, file))
|
|
4291
|
+
out.append((name, "member", _member(name, local, file, line)))
|
|
4292
|
+
passes[(lib, here)] = out
|
|
4293
|
+
return passes[(lib, here)]
|
|
4294
|
+
|
|
4295
|
+
member_lists: dict[tuple[str, tuple[str, ...]], list[dict]] = {}
|
|
4296
|
+
|
|
4297
|
+
def members(lib: str, here: list[str]) -> list[dict] | None:
|
|
4298
|
+
"""The members of the module at `here`; None when no file of its own holds it."""
|
|
4299
|
+
if _rust_file(lib, here, tree_set) is None:
|
|
4300
|
+
return None
|
|
4301
|
+
key = (lib, tuple(here))
|
|
4302
|
+
if key not in member_lists:
|
|
4303
|
+
member_lists[key] = _flatten(tuple(here), lambda path: direct(lib, path), "rust")
|
|
4304
|
+
return member_lists[key]
|
|
4305
|
+
|
|
3677
4306
|
def publish(lib: str, here: list[str], name: str, via: str, source: str | None,
|
|
3678
4307
|
reached: set[tuple[str, ...]]) -> None:
|
|
3679
4308
|
"""Add public `name`, which the module at `here` exports, traced to
|
|
@@ -3685,7 +4314,7 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
|
|
|
3685
4314
|
surface.add(name, lib, via, None, None, source, local)
|
|
3686
4315
|
surface.unresolve(lib, name, *stop)
|
|
3687
4316
|
elif sub is not None:
|
|
3688
|
-
surface.add(name, lib, "namespace", file, line, "::".join(sub))
|
|
4317
|
+
surface.add(name, lib, "namespace", file, line, "::".join(sub), members=members(lib, sub))
|
|
3689
4318
|
reach(lib, sub, reached)
|
|
3690
4319
|
else:
|
|
3691
4320
|
surface.add(name, lib, via, file, line, source, local)
|
|
@@ -3700,7 +4329,8 @@ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface:
|
|
|
3700
4329
|
continue
|
|
3701
4330
|
sub = [name.removeprefix("r#")]
|
|
3702
4331
|
sub_file = _rust_file(lib, sub, tree_set)
|
|
3703
|
-
surface.add(name, lib, "namespace", sub_file or lib, None if sub_file else line
|
|
4332
|
+
surface.add(name, lib, "namespace", sub_file or lib, None if sub_file else line,
|
|
4333
|
+
members=members(lib, sub))
|
|
3704
4334
|
reach(lib, sub, reached)
|
|
3705
4335
|
for path, alias, glob, _ in info["uses"]:
|
|
3706
4336
|
if not glob:
|
|
@@ -3770,22 +4400,34 @@ def _go_surface(root: Path, files: list[str], surface: _Surface, entries: list[d
|
|
|
3770
4400
|
|
|
3771
4401
|
def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries: list[dict],
|
|
3772
4402
|
in_scope: set[str]) -> tuple[dict[str, str], dict[tuple[str, str], dict], list[dict],
|
|
3773
|
-
list[tuple[str, dict]], list[tuple[str, dict]]
|
|
4403
|
+
list[tuple[str, dict]], list[tuple[str, dict]],
|
|
4404
|
+
list[tuple[str, dict]]]:
|
|
3774
4405
|
"""The entry-point diff, one language family at a time: (status by
|
|
3775
4406
|
family, public names {(family, name): info}, internal export records,
|
|
3776
|
-
and (family, info) of the extraction gaps
|
|
3777
|
-
|
|
4407
|
+
and (family, info) of the extraction gaps, of the names outside scope
|
|
4408
|
+
and of the other in-scope definitions of public names, `_Surface`'s
|
|
4409
|
+
`others`, which no gap or outside_scope row reads: those go by name).
|
|
4410
|
+
A family whose entry points declare names diffs them with its
|
|
3778
4411
|
exports: an export no entry point names, nor traces back to, nor makes
|
|
3779
4412
|
reachable (a name, or any export of a reachable file), is internal. A
|
|
3780
|
-
declared name
|
|
3781
|
-
|
|
3782
|
-
|
|
3783
|
-
|
|
4413
|
+
declared name, at the file `_Surface` kept for it (not always the first
|
|
4414
|
+
entry point's in path order), is outside scope when a file out of scope
|
|
4415
|
+
defines it, even when a recipe found its name in another file (another
|
|
4416
|
+
definition: no recipe reads a file out of scope), and a gap when no
|
|
4417
|
+
recipe found it and a file in scope defines it (or the trace found no
|
|
4418
|
+
file); a module is no recipe's target, so neither: a namespace with
|
|
4419
|
+
`members` has each member diffed in its place, found when a recipe
|
|
4420
|
+
found its name, or its `local` one, in its file, and given no row of
|
|
4421
|
+
its own when a declared name has its name and file (or another
|
|
4422
|
+
namespace gave it one); one whose members are unknown (None), or a
|
|
4423
|
+
member that is such a module (`opaque`), is no gap. In a family with
|
|
4424
|
+
no entry point, or only empty ones, every export is public."""
|
|
3784
4425
|
statuses: dict[str, str] = {}
|
|
3785
4426
|
public: dict[tuple[str, str], dict] = {}
|
|
3786
4427
|
internal: list[dict] = []
|
|
3787
4428
|
gaps: list[tuple[str, dict]] = []
|
|
3788
4429
|
outside: list[tuple[str, dict]] = []
|
|
4430
|
+
others: list[tuple[str, dict]] = []
|
|
3789
4431
|
entry_families = {FAMILY_OF[e["language"]] for e in entries}
|
|
3790
4432
|
families = {FAMILY_OF[e["language"]] for e in exports} | entry_families
|
|
3791
4433
|
for family in sorted(families | {f for f, s in surfaces.items() if s.names}):
|
|
@@ -3803,6 +4445,7 @@ def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries
|
|
|
3803
4445
|
names = {e["export_name"] for e in records}
|
|
3804
4446
|
traced = {info["local"] for info in declared.values() if info["local"]} | surface.reachable
|
|
3805
4447
|
public.update(((family, name), info) for name, info in declared.items())
|
|
4448
|
+
others.extend((family, dict(info)) for name in sorted(surface.others) for info in surface.others[name])
|
|
3806
4449
|
seen: set[str] = set()
|
|
3807
4450
|
for e in records:
|
|
3808
4451
|
name = e["export_name"]
|
|
@@ -3811,10 +4454,58 @@ def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries
|
|
|
3811
4454
|
seen.add(name)
|
|
3812
4455
|
internal.append(e)
|
|
3813
4456
|
for name, info in declared.items():
|
|
3814
|
-
if
|
|
4457
|
+
if info["via"] == "namespace":
|
|
3815
4458
|
continue
|
|
3816
|
-
|
|
3817
|
-
|
|
4459
|
+
# A recipe never reads a file out of scope: a name it found elsewhere is another definition.
|
|
4460
|
+
if info["file"] is not None and info["file"] not in in_scope:
|
|
4461
|
+
outside.append((family, info))
|
|
4462
|
+
elif name not in names and (info["local"] or name) not in names:
|
|
4463
|
+
gaps.append((family, info))
|
|
4464
|
+
# A namespace's members, each by its name (or `local`) and file, or
|
|
4465
|
+
# by name alone when it has no file, as a declared name is: a name
|
|
4466
|
+
# at a file a declared name already gives counts once.
|
|
4467
|
+
found = {(e["export_name"], e["source_file"]) for e in records}
|
|
4468
|
+
given = {(info["name"], info["file"]) for info in declared.values() if not _has_members(info)}
|
|
4469
|
+
for info in declared.values():
|
|
4470
|
+
for member in info["members"] if _has_members(info) else []:
|
|
4471
|
+
key = (member["name"], member["file"])
|
|
4472
|
+
if (key in given or key in found or key in surface.opaque
|
|
4473
|
+
or (member["local"] and (member["local"], member["file"]) in found)
|
|
4474
|
+
or (member["file"] is None and {member["name"], member["local"]} & names)):
|
|
4475
|
+
continue
|
|
4476
|
+
given.add(key)
|
|
4477
|
+
row = {"name": member["name"], "entry": info["entry"], "file": member["file"],
|
|
4478
|
+
"line": member["line"]}
|
|
4479
|
+
(gaps if member["file"] is None or member["file"] in in_scope else outside).append((family, row))
|
|
4480
|
+
return statuses, public, internal, gaps, outside, others
|
|
4481
|
+
|
|
4482
|
+
|
|
4483
|
+
def _has_members(info: dict) -> bool:
|
|
4484
|
+
"""A namespace item whose module's members the trace read (a list, maybe empty)."""
|
|
4485
|
+
return info.get("via") == "namespace" and isinstance(info.get("members"), list)
|
|
4486
|
+
|
|
4487
|
+
|
|
4488
|
+
def _public_keys(public: dict[tuple[str, str], dict]) -> set[tuple[str, str, str | None]]:
|
|
4489
|
+
"""The (family, name, file) exports_public_api counts: each public name
|
|
4490
|
+
but a namespace with members, in its place each member, and so a member
|
|
4491
|
+
that is also a declared name (or another namespace's member) once."""
|
|
4492
|
+
keys: set[tuple[str, str, str | None]] = set()
|
|
4493
|
+
for (family, _), info in public.items():
|
|
4494
|
+
if _has_members(info):
|
|
4495
|
+
keys.update((family, m["name"], m["file"]) for m in info["members"])
|
|
4496
|
+
else:
|
|
4497
|
+
keys.add((family, info["name"], info["file"]))
|
|
4498
|
+
return keys
|
|
4499
|
+
|
|
4500
|
+
|
|
4501
|
+
def _other_keys(others: list[tuple[str, dict]]) -> set[tuple[str, str, str | None]]:
|
|
4502
|
+
"""The (family, name, file) of the other in-scope definitions of public
|
|
4503
|
+
names (#830), which join the public keys in the set effective_denominator
|
|
4504
|
+
and curated_subset read, never exports_public_api: a name still counts
|
|
4505
|
+
once, when any of its files matches."""
|
|
4506
|
+
return {(family, info["name"], info["file"]) for family, info in others
|
|
4507
|
+
if isinstance(info.get("name"), str) and info["name"] and isinstance(info.get("file"), str)
|
|
4508
|
+
and info["file"]}
|
|
3818
4509
|
|
|
3819
4510
|
|
|
3820
4511
|
def _head_cap(value: int | None, tier: str | None, scope_type: str | None) -> int | None:
|
|
@@ -3823,27 +4514,133 @@ def _head_cap(value: int | None, tier: str | None, scope_type: str | None) -> in
|
|
|
3823
4514
|
return HEAD_CAPS.get((tier, scope_type), DEFAULT_HEAD_CAP)
|
|
3824
4515
|
|
|
3825
4516
|
|
|
3826
|
-
def
|
|
3827
|
-
"""skf-detect-workspaces.py's
|
|
4517
|
+
def _workspaces(root: Path, tree: list[str], tree_set: set[str]) -> dict:
|
|
4518
|
+
"""skf-detect-workspaces.py's result over `tree` (its workspaces
|
|
4519
|
+
resolved against those paths), with the content of each root manifest
|
|
4520
|
+
of WORKSPACE_MANIFESTS that `tree_set` lists. No brief scope applies:
|
|
4521
|
+
the monorepo arm reads it through _monorepo."""
|
|
3828
4522
|
manifests = {}
|
|
3829
4523
|
for name in WORKSPACE_MANIFESTS:
|
|
3830
4524
|
if name in tree_set:
|
|
3831
4525
|
text = _read_text(root, name)
|
|
3832
4526
|
if text is not None:
|
|
3833
4527
|
manifests[name] = text
|
|
3834
|
-
|
|
4528
|
+
return _sibling("skf-detect-workspaces.py").detect({"tree": tree, "manifests": manifests})
|
|
4529
|
+
|
|
4530
|
+
|
|
4531
|
+
def _monorepo(root: Path, tree: list[str], excludes: list[str],
|
|
4532
|
+
glob_match: Callable[[str, str], bool]) -> tuple[bool, str | None]:
|
|
4533
|
+
"""Arm (a): (is the source a monorepo, its manifest kind), from
|
|
4534
|
+
skf-detect-workspaces.py over the tree the brief keeps, its
|
|
4535
|
+
scoped_tree: a workspace member counts only through the files no
|
|
4536
|
+
scope.exclude glob matches, every root manifest stays, and a workspace
|
|
4537
|
+
with no member left is none (#780)."""
|
|
4538
|
+
kept = _sibling("skf-detect-workspaces.py").scoped_tree(tree, excludes, glob_match)
|
|
4539
|
+
result = _workspaces(root, kept, set(kept))
|
|
3835
4540
|
return bool(result.get("is_monorepo")), result.get("manifest_kind")
|
|
3836
4541
|
|
|
3837
4542
|
|
|
4543
|
+
def _workspace_sources(result: dict, tree: list[str], wanted: set[str]) -> list[set[str]]:
|
|
4544
|
+
"""The source files (of a language --language selects) under each
|
|
4545
|
+
workspace of `result`, one set per workspace that holds any. _scope_facts
|
|
4546
|
+
passes the workspaces skf-detect-workspaces.py finds over the whole
|
|
4547
|
+
tree, scope.exclude aside (unlike arm (a)): a member the brief drops is
|
|
4548
|
+
still part of the repository the brief chose a subset of."""
|
|
4549
|
+
sources = []
|
|
4550
|
+
for workspace in result.get("workspaces") or [] if result.get("is_monorepo") else []:
|
|
4551
|
+
folder = _norm(workspace.get("path") if isinstance(workspace, dict) else "").strip("/")
|
|
4552
|
+
files = {f for f in tree if (not folder or f.startswith(folder + "/")) and _file_language(f) in wanted}
|
|
4553
|
+
if files:
|
|
4554
|
+
sources.append(files)
|
|
4555
|
+
return sources
|
|
4556
|
+
|
|
4557
|
+
|
|
4558
|
+
def denominator_count(counted: set[tuple[str, str, str | None]], tree: list[str], includes: list[str],
|
|
4559
|
+
excludes: list[str], tier_a: list[str], glob_match: Callable[[str, str], bool],
|
|
4560
|
+
read_all: bool) -> tuple[int, str, set[str], str | None]:
|
|
4561
|
+
"""(effective_denominator, its basis, the files it counts, a warning or None)
|
|
4562
|
+
over the (family, name, file) keys `counted`: each name once whose file
|
|
4563
|
+
matches scope.include (any file with no glob), scope.tier_a_include when
|
|
4564
|
+
given (a tier A name counts only in a file scope.include matches too) and
|
|
4565
|
+
no scope.exclude glob. A tier A list that counts no name falls back to
|
|
4566
|
+
scope.include (else every file) once a name was counted and every file
|
|
4567
|
+
was read (`read_all`)."""
|
|
4568
|
+
|
|
4569
|
+
def files_matching(globs: list[str]) -> set[str]:
|
|
4570
|
+
return {f for f in tree if (not globs or any(glob_match(f, g) for g in globs))
|
|
4571
|
+
and not any(glob_match(f, g) for g in excludes)}
|
|
4572
|
+
|
|
4573
|
+
in_include = files_matching(includes)
|
|
4574
|
+
denominator_files = (files_matching(tier_a) & in_include) if tier_a else in_include
|
|
4575
|
+
basis = "tier_a_include" if tier_a else ("scope.include" if includes else "all-files")
|
|
4576
|
+
warning = None
|
|
4577
|
+
if tier_a and counted and read_all and not any(file in denominator_files for _, _, file in counted):
|
|
4578
|
+
basis = "scope.include" if includes else "all-files"
|
|
4579
|
+
denominator_files = in_include
|
|
4580
|
+
counts_what = "scope.include" if includes else "every file"
|
|
4581
|
+
warning = f"scope.tier_a_include counts no public name: effective_denominator counts {counts_what}"
|
|
4582
|
+
# Each name once, as Test Skill's sets of names count it: a name the entry points pass on
|
|
4583
|
+
# from two files, or in two languages, is two keys but one name.
|
|
4584
|
+
count = len({name for _, name, file in counted if file in denominator_files})
|
|
4585
|
+
return count, basis, denominator_files, warning
|
|
4586
|
+
|
|
4587
|
+
|
|
4588
|
+
def curated_subset(counted: set[tuple[str, str, str | None]], basis: str, tree: list[str], includes: list[str],
|
|
4589
|
+
excludes: list[str], tier_a: list[str], glob_match: Callable[[str, str], bool],
|
|
4590
|
+
arms: dict, workspace_sources: list[set[str]]) -> bool:
|
|
4591
|
+
"""Whether the files the count read are a curated subset rather than the
|
|
4592
|
+
whole workspace (in a single-package repository, the whole package): the
|
|
4593
|
+
files the basis globs list (a tier A list's, among those scope.include
|
|
4594
|
+
lists; under arm (b), less those scope.exclude drops) leave out a file
|
|
4595
|
+
that defines a counted public name or every source file of one of
|
|
4596
|
+
`workspace_sources` (the workspaces found over the whole tree,
|
|
4597
|
+
scope.exclude aside, so excluding a member the include globs leave out
|
|
4598
|
+
anyway changes nothing). A basis of all-files lists none."""
|
|
4599
|
+
if basis == "all-files":
|
|
4600
|
+
return False
|
|
4601
|
+
files = {f for f in tree if not includes or any(glob_match(f, g) for g in includes)}
|
|
4602
|
+
if basis == "tier_a_include":
|
|
4603
|
+
files = {f for f in files if any(glob_match(f, g) for g in tier_a)}
|
|
4604
|
+
if arms.get("specific_modules"):
|
|
4605
|
+
files = {f for f in files if not any(glob_match(f, g) for g in excludes)}
|
|
4606
|
+
tree_set = set(tree)
|
|
4607
|
+
defining = {file for _, _, file in counted if file in tree_set}
|
|
4608
|
+
return bool(defining - files) or any(not (sources & files) for sources in workspace_sources)
|
|
4609
|
+
|
|
4610
|
+
|
|
4611
|
+
def effective_denominator_decision(arms: dict, scope_type: str | None, basis: str, curated: bool, count: int,
|
|
4612
|
+
incomplete: bool) -> dict:
|
|
4613
|
+
"""Whether create-skill and update-skill store the count as
|
|
4614
|
+
stats.effective_denominator, the brief's scope.notes aside: {store, reason},
|
|
4615
|
+
the reason naming the first condition that fails."""
|
|
4616
|
+
if scope_type == "reference-app":
|
|
4617
|
+
reason = "reference-app"
|
|
4618
|
+
elif incomplete:
|
|
4619
|
+
reason = "incomplete-run"
|
|
4620
|
+
elif not any(arms.get(arm) for arm in DENOMINATOR_ARMS):
|
|
4621
|
+
reason = "no-arm"
|
|
4622
|
+
elif scope_type == "full-library":
|
|
4623
|
+
reason = "full-library"
|
|
4624
|
+
elif basis == "all-files":
|
|
4625
|
+
reason = "all-files"
|
|
4626
|
+
elif not curated:
|
|
4627
|
+
reason = "not-curated"
|
|
4628
|
+
elif count == 0:
|
|
4629
|
+
reason = "zero-count"
|
|
4630
|
+
else:
|
|
4631
|
+
return {"store": True, "reason": "curated-subset"}
|
|
4632
|
+
return {"store": False, "reason": reason}
|
|
4633
|
+
|
|
4634
|
+
|
|
3838
4635
|
def _select_files(root: Path, candidates: list[str], listed: bool, includes: list[str], excludes: list[str],
|
|
3839
4636
|
wanted: set[str], glob_match: Callable[[str, str], bool], issues: list[dict],
|
|
3840
|
-
warnings: list[str]) -> tuple[list[str],
|
|
3841
|
-
"""(the files in scope,
|
|
3842
|
-
|
|
3843
|
-
recipe reads, is an issue; an include glob
|
|
3844
|
-
warning."""
|
|
4637
|
+
warnings: list[str]) -> tuple[list[str], list[str], list[str]]:
|
|
4638
|
+
"""(the files in scope, the in-scope source files no recipe reads, the
|
|
4639
|
+
include globs no candidate matches). A file a --files-from list names
|
|
4640
|
+
that is missing, or that no recipe reads, is an issue; an include glob
|
|
4641
|
+
no candidate matches, a warning too."""
|
|
3845
4642
|
in_scope: list[str] = []
|
|
3846
|
-
without_recipes:
|
|
4643
|
+
without_recipes: list[str] = []
|
|
3847
4644
|
used = dict.fromkeys(includes, False)
|
|
3848
4645
|
for rel in candidates:
|
|
3849
4646
|
hits = [g for g in includes if glob_match(rel, g)]
|
|
@@ -3859,55 +4656,70 @@ def _select_files(root: Path, candidates: list[str], listed: bool, includes: lis
|
|
|
3859
4656
|
if listed:
|
|
3860
4657
|
issues.append({"file": rel, "issue": "no-recipes"})
|
|
3861
4658
|
elif extension in NO_RECIPE_EXTENSIONS:
|
|
3862
|
-
without_recipes
|
|
4659
|
+
without_recipes.append(rel)
|
|
3863
4660
|
continue
|
|
3864
4661
|
in_scope.append(rel)
|
|
3865
|
-
|
|
3866
|
-
|
|
4662
|
+
unmatched = [g for g, hit in used.items() if not hit]
|
|
4663
|
+
warnings += [f"scope.include pattern {g!r} matches no file" for g in unmatched]
|
|
4664
|
+
return in_scope, without_recipes, unmatched
|
|
3867
4665
|
|
|
3868
4666
|
|
|
3869
|
-
def
|
|
3870
|
-
"""
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
|
|
3875
|
-
|
|
3876
|
-
|
|
4667
|
+
def _by_extension(files: list[str]) -> dict[str, int]:
|
|
4668
|
+
"""`files_without_recipes`: how many of `files` each extension has."""
|
|
4669
|
+
return dict(sorted(Counter(posixpath.splitext(f)[1].lower() for f in files).items()))
|
|
4670
|
+
|
|
4671
|
+
|
|
4672
|
+
def _unmatched_tier_a(tier_a: list[str], includes: list[str], tree: list[str],
|
|
4673
|
+
glob_match: Callable[[str, str], bool], warnings: list[str]) -> list[str]:
|
|
4674
|
+
"""The tier A globs that match no file of `tree` (the whole tree, regardless
|
|
4675
|
+
of --files-from and before the excludes), with a warning for each, and one
|
|
4676
|
+
for each that matches only files outside scope.include."""
|
|
4677
|
+
unmatched = [g for g in dict.fromkeys(tier_a) if not any(glob_match(f, g) for f in tree)]
|
|
4678
|
+
warnings += [f"scope.tier_a_include pattern {g!r} matches no file" for g in unmatched]
|
|
4679
|
+
warnings += [f"scope.tier_a_include pattern {g!r} matches only files outside scope.include"
|
|
4680
|
+
for g in dict.fromkeys(tier_a) if includes and g not in unmatched
|
|
4681
|
+
and not any(glob_match(f, g) and any(glob_match(f, i) for i in includes) for f in tree)]
|
|
4682
|
+
return unmatched
|
|
4683
|
+
|
|
4684
|
+
|
|
4685
|
+
def _brief_scope(args: argparse.Namespace) -> dict:
|
|
4686
|
+
"""The scope globs, scope type and languages: the flags', else the brief's."""
|
|
3877
4687
|
brief = _read_brief(Path(args.brief)) if args.brief else {}
|
|
3878
|
-
includes = [g for g in (_norm(v) for v in (args.include or brief.get("include") or [])) if g]
|
|
3879
|
-
excludes = [g for g in (_norm(v) for v in (args.exclude or brief.get("exclude") or [])) if g]
|
|
3880
|
-
tier_a = [g for g in (_norm(v) for v in (args.tier_a_include or brief.get("tier_a_include") or [])) if g]
|
|
3881
|
-
scope_type = args.scope_type or brief.get("type")
|
|
3882
4688
|
language_values = args.language or ([brief["language"]] if brief.get("language") else [])
|
|
3883
4689
|
wanted, unknown_languages = _wanted_languages(language_values)
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
4690
|
+
return {
|
|
4691
|
+
"includes": [g for g in (_norm(v) for v in (args.include or brief.get("include") or [])) if g],
|
|
4692
|
+
"excludes": [g for g in (_norm(v) for v in (args.exclude or brief.get("exclude") or [])) if g],
|
|
4693
|
+
"tier_a": [g for g in (_norm(v) for v in (args.tier_a_include or brief.get("tier_a_include") or [])) if g],
|
|
4694
|
+
"scope_type": args.scope_type or brief.get("type"),
|
|
4695
|
+
"wanted": wanted,
|
|
4696
|
+
"glob_match": _sibling("skf-resolve-authoritative-files.py").glob_match,
|
|
4697
|
+
"warnings": [f"no recipe reads {value} files" for value in unknown_languages],
|
|
4698
|
+
}
|
|
3887
4699
|
|
|
3888
|
-
|
|
3889
|
-
|
|
4700
|
+
|
|
4701
|
+
def _scope_facts(args: argparse.Namespace, root: Path) -> dict:
|
|
4702
|
+
"""What full mode and denominator mode know before any recipe runs: the
|
|
4703
|
+
brief's scope, the files in scope, the entry points and the arms."""
|
|
4704
|
+
scope = _brief_scope(args)
|
|
4705
|
+
includes, excludes, tier_a = scope["includes"], scope["excludes"], scope["tier_a"]
|
|
4706
|
+
scope_type, wanted, glob_match = scope["scope_type"], scope["wanted"], scope["glob_match"]
|
|
4707
|
+
warnings: list[str] = scope["warnings"]
|
|
3890
4708
|
issues: list[dict] = []
|
|
3891
4709
|
tree = list_tree(root)
|
|
3892
4710
|
tree_set = set(tree)
|
|
3893
|
-
|
|
3894
|
-
|
|
3895
|
-
|
|
3896
|
-
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
3901
|
-
issues.append({"file": rel, "issue": "unreadable"})
|
|
3902
|
-
continue
|
|
3903
|
-
except UnicodeDecodeError:
|
|
3904
|
-
issues.append({"file": rel, "issue": "not-utf8"})
|
|
3905
|
-
continue
|
|
3906
|
-
readable.append(rel)
|
|
4711
|
+
files_from = getattr(args, "files_from", None)
|
|
4712
|
+
listed = _read_file_list(Path(files_from)) if files_from else None
|
|
4713
|
+
in_scope, without_recipes, unmatched_include = _select_files(
|
|
4714
|
+
root, tree if listed is None else listed, listed is not None, includes, excludes, wanted, glob_match,
|
|
4715
|
+
issues, warnings)
|
|
4716
|
+
# A tier A glob is matched against the whole tree under --source-root (the
|
|
4717
|
+
# tree effective_denominator reads).
|
|
4718
|
+
unmatched_tier_a = _unmatched_tier_a(tier_a, includes, tree, glob_match, warnings)
|
|
3907
4719
|
|
|
3908
4720
|
# The entry points and the arms need no ast-grep.
|
|
3909
4721
|
in_scope_set = set(in_scope)
|
|
3910
|
-
surfaces = {family: _Surface() for family in FAMILIES}
|
|
4722
|
+
surfaces = {family: _Surface(family, in_scope_set) for family in FAMILIES}
|
|
3911
4723
|
js_entries, maps, unresolved_subpaths = _js_entries(root, in_scope, tree, tree_set, warnings)
|
|
3912
4724
|
py_entries = _py_entries(in_scope, tree_set)
|
|
3913
4725
|
rust_entries = _rust_entries(root, in_scope, tree, tree_set)
|
|
@@ -3922,13 +4734,85 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
3922
4734
|
f"{entry['package']}) is outside the files in scope: widen scope.include to document "
|
|
3923
4735
|
"the names it exports (entry_point_diff.outside_scope), which effective_denominator "
|
|
3924
4736
|
"leaves out")
|
|
3925
|
-
monorepo, monorepo_kind = _monorepo(root, tree,
|
|
4737
|
+
monorepo, monorepo_kind = _monorepo(root, tree, excludes, glob_match)
|
|
3926
4738
|
arms = {
|
|
3927
4739
|
"monorepo": monorepo,
|
|
3928
4740
|
"monorepo_kind": monorepo_kind,
|
|
3929
4741
|
"specific_modules": scope_type == "specific-modules" and not monorepo,
|
|
3930
4742
|
"multi_subpath_exports": not monorepo and any(m["non_root_subpaths"] > 1 for m in maps),
|
|
3931
4743
|
}
|
|
4744
|
+
return {
|
|
4745
|
+
"includes": includes, "excludes": excludes, "tier_a": tier_a, "scope_type": scope_type, "wanted": wanted,
|
|
4746
|
+
"glob_match": glob_match, "warnings": warnings, "issues": issues, "tree": tree, "tree_set": tree_set,
|
|
4747
|
+
"in_scope": in_scope, "without_recipes": without_recipes, "unmatched_include": unmatched_include,
|
|
4748
|
+
"unmatched_tier_a": unmatched_tier_a, "surfaces": surfaces, "js_entries": js_entries, "maps": maps,
|
|
4749
|
+
"unresolved_subpaths": unresolved_subpaths, "py_entries": py_entries, "rust_entries": rust_entries,
|
|
4750
|
+
"entries": entries, "arms": arms,
|
|
4751
|
+
"workspace_sources": _workspace_sources(_workspaces(root, tree, tree_set), tree, wanted),
|
|
4752
|
+
}
|
|
4753
|
+
|
|
4754
|
+
|
|
4755
|
+
def _scope_record(args: argparse.Namespace, facts: dict) -> dict:
|
|
4756
|
+
"""The `scope` object full mode and denominator mode write."""
|
|
4757
|
+
return {"include": facts["includes"], "exclude": facts["excludes"], "tier_a_include": facts["tier_a"] or None,
|
|
4758
|
+
"type": facts["scope_type"], "languages": sorted(facts["wanted"]),
|
|
4759
|
+
"files_from": getattr(args, "files_from", None), "unmatched_include": facts["unmatched_include"],
|
|
4760
|
+
"unmatched_tier_a_include": facts["unmatched_tier_a"]}
|
|
4761
|
+
|
|
4762
|
+
|
|
4763
|
+
def _denominator_counts(facts: dict, counted: set[tuple[str, str, str | None]], public_api: int, internal: int,
|
|
4764
|
+
incomplete: bool, warnings: list[str]) -> dict:
|
|
4765
|
+
"""The `counts` object: the two API counts, effective_denominator with its
|
|
4766
|
+
basis and files, whether those files are a curated subset, and the
|
|
4767
|
+
decision whether to store the count."""
|
|
4768
|
+
count, basis, denominator_files, warning = denominator_count(
|
|
4769
|
+
counted, facts["tree"], facts["includes"], facts["excludes"], facts["tier_a"], facts["glob_match"],
|
|
4770
|
+
not incomplete)
|
|
4771
|
+
if warning:
|
|
4772
|
+
warnings.append(warning)
|
|
4773
|
+
curated = curated_subset(counted, basis, facts["tree"], facts["includes"], facts["excludes"], facts["tier_a"],
|
|
4774
|
+
facts["glob_match"], facts["arms"], facts["workspace_sources"])
|
|
4775
|
+
return {
|
|
4776
|
+
"exports_public_api": public_api,
|
|
4777
|
+
"exports_internal": internal,
|
|
4778
|
+
"effective_denominator": count,
|
|
4779
|
+
"effective_denominator_basis": basis,
|
|
4780
|
+
"denominator_files": len(denominator_files),
|
|
4781
|
+
"curated_subset": curated,
|
|
4782
|
+
"effective_denominator_decision": effective_denominator_decision(
|
|
4783
|
+
facts["arms"], facts["scope_type"], basis, curated, count, incomplete),
|
|
4784
|
+
}
|
|
4785
|
+
|
|
4786
|
+
|
|
4787
|
+
def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
4788
|
+
"""Run the recipes over the source tree; (result, exit code)."""
|
|
4789
|
+
clock = _Clock(DEFAULT_TIMEOUT_SEC if args.timeout is None else args.timeout)
|
|
4790
|
+
root = Path(args.source_root)
|
|
4791
|
+
if not root.is_dir():
|
|
4792
|
+
raise RunnerError(f"source root not found: {args.source_root}")
|
|
4793
|
+
recipes_path = Path(args.recipes) if args.recipes else RECIPES_FILE
|
|
4794
|
+
recipes = load_recipe_file(recipes_path)
|
|
4795
|
+
facts = _scope_facts(args, root)
|
|
4796
|
+
scope_type, warnings, issues = facts["scope_type"], facts["warnings"], facts["issues"]
|
|
4797
|
+
tree_set, in_scope = facts["tree_set"], facts["in_scope"]
|
|
4798
|
+
in_scope_set = set(in_scope)
|
|
4799
|
+
surfaces, js_entries, maps = facts["surfaces"], facts["js_entries"], facts["maps"]
|
|
4800
|
+
py_entries, rust_entries, entries = facts["py_entries"], facts["rust_entries"], facts["entries"]
|
|
4801
|
+
unresolved_subpaths, arms = facts["unresolved_subpaths"], facts["arms"]
|
|
4802
|
+
recipe_set = args.recipe_set or ("component-library" if scope_type == "component-library" else "standard")
|
|
4803
|
+
head_cap = _head_cap(args.head_cap, args.tier, scope_type)
|
|
4804
|
+
errors: list[dict] = []
|
|
4805
|
+
readable = []
|
|
4806
|
+
for rel in in_scope:
|
|
4807
|
+
try:
|
|
4808
|
+
(root / rel).read_bytes().decode("utf-8")
|
|
4809
|
+
except OSError:
|
|
4810
|
+
issues.append({"file": rel, "issue": "unreadable"})
|
|
4811
|
+
continue
|
|
4812
|
+
except UnicodeDecodeError:
|
|
4813
|
+
issues.append({"file": rel, "issue": "not-utf8"})
|
|
4814
|
+
continue
|
|
4815
|
+
readable.append(rel)
|
|
3932
4816
|
|
|
3933
4817
|
exe, version, shim = _ast_grep()
|
|
3934
4818
|
result: dict = {
|
|
@@ -3938,12 +4822,10 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
3938
4822
|
"recipes_file": recipes_path.as_posix(),
|
|
3939
4823
|
"recipe_set": recipe_set,
|
|
3940
4824
|
"ast_grep": {"path": exe, "version": version},
|
|
3941
|
-
"scope":
|
|
3942
|
-
"type": scope_type, "languages": sorted(wanted),
|
|
3943
|
-
"files_from": args.files_from},
|
|
4825
|
+
"scope": _scope_record(args, facts),
|
|
3944
4826
|
"files_in_scope": len(in_scope),
|
|
3945
4827
|
"files_by_language": dict(sorted(Counter(_file_language(f) for f in in_scope).items())),
|
|
3946
|
-
"files_without_recipes": without_recipes,
|
|
4828
|
+
"files_without_recipes": _by_extension(facts["without_recipes"]),
|
|
3947
4829
|
"file_issues": issues,
|
|
3948
4830
|
"head_cap": head_cap,
|
|
3949
4831
|
"truncated": False,
|
|
@@ -3958,6 +4840,9 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
3958
4840
|
"arms": arms,
|
|
3959
4841
|
"errors": errors,
|
|
3960
4842
|
"warnings": warnings,
|
|
4843
|
+
# Last: only a run without the recipes reads them, and a large tree lists thousands.
|
|
4844
|
+
"files_in_scope_list": in_scope,
|
|
4845
|
+
"files_without_recipes_list": facts["without_recipes"],
|
|
3961
4846
|
}
|
|
3962
4847
|
if exe is None:
|
|
3963
4848
|
result["status"] = "no-ast-grep"
|
|
@@ -4007,16 +4892,20 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
4007
4892
|
for record in exports:
|
|
4008
4893
|
if record["from"] and record["language"] in JS_FAMILY:
|
|
4009
4894
|
record["from_file"] = resolve_js_module(record["from"], record["source_file"], tree_set)
|
|
4010
|
-
statuses, public, internal, gaps, outside = entry_point_diff(exports, surfaces, entries, in_scope_set)
|
|
4895
|
+
statuses, public, internal, gaps, outside, others = entry_point_diff(exports, surfaces, entries, in_scope_set)
|
|
4011
4896
|
if "barrel" in statuses.values():
|
|
4012
4897
|
status = "barrel"
|
|
4013
4898
|
else:
|
|
4014
4899
|
status = "empty-barrel" if "empty-barrel" in statuses.values() else "no-entry-point"
|
|
4015
4900
|
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
4019
|
-
|
|
4901
|
+
# A tier A list that counts no public name (each glob matches no file, or
|
|
4902
|
+
# only files an exclude drops, that scope.include leaves out or that
|
|
4903
|
+
# define no public name) narrows nothing, once the run counted a name
|
|
4904
|
+
# and read every file (no error).
|
|
4905
|
+
# Each other in-scope definition of a public name counts its name where any of its files is counted (#830).
|
|
4906
|
+
public_keys = _public_keys(public)
|
|
4907
|
+
counted = public_keys | _other_keys(others)
|
|
4908
|
+
counts = _denominator_counts(facts, counted, len(public_keys), len(internal), bool(errors), warnings)
|
|
4020
4909
|
|
|
4021
4910
|
by_type: Counter = Counter(e["export_type"] for e in exports)
|
|
4022
4911
|
ordered = sorted(public.items(), key=lambda item: (item[0][1], item[0][0]))
|
|
@@ -4030,6 +4919,8 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
4030
4919
|
"unresolved": unresolved_subpaths + [u for s in surfaces.values() for u in s.unresolved]},
|
|
4031
4920
|
"entry_point_diff": {
|
|
4032
4921
|
"public": [dict(info, language=family) for (family, _), info in ordered],
|
|
4922
|
+
"other_definitions": [dict(info, language=family) for family, info in
|
|
4923
|
+
sorted(others, key=lambda o: (o[1]["name"], o[1]["file"], o[0]))],
|
|
4033
4924
|
"internal": [{"name": e["export_name"], "language": FAMILY_OF[e["language"]],
|
|
4034
4925
|
"source_file": e["source_file"], "source_line": e["source_line"]} for e in internal],
|
|
4035
4926
|
"extraction_gaps": [{"name": info["name"], "language": family, "entry": info["entry"],
|
|
@@ -4043,13 +4934,7 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
4043
4934
|
{"name": info["name"], "language": family, "local": info["local"], "from": info["from"],
|
|
4044
4935
|
"entry": info["entry"], "file": info["file"], "line": info["line"], "via": info["via"]}
|
|
4045
4936
|
for (family, _), info in ordered if info["via"] in ("re-export", "star", "namespace")],
|
|
4046
|
-
"counts":
|
|
4047
|
-
"exports_public_api": len(public),
|
|
4048
|
-
"exports_internal": len(internal),
|
|
4049
|
-
"effective_denominator": sum(1 for info in public.values() if info["file"] in denominator_files),
|
|
4050
|
-
"effective_denominator_basis": basis,
|
|
4051
|
-
"denominator_files": len(denominator_files),
|
|
4052
|
-
},
|
|
4937
|
+
"counts": counts,
|
|
4053
4938
|
})
|
|
4054
4939
|
result["file_issues"] = sorted(issues, key=lambda i: (i["file"], i["issue"]))
|
|
4055
4940
|
if errors:
|
|
@@ -4058,37 +4943,273 @@ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
|
4058
4943
|
return result, 0
|
|
4059
4944
|
|
|
4060
4945
|
|
|
4946
|
+
def _full_exit_2(message: str) -> int:
|
|
4947
|
+
"""Exit 2 of full mode (an input error, a failed -o write or a crash):
|
|
4948
|
+
`message` on one stderr line, each line break and the whitespace around
|
|
4949
|
+
it joined into one space (a YAML error spans several lines) and every
|
|
4950
|
+
other space kept, so a path is quoted as written. Callers quote the line
|
|
4951
|
+
as the reason they stop or fall back."""
|
|
4952
|
+
sys.stderr.write("error: " + re.sub(r"\s*[\r\n]\s*", " ", message.strip("\r\n")) + "\n")
|
|
4953
|
+
return 2
|
|
4954
|
+
|
|
4955
|
+
|
|
4061
4956
|
def _main_full(args: argparse.Namespace) -> int:
|
|
4062
4957
|
if not args.source_root:
|
|
4063
|
-
|
|
4064
|
-
return 2
|
|
4958
|
+
return _full_exit_2("--mode full needs --source-root")
|
|
4065
4959
|
if args.head_cap is not None and args.head_cap < 0:
|
|
4066
|
-
|
|
4067
|
-
return 2
|
|
4960
|
+
return _full_exit_2("--head-cap must be 0 (no cap) or more")
|
|
4068
4961
|
if args.timeout is not None and not args.timeout > 0:
|
|
4069
|
-
|
|
4070
|
-
return 2
|
|
4962
|
+
return _full_exit_2("--timeout must be more than 0 seconds")
|
|
4071
4963
|
try:
|
|
4072
4964
|
result, code = run_full(args)
|
|
4073
4965
|
except RunnerError as exc:
|
|
4074
|
-
|
|
4075
|
-
return 2
|
|
4966
|
+
return _full_exit_2(str(exc))
|
|
4076
4967
|
except Exception as exc: # noqa: BLE001 - one stderr line and exit 2, never a traceback
|
|
4077
|
-
|
|
4078
|
-
sys.stderr.write(f"error: the run failed: {type(exc).__name__}: {detail}\n")
|
|
4079
|
-
return 2
|
|
4968
|
+
return _full_exit_2(f"the run failed: {type(exc).__name__}: {exc}")
|
|
4080
4969
|
text = json.dumps(result, indent=2) + "\n"
|
|
4081
4970
|
if args.output:
|
|
4082
4971
|
try:
|
|
4083
4972
|
Path(args.output).write_text(text, encoding="utf-8")
|
|
4084
4973
|
except OSError as exc:
|
|
4085
|
-
|
|
4086
|
-
return 2
|
|
4974
|
+
return _full_exit_2(f"cannot write {args.output}: {exc}")
|
|
4087
4975
|
else:
|
|
4088
4976
|
sys.stdout.write(text)
|
|
4089
4977
|
return code
|
|
4090
4978
|
|
|
4091
4979
|
|
|
4980
|
+
# --------------------------------------------------------------------------
|
|
4981
|
+
# Denominator mode: the counts and the effective_denominator decision of
|
|
4982
|
+
# names counted without the runner's diff
|
|
4983
|
+
# --------------------------------------------------------------------------
|
|
4984
|
+
|
|
4985
|
+
NO_ARMS = {"monorepo": False, "monorepo_kind": None, "specific_modules": False, "multi_subpath_exports": False}
|
|
4986
|
+
|
|
4987
|
+
|
|
4988
|
+
class DenominatorInputError(Exception):
|
|
4989
|
+
"""Exit 2 of denominator mode."""
|
|
4990
|
+
|
|
4991
|
+
|
|
4992
|
+
def _name_family(language: object, file: object) -> str:
|
|
4993
|
+
"""The language family a name counts in: its file's, else its language's."""
|
|
4994
|
+
family = FAMILY_OF.get(_file_language(file)) if isinstance(file, str) and file else None
|
|
4995
|
+
if family is not None:
|
|
4996
|
+
return family
|
|
4997
|
+
value = str(language or "").strip().lower()
|
|
4998
|
+
selected = LANGUAGE_FAMILIES.get(value)
|
|
4999
|
+
return FAMILY_OF[selected[0]] if selected else value
|
|
5000
|
+
|
|
5001
|
+
|
|
5002
|
+
def _payload_names(items: object, where: str) -> list[dict]:
|
|
5003
|
+
"""The {name, file, language, members} items of a payload list, checked."""
|
|
5004
|
+
if items is None:
|
|
5005
|
+
return []
|
|
5006
|
+
if not isinstance(items, list):
|
|
5007
|
+
raise DenominatorInputError(f"{where} is not a list")
|
|
5008
|
+
names = []
|
|
5009
|
+
for i, item in enumerate(items):
|
|
5010
|
+
if not isinstance(item, dict) or not isinstance(item.get("name"), str) or not item["name"].strip():
|
|
5011
|
+
raise DenominatorInputError(f"{where}[{i}] is not an object with a name")
|
|
5012
|
+
if item.get("file") is not None and not isinstance(item["file"], str):
|
|
5013
|
+
raise DenominatorInputError(f"{where}[{i}].file is not a path")
|
|
5014
|
+
members = item.get("members")
|
|
5015
|
+
if members is not None:
|
|
5016
|
+
if not isinstance(members, list):
|
|
5017
|
+
raise DenominatorInputError(f"{where}[{i}].members is not a list")
|
|
5018
|
+
members = _payload_names(members, f"{where}[{i}].members")
|
|
5019
|
+
file = _norm(item["file"]) if item.get("file") else None
|
|
5020
|
+
names.append({"name": item["name"].strip(), "file": file, "language": item.get("language"),
|
|
5021
|
+
"members": members})
|
|
5022
|
+
return names
|
|
5023
|
+
|
|
5024
|
+
|
|
5025
|
+
def _read_json_file(path: str, flag: str) -> dict:
|
|
5026
|
+
try:
|
|
5027
|
+
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
5028
|
+
except (OSError, UnicodeDecodeError, ValueError) as exc:
|
|
5029
|
+
raise DenominatorInputError(f"cannot read {flag} {path}: {exc}") from exc
|
|
5030
|
+
if not isinstance(data, dict):
|
|
5031
|
+
raise DenominatorInputError(f"{flag} {path} is not a JSON object")
|
|
5032
|
+
return data
|
|
5033
|
+
|
|
5034
|
+
|
|
5035
|
+
def run_denominator(args: argparse.Namespace, payload: dict) -> dict:
|
|
5036
|
+
"""The counts of the public names `payload`, --quick and --extraction
|
|
5037
|
+
give, with the effective_denominator decision full mode makes."""
|
|
5038
|
+
public: dict[tuple[str, str], dict] = {}
|
|
5039
|
+
runner_public: dict[tuple[str, str], dict] = {}
|
|
5040
|
+
# The exports found, each key at its file: the payload's and the runner's.
|
|
5041
|
+
found: dict[tuple[str, str], str | None] = {}
|
|
5042
|
+
internal: set[tuple[str, str]] = set()
|
|
5043
|
+
# The runner's other in-scope definitions of its public names (#830), counted where the names are.
|
|
5044
|
+
others: list[tuple[str, dict]] = []
|
|
5045
|
+
incomplete = False
|
|
5046
|
+
runner_status = None
|
|
5047
|
+
if args.extraction:
|
|
5048
|
+
runner = _read_json_file(args.extraction, "--extraction")
|
|
5049
|
+
if runner.get("mode") != "full":
|
|
5050
|
+
raise DenominatorInputError(f"--extraction {args.extraction} is not --mode full output")
|
|
5051
|
+
runner_status = runner.get("status")
|
|
5052
|
+
incomplete = runner_status == "incomplete" or bool(runner.get("errors"))
|
|
5053
|
+
diff = runner.get("entry_point_diff") if isinstance(runner.get("entry_point_diff"), dict) else {}
|
|
5054
|
+
for info in diff.get("public") or []:
|
|
5055
|
+
if isinstance(info, dict) and isinstance(info.get("name"), str):
|
|
5056
|
+
key = (str(info.get("language") or ""), info["name"])
|
|
5057
|
+
public[key] = runner_public[key] = info
|
|
5058
|
+
found[key] = info.get("file")
|
|
5059
|
+
others = [(str(info.get("language") or ""), info) for info in diff.get("other_definitions") or []
|
|
5060
|
+
if isinstance(info, dict)]
|
|
5061
|
+
for item in diff.get("internal") or []:
|
|
5062
|
+
if isinstance(item, dict) and isinstance(item.get("name"), str):
|
|
5063
|
+
internal.add((str(item.get("language") or ""), item["name"]))
|
|
5064
|
+
found[(str(item.get("language") or ""), item["name"])] = item.get("source_file")
|
|
5065
|
+
for path in args.quick or []:
|
|
5066
|
+
quick = _read_json_file(path, "--quick")
|
|
5067
|
+
if not isinstance(quick.get("exports"), list):
|
|
5068
|
+
raise DenominatorInputError(f"--quick {path} is not --mode quick output (no exports list)")
|
|
5069
|
+
for item in quick["exports"]:
|
|
5070
|
+
if isinstance(item, dict) and isinstance(item.get("name"), str) and item["name"]:
|
|
5071
|
+
file = item.get("defined_in") or item.get("source_file")
|
|
5072
|
+
file = _norm(file) if isinstance(file, str) and file else None
|
|
5073
|
+
public[(_name_family(quick.get("language"), file), item["name"])] = {
|
|
5074
|
+
"name": item["name"], "file": file, "via": "declaration"}
|
|
5075
|
+
for item in _payload_names(payload.get("public"), "public"):
|
|
5076
|
+
family = _name_family(item["language"], item["file"])
|
|
5077
|
+
info = {"name": item["name"], "file": item["file"], "via": "declaration"}
|
|
5078
|
+
if item["members"] is not None:
|
|
5079
|
+
info.update(via="namespace", members=[{"name": m["name"], "file": m["file"]} for m in item["members"]])
|
|
5080
|
+
public[(family, item["name"])] = info
|
|
5081
|
+
for item in _payload_names(payload.get("exports"), "exports"):
|
|
5082
|
+
key = (_name_family(item["language"], item["file"]), item["name"])
|
|
5083
|
+
found.setdefault(key, item["file"])
|
|
5084
|
+
if not _private(key[0], item["name"]):
|
|
5085
|
+
internal.add(key)
|
|
5086
|
+
named = set(public) | {(family, m["name"]) for (family, _), info in public.items() if _has_members(info)
|
|
5087
|
+
for m in info["members"]}
|
|
5088
|
+
public_keys = _public_keys(public)
|
|
5089
|
+
def still_held(key: tuple[str, str]) -> bool:
|
|
5090
|
+
"""Whether the record kept for the runner's public name `key` still names the runner row's file, a
|
|
5091
|
+
namespace with members in neither (full mode keeps no other definition beside one)."""
|
|
5092
|
+
runner_row, kept = runner_public.get(key), public.get(key)
|
|
5093
|
+
if runner_row is None or kept is None or _has_members(runner_row) or _has_members(kept):
|
|
5094
|
+
return False
|
|
5095
|
+
files = [_norm(row["file"]) if isinstance(row.get("file"), str) and row["file"] else None
|
|
5096
|
+
for row in (runner_row, kept)]
|
|
5097
|
+
return files[0] == files[1]
|
|
5098
|
+
|
|
5099
|
+
# A runner row counts only while the record kept for its name names the runner's file: an orphan row, or
|
|
5100
|
+
# one of a name a payload or --quick record moved to another file, adds nothing.
|
|
5101
|
+
others = [(family, info) for family, info in others
|
|
5102
|
+
if isinstance(info.get("name"), str) and still_held((family, info["name"]))]
|
|
5103
|
+
other_keys = _other_keys(others)
|
|
5104
|
+
counted = public_keys | other_keys
|
|
5105
|
+
|
|
5106
|
+
if args.source_root:
|
|
5107
|
+
root = Path(args.source_root)
|
|
5108
|
+
if not root.is_dir():
|
|
5109
|
+
raise RunnerError(f"source root not found: {args.source_root}")
|
|
5110
|
+
facts = _scope_facts(args, root)
|
|
5111
|
+
else:
|
|
5112
|
+
# No local tree (a remote source read by eye): the repository's listing, else the names' files.
|
|
5113
|
+
facts = _brief_scope(args)
|
|
5114
|
+
in_scope = without_recipes = None
|
|
5115
|
+
unmatched_include: list[str] = []
|
|
5116
|
+
unmatched_tier_a: list[str] = []
|
|
5117
|
+
if args.tree_file is not None:
|
|
5118
|
+
try:
|
|
5119
|
+
listed, cut = _read_listing(args.tree_file)
|
|
5120
|
+
except QuickInputError as exc:
|
|
5121
|
+
raise DenominatorInputError(str(exc)) from exc
|
|
5122
|
+
tree = sorted(set(listed))
|
|
5123
|
+
if cut:
|
|
5124
|
+
facts["warnings"].append("the listing was cut short: the file lists can miss files")
|
|
5125
|
+
in_scope, without_recipes, unmatched_include = _select_files(
|
|
5126
|
+
Path("."), tree, False, facts["includes"], facts["excludes"], facts["wanted"], facts["glob_match"],
|
|
5127
|
+
[], facts["warnings"])
|
|
5128
|
+
unmatched_tier_a = _unmatched_tier_a(facts["tier_a"], facts["includes"], tree, facts["glob_match"],
|
|
5129
|
+
facts["warnings"])
|
|
5130
|
+
else:
|
|
5131
|
+
tree = sorted({file for _, _, file in counted if file})
|
|
5132
|
+
facts.update(tree=tree, in_scope=in_scope, without_recipes=without_recipes,
|
|
5133
|
+
unmatched_include=unmatched_include, unmatched_tier_a=unmatched_tier_a, arms=dict(NO_ARMS),
|
|
5134
|
+
workspace_sources=[], entries=[], maps=[])
|
|
5135
|
+
warnings = facts["warnings"]
|
|
5136
|
+
tree_set = set(facts["tree"])
|
|
5137
|
+
outside_tree = sorted((name, file) for _, name, file in counted if file and file not in tree_set)
|
|
5138
|
+
warnings += [f"public name {name} is defined in {file}, which is not in the tree: effective_denominator "
|
|
5139
|
+
"and curated_subset leave it out" for name, file in outside_tree]
|
|
5140
|
+
counts = _denominator_counts(facts, counted, len(public_keys), len(internal - named), incomplete, warnings)
|
|
5141
|
+
if not args.source_root and counts["effective_denominator_decision"]["reason"] not in ("reference-app",
|
|
5142
|
+
"incomplete-run"):
|
|
5143
|
+
counts["effective_denominator_decision"] = {"store": False, "reason": "no-source-tree"}
|
|
5144
|
+
without_recipes = facts["without_recipes"]
|
|
5145
|
+
|
|
5146
|
+
def row(key: tuple[str, str], file: str | None) -> dict:
|
|
5147
|
+
return {"name": key[1], "language": key[0], "file": file}
|
|
5148
|
+
|
|
5149
|
+
def by_name(keys):
|
|
5150
|
+
return sorted(keys, key=lambda k: (k[1], k[0]))
|
|
5151
|
+
|
|
5152
|
+
# The names the payload and --quick gave (the runner diffed its own), a namespace as its members.
|
|
5153
|
+
given: dict[tuple[str, str], str | None] = {}
|
|
5154
|
+
for key, info in public.items():
|
|
5155
|
+
if info is runner_public.get(key):
|
|
5156
|
+
continue
|
|
5157
|
+
for member in info["members"] if _has_members(info) else [info]:
|
|
5158
|
+
given.setdefault((key[0], member["name"]), member["file"])
|
|
5159
|
+
return {
|
|
5160
|
+
"mode": "denominator",
|
|
5161
|
+
"source_root": args.source_root,
|
|
5162
|
+
"scope": _scope_record(args, facts),
|
|
5163
|
+
"files_in_scope": None if facts["in_scope"] is None else len(facts["in_scope"]),
|
|
5164
|
+
"files_without_recipes": None if without_recipes is None else _by_extension(without_recipes),
|
|
5165
|
+
"entry_points": {"files": facts["entries"], "exports_maps": facts["maps"]},
|
|
5166
|
+
"arms": facts["arms"],
|
|
5167
|
+
"runner_status": runner_status,
|
|
5168
|
+
"counts": counts,
|
|
5169
|
+
"entry_point_diff": {
|
|
5170
|
+
"public": [row((family, name), file)
|
|
5171
|
+
for family, name, file in sorted(public_keys, key=lambda k: (k[1], k[0], k[2] or ""))],
|
|
5172
|
+
"other_definitions": [row((family, name), file)
|
|
5173
|
+
for family, name, file in sorted(other_keys, key=lambda k: (k[1], k[2], k[0]))],
|
|
5174
|
+
"internal": [row(key, found.get(key)) for key in by_name(internal - named)],
|
|
5175
|
+
"extraction_gaps": [row(key, given[key]) for key in by_name(given) if key not in found],
|
|
5176
|
+
},
|
|
5177
|
+
"warnings": warnings,
|
|
5178
|
+
"files_in_scope_list": facts["in_scope"],
|
|
5179
|
+
"files_without_recipes_list": without_recipes,
|
|
5180
|
+
}
|
|
5181
|
+
|
|
5182
|
+
|
|
5183
|
+
def _main_denominator(args: argparse.Namespace) -> int:
|
|
5184
|
+
if args.source_root and args.tree_file is not None:
|
|
5185
|
+
return _full_exit_2("--tree-file stands for a remote source with no --source-root: pass one of them")
|
|
5186
|
+
# With `--tree-file -` stdin holds the listing, so there is no payload.
|
|
5187
|
+
reads_stdin = args.tree_file != "-" and sys.stdin is not None and not sys.stdin.isatty()
|
|
5188
|
+
raw = sys.stdin.read() if reads_stdin else ""
|
|
5189
|
+
try:
|
|
5190
|
+
payload = json.loads(raw) if raw.strip() else {}
|
|
5191
|
+
except ValueError as exc:
|
|
5192
|
+
return _full_exit_2(f"stdin is not JSON: {exc}")
|
|
5193
|
+
if not isinstance(payload, dict):
|
|
5194
|
+
return _full_exit_2("stdin must be a JSON object: {\"public\": [...], \"exports\": [...]}")
|
|
5195
|
+
try:
|
|
5196
|
+
result = run_denominator(args, payload)
|
|
5197
|
+
except (RunnerError, DenominatorInputError) as exc:
|
|
5198
|
+
return _full_exit_2(str(exc))
|
|
5199
|
+
except Exception as exc: # noqa: BLE001 - one stderr line and exit 2, never a traceback
|
|
5200
|
+
return _full_exit_2(f"the run failed: {type(exc).__name__}: {exc}")
|
|
5201
|
+
text = json.dumps(result, indent=2) + "\n"
|
|
5202
|
+
if args.output:
|
|
5203
|
+
try:
|
|
5204
|
+
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
|
5205
|
+
Path(args.output).write_text(text, encoding="utf-8")
|
|
5206
|
+
except OSError as exc:
|
|
5207
|
+
return _full_exit_2(f"cannot write {args.output}: {exc}")
|
|
5208
|
+
else:
|
|
5209
|
+
sys.stdout.write(text)
|
|
5210
|
+
return 0
|
|
5211
|
+
|
|
5212
|
+
|
|
4092
5213
|
# --------------------------------------------------------------------------
|
|
4093
5214
|
# Entries mode: the files quick mode reads, found in the listing
|
|
4094
5215
|
# --------------------------------------------------------------------------
|
|
@@ -4281,10 +5402,11 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
4281
5402
|
parser.add_argument(
|
|
4282
5403
|
"--mode",
|
|
4283
5404
|
default="quick",
|
|
4284
|
-
choices=("quick", "full", "entries"),
|
|
5405
|
+
choices=("quick", "full", "entries", "denominator"),
|
|
4285
5406
|
help="quick (default): parse the stdin payload, or the --manifest-file and --entry-file files; "
|
|
4286
5407
|
"full: run the recipes over --source-root; entries: list one package's entry files from "
|
|
4287
|
-
"--tree-file
|
|
5408
|
+
"--tree-file; denominator: count the public names a stdin payload, --quick and --extraction give, "
|
|
5409
|
+
"and decide whether to store effective_denominator, as full mode does.",
|
|
4288
5410
|
)
|
|
4289
5411
|
quick = parser.add_argument_group("quick mode, files read from disk")
|
|
4290
5412
|
quick.add_argument("--manifest-file", metavar="PATH",
|
|
@@ -4298,16 +5420,23 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
4298
5420
|
help=f"read each module an unlisted[] record names as one more --follow-file, round "
|
|
4299
5421
|
f"after round (at most {FOLLOW_ROUNDS}), until none is left; needs --tree-file and "
|
|
4300
5422
|
f"--source-root")
|
|
5423
|
+
quick.add_argument("--resolve-version", action="store_true",
|
|
5424
|
+
help="look up a version the manifest does not state: a version file, a Cargo workspace "
|
|
5425
|
+
"root, then a release tag (of --repo at --ref, else git describe in --source-root); "
|
|
5426
|
+
"needs --source-root")
|
|
4301
5427
|
quick.add_argument("--repo", metavar="OWNER/REPO",
|
|
4302
5428
|
help="with --follow: fetch each module from this GitHub repository into --source-root "
|
|
4303
|
-
"first (skf-github-fetch.py)"
|
|
5429
|
+
"first (skf-github-fetch.py); with --resolve-version: the repository whose tags "
|
|
5430
|
+
"it reads, and whose files it fetches with --tree-file")
|
|
4304
5431
|
quick.add_argument("--ref", metavar="REF",
|
|
4305
|
-
help="with --repo: the branch, tag or commit to fetch
|
|
5432
|
+
help="with --repo: the branch, tag or commit to fetch and read the version at "
|
|
5433
|
+
"(default: the default branch)")
|
|
4306
5434
|
listing = parser.add_argument_group("entries mode, and the listing quick mode follows chains in")
|
|
4307
5435
|
listing.add_argument("--tree-file", metavar="PATH",
|
|
4308
5436
|
help="the repository's file listing (skf-github-probe.py tree output, JSON, or one "
|
|
4309
|
-
"path per line): entries mode finds the entry files in it,
|
|
4310
|
-
"resolves each unlisted[] statement's module_file against it"
|
|
5437
|
+
"path per line): entries mode finds the entry files in it, quick mode "
|
|
5438
|
+
"resolves each unlisted[] statement's module_file against it, and denominator "
|
|
5439
|
+
"mode, with no --source-root, takes it as the tree (- reads it from stdin)")
|
|
4311
5440
|
listing.add_argument("--scope", metavar="FOLDER",
|
|
4312
5441
|
help="entries mode: the package folder to read (default: the repository root)")
|
|
4313
5442
|
listing.add_argument("--fetch-list", metavar="FILE",
|
|
@@ -4322,7 +5451,11 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
4322
5451
|
full.add_argument("--exclude", action="append", metavar="GLOB",
|
|
4323
5452
|
help="an exclude glob, repeatable (replaces the brief's scope.exclude)")
|
|
4324
5453
|
full.add_argument("--tier-a-include", action="append", metavar="GLOB",
|
|
4325
|
-
help="a tier-A glob for effective_denominator, repeatable (replaces scope.tier_a_include)"
|
|
5454
|
+
help="a tier-A glob for effective_denominator, repeatable (replaces scope.tier_a_include), "
|
|
5455
|
+
"that counts a name only in a file scope.include matches too; one that matches no "
|
|
5456
|
+
"file is listed in scope.unmatched_tier_a_include, one that matches only files "
|
|
5457
|
+
"scope.include leaves out is named in warnings, and a list that counts no public name "
|
|
5458
|
+
"gives way to scope.include (else every file)")
|
|
4326
5459
|
full.add_argument("--scope-type", help="the scope type (replaces the brief's scope.type)")
|
|
4327
5460
|
full.add_argument("--language", action="append",
|
|
4328
5461
|
help="a language family to extract, repeatable (replaces the brief's language); quick "
|
|
@@ -4339,6 +5472,14 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
4339
5472
|
help=f"stop the ast-grep runs after this many seconds and still write the JSON, "
|
|
4340
5473
|
f"status incomplete (default {DEFAULT_TIMEOUT_SEC})")
|
|
4341
5474
|
full.add_argument("-o", "--output", metavar="FILE", help="write the JSON here instead of stdout")
|
|
5475
|
+
denominator = parser.add_argument_group("denominator mode (with --source-root, --brief, the scope flags, "
|
|
5476
|
+
"--language and -o)")
|
|
5477
|
+
denominator.add_argument("--quick", action="append", metavar="FILE",
|
|
5478
|
+
help="a --mode quick output whose exports are public names, each at its defined_in "
|
|
5479
|
+
"(else its source_file), repeatable")
|
|
5480
|
+
denominator.add_argument("--extraction", metavar="FILE",
|
|
5481
|
+
help="a --mode full output whose entry_point_diff public and internal names count "
|
|
5482
|
+
"too; an incomplete one is never stored")
|
|
4342
5483
|
return parser
|
|
4343
5484
|
|
|
4344
5485
|
|
|
@@ -4346,10 +5487,14 @@ FULL_MODE_FLAGS = ("source_root", "brief", "include", "exclude", "tier_a_include
|
|
|
4346
5487
|
"files_from", "recipe_set", "recipes", "tier", "head_cap", "timeout", "output")
|
|
4347
5488
|
# The full mode flags quick mode takes with its file inputs, and entries mode always.
|
|
4348
5489
|
QUICK_FILE_FLAGS = ("source_root", "language")
|
|
5490
|
+
# The full mode flags denominator mode takes, and its own.
|
|
5491
|
+
DENOMINATOR_FLAGS = ("source_root", "brief", "include", "exclude", "tier_a_include", "scope_type", "language",
|
|
5492
|
+
"output")
|
|
5493
|
+
DENOMINATOR_OWN_FLAGS = ("quick", "extraction")
|
|
4349
5494
|
# The flags of entries mode; quick mode takes --tree-file and --fetch-list with its file inputs.
|
|
4350
5495
|
LISTING_FLAGS = ("tree_file", "scope", "fetch_list")
|
|
4351
5496
|
# Quick mode's own flags with its file inputs: the chains it follows itself.
|
|
4352
|
-
FOLLOW_FLAGS = ("follow", "repo", "ref")
|
|
5497
|
+
FOLLOW_FLAGS = ("follow", "repo", "ref", "resolve_version")
|
|
4353
5498
|
FOLLOW_ROUNDS = 5
|
|
4354
5499
|
|
|
4355
5500
|
|
|
@@ -4479,6 +5624,327 @@ def _follow(args: argparse.Namespace, result: dict) -> dict:
|
|
|
4479
5624
|
return result
|
|
4480
5625
|
|
|
4481
5626
|
|
|
5627
|
+
# --resolve-version: a version the manifest does not state
|
|
5628
|
+
_PY_VERSION_RE = re.compile(r"""^__version__[ \t]*(?::[^=\n]*)?=[ \t]*(['"])([^'"\n]+)\1""", re.M)
|
|
5629
|
+
_JS_PRERELEASE_RE = re.compile(r"-(?:alpha|beta|rc|next|canary)", re.I)
|
|
5630
|
+
JS_FAMILY = ("js", "ts", "javascript", "typescript")
|
|
5631
|
+
|
|
5632
|
+
|
|
5633
|
+
def _version_reader(args: argparse.Namespace, tree: list[str] | None) -> Callable[[str], str | None]:
|
|
5634
|
+
"""path -> the text of that file under --source-root, else None. With
|
|
5635
|
+
--repo and --tree-file, a file the listing holds and the folder does not
|
|
5636
|
+
is fetched there first, as --follow fetches a module. Raises
|
|
5637
|
+
QuickInputError."""
|
|
5638
|
+
root = Path(args.source_root)
|
|
5639
|
+
listed = set(tree or [])
|
|
5640
|
+
fetch: list = []
|
|
5641
|
+
|
|
5642
|
+
def read(path: str) -> str | None:
|
|
5643
|
+
target = root / path
|
|
5644
|
+
if not target.is_file() and args.repo is not None and args.tree_file is not None and path in listed:
|
|
5645
|
+
if not fetch:
|
|
5646
|
+
fetch.append(_follow_reader(args))
|
|
5647
|
+
fetch[0]([path])
|
|
5648
|
+
try:
|
|
5649
|
+
if target.is_file() and target.resolve().is_relative_to(root.resolve()):
|
|
5650
|
+
return target.read_bytes().decode("utf-8-sig", errors="replace")
|
|
5651
|
+
except OSError:
|
|
5652
|
+
pass
|
|
5653
|
+
return None
|
|
5654
|
+
return read
|
|
5655
|
+
|
|
5656
|
+
|
|
5657
|
+
def _version_value(value: object) -> str | None:
|
|
5658
|
+
"""A version a version file writes: a literal that starts with a digit, not a placeholder."""
|
|
5659
|
+
if not isinstance(value, str) or not re.match(r"\d", value.strip()):
|
|
5660
|
+
return None
|
|
5661
|
+
return None if _detect_placeholder_version(value) else value.strip()
|
|
5662
|
+
|
|
5663
|
+
|
|
5664
|
+
def _python_version_file(manifest: str | None, entries: list[str],
|
|
5665
|
+
read: Callable[[str], str | None]) -> tuple[str, str] | None:
|
|
5666
|
+
"""(version, path): the `__version__` literal of an entry __init__.py,
|
|
5667
|
+
then of the _version.py beside it, then setup.py's `version=` literal
|
|
5668
|
+
beside the manifest."""
|
|
5669
|
+
inits = [e for e in entries if e.rsplit("/", 1)[-1] == "__init__.py"]
|
|
5670
|
+
folder = posixpath.dirname(manifest or "")
|
|
5671
|
+
candidates = [(path, "py") for path in inits]
|
|
5672
|
+
candidates += [(posixpath.join(posixpath.dirname(path), "_version.py"), "py") for path in inits]
|
|
5673
|
+
if posixpath.basename(manifest or "") != "setup.py":
|
|
5674
|
+
candidates.append((posixpath.join(folder, "setup.py"), "setup.py"))
|
|
5675
|
+
for path, kind in dict.fromkeys(candidates):
|
|
5676
|
+
text = read(path)
|
|
5677
|
+
if text is None:
|
|
5678
|
+
continue
|
|
5679
|
+
if kind == "py":
|
|
5680
|
+
m = _PY_VERSION_RE.search(text)
|
|
5681
|
+
version = _version_value(m.group(2)) if m else None
|
|
5682
|
+
else:
|
|
5683
|
+
version = _version_value(parse_setup_py(text).get("version"))
|
|
5684
|
+
if version:
|
|
5685
|
+
return version, path
|
|
5686
|
+
return None
|
|
5687
|
+
|
|
5688
|
+
|
|
5689
|
+
def _cargo_workspace_version(manifest: str | None, read: Callable[[str], str | None]) -> tuple[str, str] | None:
|
|
5690
|
+
"""(version, path): the `[workspace.package] version` of the workspace
|
|
5691
|
+
root, the nearest Cargo.toml at or above the manifest's folder that
|
|
5692
|
+
holds a `[workspace]` table."""
|
|
5693
|
+
if not manifest:
|
|
5694
|
+
return None
|
|
5695
|
+
folder = posixpath.dirname(manifest)
|
|
5696
|
+
while True:
|
|
5697
|
+
path = posixpath.join(folder, "Cargo.toml")
|
|
5698
|
+
text = read(path)
|
|
5699
|
+
try:
|
|
5700
|
+
data = tomllib.loads(text) if text is not None else None
|
|
5701
|
+
except tomllib.TOMLDecodeError:
|
|
5702
|
+
data = None
|
|
5703
|
+
workspace = data.get("workspace") if isinstance(data, dict) else None
|
|
5704
|
+
if isinstance(workspace, dict):
|
|
5705
|
+
package = workspace.get("package")
|
|
5706
|
+
version = _version_value(package.get("version")) if isinstance(package, dict) else None
|
|
5707
|
+
return (version, path) if version else None
|
|
5708
|
+
if not folder:
|
|
5709
|
+
return None
|
|
5710
|
+
folder = posixpath.dirname(folder)
|
|
5711
|
+
|
|
5712
|
+
|
|
5713
|
+
def _release_tag(args: argparse.Namespace, warnings: list[str]) -> tuple[str, str] | None:
|
|
5714
|
+
"""(version, tag) of the tag a version is read from: with --repo, the
|
|
5715
|
+
`release` skf-github-probe.py's tag listing gives at --ref (the newest
|
|
5716
|
+
release tag at the default branch, the pinned tag itself at a tag);
|
|
5717
|
+
without, the tag `git describe --tags --abbrev=0` names in
|
|
5718
|
+
--source-root, when its name is a version. The version drops a
|
|
5719
|
+
leading v."""
|
|
5720
|
+
try:
|
|
5721
|
+
probe = _sibling("skf-github-probe.py")
|
|
5722
|
+
except RunnerError as exc:
|
|
5723
|
+
warnings.append(f"--resolve-version could not read tags: {exc}")
|
|
5724
|
+
return None
|
|
5725
|
+
if args.repo is not None:
|
|
5726
|
+
parsed = probe.parse_repo(args.repo)
|
|
5727
|
+
if parsed is None:
|
|
5728
|
+
warnings.append(f"--resolve-version read no tags: {args.repo} is not a github.com repository")
|
|
5729
|
+
return None
|
|
5730
|
+
probe._start_clock(probe.DEFAULT_TIMEOUT_SEC)
|
|
5731
|
+
out = probe.probe_tags(*parsed, [], 1, ref=args.ref)
|
|
5732
|
+
if out["status"] != "ok":
|
|
5733
|
+
warnings.append(f"the tags of {args.repo} could not be listed ({out['message']}), so no release "
|
|
5734
|
+
f"tag gave a version")
|
|
5735
|
+
return None
|
|
5736
|
+
release = out["release"]
|
|
5737
|
+
return (release["version"], release["tag"]) if release else None
|
|
5738
|
+
try:
|
|
5739
|
+
res = _sibling("skf-source-tree.py")._git(Path(args.source_root), "describe", "--tags", "--abbrev=0")
|
|
5740
|
+
except RunnerError:
|
|
5741
|
+
return None
|
|
5742
|
+
if res is None or res[0] != 0:
|
|
5743
|
+
return None
|
|
5744
|
+
tag = res[1].decode("utf-8", errors="replace").strip()
|
|
5745
|
+
version = probe.tag_version(tag)
|
|
5746
|
+
return (version, tag) if version else None
|
|
5747
|
+
|
|
5748
|
+
|
|
5749
|
+
def resolve_version(args: argparse.Namespace, result: dict, tree: list[str] | None) -> None:
|
|
5750
|
+
"""--resolve-version: when the manifest states no version, look it up
|
|
5751
|
+
(Python: _python_version_file; Rust: _cargo_workspace_version; then any
|
|
5752
|
+
language: _release_tag), and for JavaScript or TypeScript read from
|
|
5753
|
+
--repo at the default branch, let the newest release replace a manifest
|
|
5754
|
+
pre-release (-alpha, -beta, -rc, -next, -canary). Sets `version` and
|
|
5755
|
+
`version_source`: "manifest", "file:<path>", "tag:<tag>", or null with
|
|
5756
|
+
a warning when nothing gave one. Raises QuickInputError."""
|
|
5757
|
+
warnings = result["warnings"]
|
|
5758
|
+
version = result["version"]
|
|
5759
|
+
source = "manifest" if version is not None else None
|
|
5760
|
+
manifest = Path(args.manifest_file).as_posix() if args.manifest_file is not None else None
|
|
5761
|
+
if version is None:
|
|
5762
|
+
read = _version_reader(args, tree)
|
|
5763
|
+
found = None
|
|
5764
|
+
if result["language"] == "python":
|
|
5765
|
+
found = _python_version_file(manifest, [Path(e).as_posix() for e in args.entry_file or []], read)
|
|
5766
|
+
elif result["language"] == "rust":
|
|
5767
|
+
found = _cargo_workspace_version(manifest, read)
|
|
5768
|
+
if found:
|
|
5769
|
+
version, source = found[0], f"file:{found[1]}"
|
|
5770
|
+
prerelease = (version is not None and source == "manifest" and result["language"] in JS_FAMILY
|
|
5771
|
+
and args.repo is not None and (args.ref or "").strip().lower() in ("", "null", "none", "head")
|
|
5772
|
+
and _JS_PRERELEASE_RE.search(version) is not None)
|
|
5773
|
+
if version is None or prerelease:
|
|
5774
|
+
tag = _release_tag(args, warnings)
|
|
5775
|
+
if tag and prerelease:
|
|
5776
|
+
warnings.append(f"manifest version {version!r} is a pre-release on the default branch; version is "
|
|
5777
|
+
f"the newest release, {tag[0]} (tag {tag[1]})")
|
|
5778
|
+
if tag:
|
|
5779
|
+
version, source = tag[0], f"tag:{tag[1]}"
|
|
5780
|
+
if version is None:
|
|
5781
|
+
warnings.append("no version found in the manifest, a version file or a release tag; version is null")
|
|
5782
|
+
result["version"] = version
|
|
5783
|
+
result["version_source"] = source
|
|
5784
|
+
|
|
5785
|
+
|
|
5786
|
+
_JS_LIST_SOURCE_RE = re.compile(r"\s*from\s*(['\"])")
|
|
5787
|
+
_JS_IMPORT_RE = re.compile(r"^import\s+(?:type\s+)?([A-Za-z_$*{][^;'\"]*?)\s*from\s*(['\"])", re.M)
|
|
5788
|
+
_JS_BRACES_RE = re.compile(r"\{([^}]*)\}")
|
|
5789
|
+
DEFINED_IN_FAMILIES = {"python": "python", "js": "js", "ts": "js", "javascript": "js", "typescript": "js"}
|
|
5790
|
+
|
|
5791
|
+
|
|
5792
|
+
def _quick_js_facts(content: str) -> dict:
|
|
5793
|
+
"""What one JS/TS module exports, read from its text as quick mode reads
|
|
5794
|
+
an entry file, in _js_facts's shape: {"names": {exposed name: {"how",
|
|
5795
|
+
"local", "source"}}, "stars": [source], "default": the local name of
|
|
5796
|
+
its default export, or None}. A declaration beats a list item, and an
|
|
5797
|
+
exported binding an import brings in is that import's re-export (or
|
|
5798
|
+
namespace, for `import * as`)."""
|
|
5799
|
+
code = _blank(content, "js")
|
|
5800
|
+
imports: dict[str, tuple[str, str]] = {}
|
|
5801
|
+
for m in _JS_IMPORT_RE.finditer(code):
|
|
5802
|
+
source = _js_string(content, code, m.end(2) - 1)
|
|
5803
|
+
braces = _JS_BRACES_RE.search(m.group(1))
|
|
5804
|
+
for piece in braces.group(1).split(",") if braces else []:
|
|
5805
|
+
words = piece.split()
|
|
5806
|
+
if words and _JS_NAME_RE.fullmatch(words[-1]):
|
|
5807
|
+
imports.setdefault(words[-1], (_specifier_local(piece), source))
|
|
5808
|
+
for part in _JS_BRACES_RE.sub("", m.group(1)).split(","):
|
|
5809
|
+
words = part.split()
|
|
5810
|
+
if len(words) == 3 and words[:2] == ["*", "as"]:
|
|
5811
|
+
imports.setdefault(words[2], ("*", source))
|
|
5812
|
+
elif len(words) == 1 and _JS_NAME_RE.fullmatch(words[0]):
|
|
5813
|
+
imports.setdefault(words[0], ("default", source))
|
|
5814
|
+
names: dict[str, dict] = {}
|
|
5815
|
+
|
|
5816
|
+
def put(name: str, how: str, local: str | None, source: str | None) -> None:
|
|
5817
|
+
names.setdefault(name, {"how": how, "local": local, "source": source})
|
|
5818
|
+
|
|
5819
|
+
def bound(name: str, local: str) -> None:
|
|
5820
|
+
imported, source = imports.get(local, (None, None))
|
|
5821
|
+
if imported == "*":
|
|
5822
|
+
put(name, "namespace", None, source)
|
|
5823
|
+
elif imported is not None:
|
|
5824
|
+
put(name, "re-export", imported, source)
|
|
5825
|
+
else:
|
|
5826
|
+
put(name, "declaration", local, None)
|
|
5827
|
+
|
|
5828
|
+
for export in scan_exports_js(content, "", None, None, defaults=False):
|
|
5829
|
+
if export["type"] not in ("re-export", "namespace"):
|
|
5830
|
+
put(export["name"], "declaration", export["name"], None)
|
|
5831
|
+
default = None
|
|
5832
|
+
for m in _JS_DECL_RE.finditer(code):
|
|
5833
|
+
if m.group(1) and "default" in m.group(1).split():
|
|
5834
|
+
default = default or m.group(3)
|
|
5835
|
+
put("default", "declaration", m.group(3), None)
|
|
5836
|
+
elif m.group(1) and _js_type(m.group(2)) == "namespace":
|
|
5837
|
+
put(m.group(3), "declaration", m.group(3), None) # `export namespace X`, `export module X`
|
|
5838
|
+
stars: list[str] = []
|
|
5839
|
+
for m in _JS_STAR_RE.finditer(code):
|
|
5840
|
+
source = _js_string(content, code, m.end(2) - 1)
|
|
5841
|
+
if m.group(1):
|
|
5842
|
+
put(m.group(1), "namespace", None, source)
|
|
5843
|
+
else:
|
|
5844
|
+
stars.append(source)
|
|
5845
|
+
for m in _JS_LIST_RE.finditer(code):
|
|
5846
|
+
after = _JS_LIST_SOURCE_RE.match(code, m.end())
|
|
5847
|
+
source = _js_string(content, code, after.end(1) - 1) if after else None
|
|
5848
|
+
for piece in m.group(1).split(","):
|
|
5849
|
+
words = piece.split()
|
|
5850
|
+
if not words or not _JS_NAME_RE.fullmatch(words[-1]):
|
|
5851
|
+
continue
|
|
5852
|
+
if source is not None:
|
|
5853
|
+
put(words[-1], "re-export", _specifier_local(piece), source)
|
|
5854
|
+
else:
|
|
5855
|
+
bound(words[-1], _specifier_local(piece))
|
|
5856
|
+
for m in _JS_DEFAULT_RE.finditer(code):
|
|
5857
|
+
if m.group(1) not in _JS_NOT_NAMES:
|
|
5858
|
+
default = default or m.group(1)
|
|
5859
|
+
bound("default", m.group(1))
|
|
5860
|
+
return {"names": names, "stars": stars, "default": default}
|
|
5861
|
+
|
|
5862
|
+
|
|
5863
|
+
def _js_defined_in(root: Path, tree: set[str]) -> Callable[[str, str], str | None]:
|
|
5864
|
+
"""(module, name it exports) -> the file that defines that name, as
|
|
5865
|
+
_js_surface's resolve traces it: through each re-export, imported
|
|
5866
|
+
binding and `export *` (a cycle ends the trace at the module it
|
|
5867
|
+
reached), a namespace export giving its module's file; None for a
|
|
5868
|
+
re-export or a namespace whose module no file under `root` holds."""
|
|
5869
|
+
cache: dict[str, dict] = {}
|
|
5870
|
+
|
|
5871
|
+
def facts(file: str) -> dict:
|
|
5872
|
+
if file not in cache:
|
|
5873
|
+
cache[file] = _quick_js_facts(_read_text(root, file) or "")
|
|
5874
|
+
return cache[file]
|
|
5875
|
+
|
|
5876
|
+
def owner(file: str, name: str) -> str | None:
|
|
5877
|
+
"""The first module the `export *` chain of `file` passes `name` on from."""
|
|
5878
|
+
stack, visited = [file], set()
|
|
5879
|
+
while stack:
|
|
5880
|
+
module = stack.pop()
|
|
5881
|
+
if module in visited:
|
|
5882
|
+
continue
|
|
5883
|
+
visited.add(module)
|
|
5884
|
+
if module != file and name in facts(module)["names"]:
|
|
5885
|
+
return module
|
|
5886
|
+
stack += reversed([t for t in (resolve_js_module(s, module, tree) for s in facts(module)["stars"]) if t])
|
|
5887
|
+
return None
|
|
5888
|
+
|
|
5889
|
+
def resolve(file: str, name: str) -> str | None:
|
|
5890
|
+
if name not in facts(file)["names"] and name == facts(file)["default"]:
|
|
5891
|
+
name = "default" # quick mode names a default export by what it declares
|
|
5892
|
+
visited: set[tuple[str, str]] = set()
|
|
5893
|
+
while (file, name) not in visited:
|
|
5894
|
+
visited.add((file, name))
|
|
5895
|
+
fact = facts(file)["names"].get(name)
|
|
5896
|
+
if fact is None:
|
|
5897
|
+
found = owner(file, name) if name != "default" else None
|
|
5898
|
+
if found is None:
|
|
5899
|
+
return file
|
|
5900
|
+
file = found
|
|
5901
|
+
continue
|
|
5902
|
+
if fact["how"] == "declaration":
|
|
5903
|
+
return file
|
|
5904
|
+
target = resolve_js_module(fact["source"], file, tree)
|
|
5905
|
+
if fact["how"] == "namespace" or target is None:
|
|
5906
|
+
return target
|
|
5907
|
+
file, name = target, fact["local"]
|
|
5908
|
+
return file
|
|
5909
|
+
|
|
5910
|
+
return resolve
|
|
5911
|
+
|
|
5912
|
+
|
|
5913
|
+
def _py_defined_in(root: Path, tree: set[str]) -> Callable[[str, str], str | None]:
|
|
5914
|
+
"""(module, name it exports) -> the file full mode's _py_surface traces
|
|
5915
|
+
that name to from that module (None when it gives none: a name another
|
|
5916
|
+
package passes on, or one the trace does not list)."""
|
|
5917
|
+
traced: dict[str, dict[str, str | None]] = {}
|
|
5918
|
+
|
|
5919
|
+
def resolve(file: str, name: str) -> str | None:
|
|
5920
|
+
if file not in traced:
|
|
5921
|
+
surface = _Surface("python")
|
|
5922
|
+
_py_surface(root, [{"file": file}], tree, surface, [])
|
|
5923
|
+
traced[file] = {n: info["file"] for n, info in surface.names.items()}
|
|
5924
|
+
return traced[file].get(name)
|
|
5925
|
+
|
|
5926
|
+
return resolve
|
|
5927
|
+
|
|
5928
|
+
|
|
5929
|
+
def _defined_in(root: Path, result: dict) -> None:
|
|
5930
|
+
"""Give each export of a Python, JavaScript or TypeScript quick result
|
|
5931
|
+
its `defined_in` (see "Quick mode" above): the file the export is
|
|
5932
|
+
declared in for a name its file binds itself, else the file the trace
|
|
5933
|
+
reaches. The tree under `root` is listed only when a name is traced."""
|
|
5934
|
+
family = DEFINED_IN_FAMILIES.get(result.get("language") or "")
|
|
5935
|
+
if family is None:
|
|
5936
|
+
return
|
|
5937
|
+
resolve: Callable[[str, str], str | None] | None = None
|
|
5938
|
+
for export in result["exports"]:
|
|
5939
|
+
if export["type"] not in ("re-export", "namespace", "module"):
|
|
5940
|
+
export["defined_in"] = export["source_file"]
|
|
5941
|
+
continue
|
|
5942
|
+
if resolve is None:
|
|
5943
|
+
tree = set(list_tree(root))
|
|
5944
|
+
resolve = (_py_defined_in if family == "python" else _js_defined_in)(root, tree)
|
|
5945
|
+
export["defined_in"] = resolve(export["source_file"], export["name"])
|
|
5946
|
+
|
|
5947
|
+
|
|
4482
5948
|
def _main_entries(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
4483
5949
|
"""`--mode entries`: the entry files of one package, as JSON on stdout."""
|
|
4484
5950
|
if args.manifest_file is not None or args.entry_file or args.follow_file:
|
|
@@ -4517,9 +5983,20 @@ def main(argv: list[str]) -> int:
|
|
|
4517
5983
|
args = parser.parse_args(argv)
|
|
4518
5984
|
files = args.manifest_file is not None or bool(args.entry_file) or bool(args.follow_file)
|
|
4519
5985
|
listing = [f"--{flag.replace('_', '-')}" for flag in LISTING_FLAGS if getattr(args, flag) is not None]
|
|
4520
|
-
following = [f"--{flag}" for flag in FOLLOW_FLAGS if getattr(args, flag) not in (None, False)]
|
|
5986
|
+
following = [f"--{flag.replace('_', '-')}" for flag in FOLLOW_FLAGS if getattr(args, flag) not in (None, False)]
|
|
4521
5987
|
if following and (args.mode != "quick" or not files):
|
|
4522
5988
|
parser.error(f"{', '.join(following)}: quick mode only, with --manifest-file or --entry-file")
|
|
5989
|
+
own = [f"--{flag}" for flag in DENOMINATOR_OWN_FLAGS if getattr(args, flag) is not None]
|
|
5990
|
+
if own and args.mode != "denominator":
|
|
5991
|
+
parser.error(f"{', '.join(own)}: denominator mode only (--mode denominator)")
|
|
5992
|
+
if args.mode == "denominator":
|
|
5993
|
+
other = [f"--{flag.replace('_', '-')}" for flag in FULL_MODE_FLAGS
|
|
5994
|
+
if getattr(args, flag) is not None and flag not in DENOMINATOR_FLAGS]
|
|
5995
|
+
listing = [flag for flag in listing if flag != "--tree-file"]
|
|
5996
|
+
if files or listing or other:
|
|
5997
|
+
parser.error(f"{', '.join(other + listing) or '--manifest-file, --entry-file, --follow-file'}: "
|
|
5998
|
+
"not denominator mode's")
|
|
5999
|
+
return _main_denominator(args)
|
|
4523
6000
|
if args.mode == "full":
|
|
4524
6001
|
if files:
|
|
4525
6002
|
parser.error("--manifest-file, --entry-file, --follow-file: quick mode only (--mode quick)")
|
|
@@ -4543,17 +6020,24 @@ def main(argv: list[str]) -> int:
|
|
|
4543
6020
|
if files:
|
|
4544
6021
|
if not args.language or len(args.language) != 1:
|
|
4545
6022
|
parser.error("--manifest-file and --entry-file take one --language")
|
|
4546
|
-
if (args.repo is not None or args.ref is not None) and not args.follow:
|
|
4547
|
-
parser.error("--repo and --ref fetch the modules --follow reads
|
|
6023
|
+
if (args.repo is not None or args.ref is not None) and not (args.follow or args.resolve_version):
|
|
6024
|
+
parser.error("--repo and --ref fetch the modules --follow reads and name the tags --resolve-version "
|
|
6025
|
+
"reads, so pass --follow or --resolve-version")
|
|
4548
6026
|
if args.ref is not None and args.repo is None:
|
|
4549
6027
|
parser.error("--ref is the ref of --repo, so pass --repo")
|
|
4550
6028
|
if args.follow and (args.tree_file is None or args.source_root is None):
|
|
4551
6029
|
parser.error("--follow finds each module in --tree-file and reads it under --source-root: pass both")
|
|
6030
|
+
if args.resolve_version and args.source_root is None:
|
|
6031
|
+
parser.error("--resolve-version reads the version files under --source-root: pass it")
|
|
4552
6032
|
try:
|
|
4553
6033
|
payload = _quick_files_payload(args)
|
|
4554
6034
|
result = extract(payload)
|
|
4555
6035
|
if args.follow and "_error" not in result:
|
|
4556
6036
|
result = _follow(args, result)
|
|
6037
|
+
if args.resolve_version and "_error" not in result:
|
|
6038
|
+
resolve_version(args, result, payload.get("tree"))
|
|
6039
|
+
if args.source_root is not None and "_error" not in result:
|
|
6040
|
+
_defined_in(Path(args.source_root), result)
|
|
4557
6041
|
if args.fetch_list is not None and "_error" not in result:
|
|
4558
6042
|
follow = [item["module_file"] for item in result["unlisted"] if item["module_file"]]
|
|
4559
6043
|
_write_fetch_list(args.fetch_list, list(dict.fromkeys(follow)))
|