bmad-module-skill-forge 3.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +3 -3
- package/docs/_data/pinned.yaml +19 -13
- package/docs/architecture.md +2 -2
- package/docs/concepts.md +4 -4
- package/docs/examples.md +33 -26
- package/docs/getting-started.md +3 -3
- package/docs/how-it-works.md +2 -2
- package/docs/index.md +1 -1
- package/docs/skill-model.md +41 -39
- package/docs/troubleshooting.md +8 -0
- package/docs/verifying-a-skill.md +46 -21
- package/docs/why-skf.md +1 -1
- package/docs/workflows.md +5 -5
- package/package.json +1 -1
- package/src/knowledge/qmd-registry.md +1 -1
- package/src/knowledge/version-paths.md +1 -1
- package/src/shared/health-check.md +2 -50
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +12 -4
- package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +212 -59
- package/src/shared/scripts/skf-build-change-manifest.py +1437 -178
- package/src/shared/scripts/skf-classify-changed-files.py +78 -0
- package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
- package/src/shared/scripts/skf-count-tokens.py +54 -10
- package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
- package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
- package/src/shared/scripts/skf-detect-workspaces.py +67 -0
- package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
- package/src/shared/scripts/skf-extract-public-api.py +1673 -189
- package/src/shared/scripts/skf-extraction-inventory.py +584 -46
- package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
- package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
- package/src/shared/scripts/skf-github-probe.py +47 -8
- package/src/shared/scripts/skf-hash-content.py +14 -8
- package/src/shared/scripts/skf-load-provenance.py +3 -1
- package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
- package/src/shared/scripts/skf-parse-gaps.py +135 -26
- package/src/shared/scripts/skf-preflight.py +15 -1
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
- package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
- package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
- package/src/shared/scripts/skf-resolve-customization.py +197 -0
- package/src/shared/scripts/skf-shard-body.py +239 -41
- package/src/shared/scripts/skf-structural-diff.py +133 -18
- package/src/shared/scripts/skf-update-active-symlink.py +115 -62
- package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
- package/src/shared/scripts/skf-validate-output.py +12 -3
- package/src/shared/scripts/skf-verify-provenance-completeness.py +1439 -79
- package/src/shared/scripts/skf-write-skill-brief.py +41 -8
- package/src/shared/tool-requirements.yaml +8 -2
- package/src/skf-analyze-source/SKILL.md +2 -2
- package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
- package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
- package/src/skf-audit-skill/SKILL.md +2 -2
- package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
- package/src/skf-audit-skill/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +2 -1
- package/src/skf-audit-skill/references/re-index.md +14 -9
- package/src/skf-audit-skill/references/report.md +2 -2
- package/src/skf-audit-skill/references/semantic-diff.md +15 -15
- package/src/skf-audit-skill/references/structural-diff.md +10 -5
- package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
- package/src/skf-brief-skill/SKILL.md +3 -3
- package/src/skf-brief-skill/assets/scope-templates.md +1 -1
- package/src/skf-brief-skill/assets/skill-brief-schema.md +4 -11
- package/src/skf-brief-skill/references/analyze-target.md +6 -6
- package/src/skf-brief-skill/references/confirm-brief.md +1 -1
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
- package/src/skf-brief-skill/references/version-resolution.md +5 -9
- package/src/skf-campaign/SKILL.md +2 -2
- package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
- package/src/skf-create-skill/SKILL.md +2 -2
- package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
- package/src/skf-create-skill/assets/skill-sections.md +7 -7
- package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
- package/src/skf-create-skill/references/compile.md +31 -60
- package/src/skf-create-skill/references/component-extraction.md +3 -3
- package/src/skf-create-skill/references/enrich.md +2 -2
- package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
- package/src/skf-create-skill/references/extract.md +23 -19
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
- package/src/skf-create-skill/references/extraction-patterns.md +10 -11
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +21 -5
- package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
- package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
- package/src/skf-create-stack-skill/SKILL.md +2 -2
- package/src/skf-create-stack-skill/references/validate.md +1 -1
- package/src/skf-drop-skill/SKILL.md +2 -2
- package/src/skf-drop-skill/references/execute.md +5 -1
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +2 -2
- package/src/skf-drop-skill/references/select.md +7 -3
- package/src/skf-export-skill/SKILL.md +2 -2
- package/src/skf-export-skill/assets/snippet-format.md +5 -5
- package/src/skf-export-skill/references/generate-snippet.md +22 -15
- package/src/skf-export-skill/references/load-skill.md +1 -1
- package/src/skf-export-skill/references/token-report.md +29 -9
- package/src/skf-forger/SKILL.md +2 -2
- package/src/skf-quick-skill/SKILL.md +2 -2
- package/src/skf-refine-architecture/SKILL.md +6 -8
- package/src/skf-rename-skill/SKILL.md +2 -2
- package/src/skf-setup/SKILL.md +2 -2
- package/src/skf-test-skill/SKILL.md +2 -2
- package/src/skf-test-skill/assets/output-section-formats.md +6 -4
- package/src/skf-test-skill/references/coherence-check.md +5 -4
- package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
- package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
- package/src/skf-test-skill/references/coverage-check.md +69 -111
- package/src/skf-test-skill/references/init.md +5 -6
- package/src/skf-test-skill/references/migration-section-rules.md +26 -14
- package/src/skf-test-skill/references/report.md +11 -9
- package/src/skf-test-skill/references/scoring-rules.md +8 -6
- package/src/skf-test-skill/references/source-access-protocol.md +15 -18
- package/src/skf-test-skill/scripts/build-result-context.py +114 -21
- package/src/skf-test-skill/scripts/gap-ledger.py +580 -80
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +646 -107
- package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
- package/src/skf-test-skill/scripts/reconcile-coverage.py +450 -23
- package/src/skf-test-skill/scripts/score-signatures.py +356 -61
- package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
- package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
- package/src/skf-test-skill/templates/test-report-template.md +1 -0
- package/src/skf-update-skill/SKILL.md +2 -2
- package/src/skf-update-skill/references/detect-changes.md +32 -13
- package/src/skf-update-skill/references/gap-driven.md +37 -22
- package/src/skf-update-skill/references/init.md +6 -2
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +17 -12
- package/src/skf-update-skill/references/re-extract.md +4 -2
- package/src/skf-update-skill/references/report.md +7 -4
- package/src/skf-update-skill/references/write.md +77 -36
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
- package/src/skf-verify-stack/SKILL.md +2 -2
- package/tools/cli/lib/tool-check.js +158 -2
|
@@ -33,8 +33,9 @@ entries are per-citation, so the two sets measure different surfaces:
|
|
|
33
33
|
Four more subcommands act on the same rules: `fix` applies the citation
|
|
34
34
|
prefix and source-line fixes a `verify` result has one answer for,
|
|
35
35
|
`definition-lines` gives the lines that define one export,
|
|
36
|
-
`classify-stale` decides which stale names are fabricated signatures
|
|
37
|
-
`kind-at` gives the node kind
|
|
36
|
+
`classify-stale` decides which stale names are fabricated signatures and
|
|
37
|
+
where the source declares the others, and `kind-at` gives the node kind
|
|
38
|
+
of the recipe that matches a source line.
|
|
38
39
|
|
|
39
40
|
An LLM set-diff can silently pass a dropped or orphaned entry, and eyeballing
|
|
40
41
|
whether a cited line still exists is not something the model can do reliably.
|
|
@@ -88,7 +89,19 @@ name (`App.update`) it is the last segment. The rules follow the
|
|
|
88
89
|
a keyword argument or a parameter never matches; a file that does not
|
|
89
90
|
tokenize is matched line by line. A `from X import *` line counts only
|
|
90
91
|
when no other line of the file defines NAME (a re-export the
|
|
91
|
-
extraction could not trace is cited at its star import).
|
|
92
|
+
extraction could not trace is cited at its star import). An
|
|
93
|
+
`@overload` group, where every line found is a `def` / `async def
|
|
94
|
+
NAME` line at one indentation in one enclosing block (one class, not
|
|
95
|
+
two) and every one but at most the last has `@overload` or
|
|
96
|
+
`@<module>.overload` (`@typing.overload`,
|
|
97
|
+
`@typing_extensions.overload`, `@t.overload` after
|
|
98
|
+
`import typing as t`) in its contiguous decorator block, defines NAME
|
|
99
|
+
at each of its lines, the stubs and the implementation. A comment line
|
|
100
|
+
may sit in the decorator block, and so may a decorator's arguments on
|
|
101
|
+
lines of their own, unless the file does not tokenize. A recorded line
|
|
102
|
+
that misses the group gives its first line alone, which is its first
|
|
103
|
+
stub: the public-API runner, its merge rule and the structural diff
|
|
104
|
+
all record that line, so `fix` can move the entry there.
|
|
92
105
|
- TS/JS (`.ts`, `.tsx`, `.mts`, `.cts`, `.js`, `.jsx`, `.mjs`, `.cjs`): a
|
|
93
106
|
`function` (or `function*`), `class`, `interface`, `type`, `enum`,
|
|
94
107
|
`namespace`, `const`, `let` or `var` declaration of NAME, with any of
|
|
@@ -97,7 +110,11 @@ name (`App.update`) it is the last segment. The rules follow the
|
|
|
97
110
|
and `class`; a name inside a decorator's arguments is not declared);
|
|
98
111
|
a later declarator of a `const`, `let` or `var` list
|
|
99
112
|
(`export const a = 1, NAME = 2;`, or `export const a = 1,` then
|
|
100
|
-
` NAME = 2;` as Prettier writes it: see Declarator lists below);
|
|
113
|
+
` NAME = 2;` as Prettier writes it: see Declarator lists below);
|
|
114
|
+
NAME bound by a destructuring pattern that begins such a list
|
|
115
|
+
(`export const { a, b: NAME } = obj;`, `const [NAME] = list`, the
|
|
116
|
+
pattern on one line or several), on the line where NAME appears (see
|
|
117
|
+
Declarator lists below); NAME
|
|
101
118
|
on the line right after a line that ends in its declaration keyword
|
|
102
119
|
(`export function`, `export interface`, `export const` and so on),
|
|
103
120
|
which is the line cited;
|
|
@@ -124,14 +141,23 @@ name (`App.update`) it is the last segment. The rules follow the
|
|
|
124
141
|
Declarator lists: a `const`, `let` or `var` list is split at its
|
|
125
142
|
top-level commas, outside strings, regular expression literals and
|
|
126
143
|
brackets, a type-argument list included (`Map<K, V>`: a `<` right after
|
|
127
|
-
a name), up to a top-level
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
counts. A `<` where an expression begins (JSX, a type assertion, a generic
|
|
144
|
+
a name), up to a top-level `;`. The list goes on to the next line when a
|
|
145
|
+
line ends inside a bracket, after a top-level comma or after an operator
|
|
146
|
+
waiting for its operand, or when the next line starts with an operator (a
|
|
147
|
+
member chain or a ternary broken over lines), so a declarator on a line
|
|
148
|
+
of its own counts. A `<` where an expression begins (JSX, a type assertion, a generic
|
|
133
149
|
arrow function's type parameters) is not followed: no later declarator of
|
|
134
|
-
the list counts after it.
|
|
150
|
+
the list counts after it. A destructuring pattern that begins the list,
|
|
151
|
+
on the keyword's line, declares the names it binds: a shorthand key, the
|
|
152
|
+
target after `key:`, an array element, a rest `...NAME` and the names of
|
|
153
|
+
a nested pattern, each on the line where it appears (the pattern may
|
|
154
|
+
span lines), with its statement's column. A renamed or computed key, a
|
|
155
|
+
default value, the type annotation after the pattern and anything inside
|
|
156
|
+
a string or a comment bind nothing, and neither does a pattern that does
|
|
157
|
+
not close. The later declarators after a pattern count as any others. A
|
|
158
|
+
pattern that is a later declarator (`export const a = 1, { b } = c`) or
|
|
159
|
+
that sits on the line after a split keyword (`export const`, then
|
|
160
|
+
` { b } = c;`) is not covered: it declares nothing.
|
|
135
161
|
|
|
136
162
|
Indentation: for a dotted name every match counts. For an undotted name,
|
|
137
163
|
matches at any indentation count, but when the file has a column-0 match
|
|
@@ -139,7 +165,8 @@ the indented ones are dropped, so a local shadow never outranks the
|
|
|
139
165
|
module-level definition. A line inside a multi-line import, export list
|
|
140
166
|
or `module.exports` object takes the column of the line its statement
|
|
141
167
|
starts on, and so do the name line of a declaration split after its
|
|
142
|
-
keyword
|
|
168
|
+
keyword, a later declarator on a line of its own and the line of a name
|
|
169
|
+
a destructuring pattern binds. When the recorded
|
|
143
170
|
line is itself an indented match that this rule dropped (a method
|
|
144
171
|
recorded under an undotted name beside a module-level function of the
|
|
145
172
|
same name), the finding lists the recorded line too, so it carries
|
|
@@ -148,7 +175,11 @@ several lines and no caller auto-moves it.
|
|
|
148
175
|
Entries in other files, and entries whose `export_type` is `module` or
|
|
149
176
|
`package` (case-insensitive; neither has a definition line), are skipped
|
|
150
177
|
and counted in `summary.line_check_skipped`. A `line-not-definition` item
|
|
151
|
-
lists every defining line in the file in `definition_lines
|
|
178
|
+
lists every defining line in the file in `definition_lines`, the recorded
|
|
179
|
+
line included when it is an indented match the column-0 rule dropped (see
|
|
180
|
+
Indentation). The one exception is a Python `@overload` group: it lists
|
|
181
|
+
the group's first line alone, unless the recorded line is such a dropped
|
|
182
|
+
indented match, which still gives several lines. An empty list
|
|
152
183
|
means the rules found no definition line, which may be a shape they do not
|
|
153
184
|
cover: the entry is unverified, not gone, and no caller auto-fixes it.
|
|
154
185
|
|
|
@@ -386,23 +417,30 @@ Subcommands:
|
|
|
386
417
|
}
|
|
387
418
|
|
|
388
419
|
With `--line`, a recorded line that is an indented match the column-0
|
|
389
|
-
rule dropped joins `definition_lines`,
|
|
420
|
+
rule dropped joins `definition_lines`, and one that misses a Python
|
|
421
|
+
`@overload` group gives its first line alone, as in `verify`.
|
|
390
422
|
|
|
391
423
|
classify-stale --names <coverage.json> --provenance <provenance-map.json>
|
|
392
|
-
--source-root <path> [
|
|
424
|
+
--source-root <path> [--inventory <inventory.json>]
|
|
425
|
+
[-o <out.json>]
|
|
393
426
|
|
|
394
427
|
test-skill's fabricated-signature test, for each stale name (a
|
|
395
428
|
documented name the enumerated source surface lacks): the
|
|
396
429
|
definition-lines rules above, run for every provenance entry whose raw
|
|
397
430
|
`export_name` is the name, at its `source_file` and `source_line`, with
|
|
398
431
|
its `export_type`. `--names` holds reconcile-coverage.py's result (its
|
|
399
|
-
`stale` names) or a JSON array of names.
|
|
432
|
+
`stale` names) or a JSON array of names. `--inventory` holds the
|
|
433
|
+
documented inventory, validate-inventory.py's `--output` file or its
|
|
434
|
+
whole result: the `kind` it gives each name decides the module rule
|
|
435
|
+
below.
|
|
400
436
|
|
|
401
437
|
Emit JSON, one item per name, in the order given (a repeated name once):
|
|
402
438
|
[
|
|
403
439
|
{"name": "<as given>",
|
|
404
440
|
"fabricated": <bool>,
|
|
405
441
|
"source": "<source_file>:<source_line>" | "<source_file>" | null,
|
|
442
|
+
"defined_at": "<path>:<line>" | null,
|
|
443
|
+
"declared_in": ["<path>:<line>", ...] | null,
|
|
406
444
|
"reason": "not-defined" | "defined" | "unchecked" | "no-entry",
|
|
407
445
|
"entries": [{"source_file": "<as recorded>",
|
|
408
446
|
"source_line": <as recorded>,
|
|
@@ -428,6 +466,81 @@ Subcommands:
|
|
|
428
466
|
entry with a `source_file` (the file alone when its line is not a
|
|
429
467
|
number), null when no entry has one.
|
|
430
468
|
|
|
469
|
+
`defined_at` is where the source declares a name that is not
|
|
470
|
+
fabricated, so test-skill can tell a documented extra (a name outside
|
|
471
|
+
the surface that the source still declares) from stale documentation;
|
|
472
|
+
a fabricated name is not looked up (null). A declaration is a line at
|
|
473
|
+
column 0 that the definition-line rules match with imports left out:
|
|
474
|
+
a `def`, `class`, assignment, annotation, type alias, TS/JS
|
|
475
|
+
declaration, `exports.NAME =`, or an alias that renames
|
|
476
|
+
(`import X as NAME`, `from X import Y as NAME`,
|
|
477
|
+
`export { Y as NAME }`, `export * as NAME from`), or a
|
|
478
|
+
`module.exports` key with a value or a method. A plain or star import,
|
|
479
|
+
an `as` that keeps the imported name (`from m import NAME as NAME`,
|
|
480
|
+
`export { NAME as NAME }`), `export import NAME =`,
|
|
481
|
+
`export default NAME`, an export-list item without `as`, a
|
|
482
|
+
`module.exports` shorthand key, a binding whose value is an import
|
|
483
|
+
(`require(...)` or `(await) import(...)`, on its line or the next:
|
|
484
|
+
`const NAME = require(...)`, `const { NAME } = require(...)`,
|
|
485
|
+
`exports.NAME = require(...)`, `{ NAME: require(...) }`), an
|
|
486
|
+
indented line and any line of a Python
|
|
487
|
+
file that does not tokenize never count. A module or package named
|
|
488
|
+
NAME (`NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
|
|
489
|
+
`NAME/index.<ext>`) declares NAME too: at its line 1 when no line of it
|
|
490
|
+
declares NAME, unless `--inventory` gives NAME a `function`, `hook`,
|
|
491
|
+
`class`, `interface`, `enum` or `type` kind (any of its kinds, when it
|
|
492
|
+
gives several; a `method` kind is not read), no entry of NAME is a
|
|
493
|
+
`module` or `package` one, and the file is not a TS/JS one that is
|
|
494
|
+
itself the export (a column-0 `export default` or `module.exports =`),
|
|
495
|
+
so that a removed function named after its file is no declaration
|
|
496
|
+
(the inventory has no `module` kind). `declared_line` gives this
|
|
497
|
+
answer for one file, the module rule optional, to a script that loads
|
|
498
|
+
this one, and
|
|
499
|
+
`baseline_gaps` applies it to the map's entries an extraction left
|
|
500
|
+
out (skf-extraction-snapshot.py's baseline gaps and
|
|
501
|
+
skf-build-change-manifest.py's `baseline-gaps`, which apply the module
|
|
502
|
+
rule only to a `module` or `package` entry).
|
|
503
|
+
|
|
504
|
+
The files: when a file the name's entries cite declares it,
|
|
505
|
+
`defined_at` is that file's first declaration (the files in entry
|
|
506
|
+
order), and `declared_in` is null. Otherwise `declared_in` lists, as
|
|
507
|
+
sorted `<path>:<line>`, every file of the walk that declares the name,
|
|
508
|
+
at its first declaration (a type stub, `NAME.pyi` or `NAME.d.ts`,
|
|
509
|
+
beside its implementation is one file with it: the implementation),
|
|
510
|
+
and `defined_at` is the one item when the list has exactly one and no
|
|
511
|
+
walked Python file that does not tokenize holds the name (a homonym,
|
|
512
|
+
two files or more, gives null: test-skill decides it by the files the
|
|
513
|
+
skill cites). The walk reads, sorted by path, every file below the
|
|
514
|
+
walk root: the deepest folder that the files of the map's entries
|
|
515
|
+
share, widened to its package root (Python: the nearest folder at or
|
|
516
|
+
above it that holds `__init__.py`, then up while the parent holds
|
|
517
|
+
one, never the source root itself; TS/JS: the nearest folder at or
|
|
518
|
+
above it that holds `package.json`), leaving out the entries whose
|
|
519
|
+
file lies outside every package; the source root when none lies in
|
|
520
|
+
one. Only the rule families (Python, TS/JS) the map's entries cite are
|
|
521
|
+
read (both when no entry cites a file). The walk skips symlinks, hidden folders, test
|
|
522
|
+
files and test configuration files (`conftest.py`, `jest.config.*`),
|
|
523
|
+
`*.min.js`, and the folders generated, built, installed, vendored or
|
|
524
|
+
test output lives in (`node_modules`, `dist`, `build`, `vendor`,
|
|
525
|
+
`_vendor`, `tests`, `testing`, `fixtures`, `__mocks__`,
|
|
526
|
+
`site-packages`, `env`, `bower_components`, `esm`, `cjs`, `umd` and
|
|
527
|
+
the like), `docs`, `examples` and `benchmarks`, each judged on the
|
|
528
|
+
path below the walk root, the end-to-end test folders `e2e` and
|
|
529
|
+
`__fixtures__` wherever they lie, and, as the first folder below the
|
|
530
|
+
walk root or below a package root inside it (a folder that holds
|
|
531
|
+
`package.json`, the top of an `__init__.py` chain), `example`, `demo`,
|
|
532
|
+
`demos`, `sample`, `samples`, `scripts`, `playground`, `bench`,
|
|
533
|
+
`benchmark` and `migrations` unless it holds a file an entry cites
|
|
534
|
+
(deeper in a package such a folder can be package code). A cited file
|
|
535
|
+
is read only inside the source root (a path that is absolute or climbs
|
|
536
|
+
out of it is not read), with no symlink on its path and under the same
|
|
537
|
+
skips, those walk-only ones aside. The path is relative to the source
|
|
538
|
+
root. A dotted name, or a map whose entries cite files
|
|
539
|
+
but no Python or TS/JS one, is not looked up (`defined_at` and
|
|
540
|
+
`declared_in` null, as for a fabricated name), and a filesystem error
|
|
541
|
+
during the lookup gives every name null in both (the exit code stays
|
|
542
|
+
0).
|
|
543
|
+
|
|
431
544
|
kind-at --file <path> --line <n> --recipes <recipes>
|
|
432
545
|
[--language <lang>] [--name <export_name>]
|
|
433
546
|
[--source-root <path>] [-o <out.json>]
|
|
@@ -483,7 +596,8 @@ CLI examples:
|
|
|
483
596
|
uv run skf-verify-provenance-completeness.py classify-stale \\
|
|
484
597
|
--names {run_dir}/coverage.json \\
|
|
485
598
|
--provenance {forge_version}/provenance-map.json \\
|
|
486
|
-
--source-root {source_root}
|
|
599
|
+
--source-root {source_root} --inventory {run_dir}/inventory.json \\
|
|
600
|
+
-o {run_dir}/stale.json
|
|
487
601
|
uv run skf-verify-provenance-completeness.py kind-at \\
|
|
488
602
|
--source-root {source_root} --file src/api.ts --line 12 --name Store \\
|
|
489
603
|
--recipes {extractionPatternsData}
|
|
@@ -507,16 +621,17 @@ Exit codes:
|
|
|
507
621
|
2 error, for every subcommand (an input file or folder not found,
|
|
508
622
|
--skill-dir without a SKILL.md, malformed JSON or YAML, a verify
|
|
509
623
|
result without stale[] and citations[], --names without a list of
|
|
510
|
-
names,
|
|
511
|
-
|
|
512
|
-
unexpected failure, such as an
|
|
513
|
-
file; once fix has started writing, stderr names the files
|
|
514
|
-
written); one line on stderr and no JSON
|
|
624
|
+
names, --inventory without an `exports` list, no recipe in
|
|
625
|
+
--recipes, no PyYAML for kind-at, an atomic write that failed, an -o
|
|
626
|
+
file that cannot be written, or any unexpected failure, such as an
|
|
627
|
+
unreadable file; once fix has started writing, stderr names the files
|
|
628
|
+
already written); one line on stderr and no JSON
|
|
515
629
|
"""
|
|
516
630
|
|
|
517
631
|
from __future__ import annotations
|
|
518
632
|
|
|
519
633
|
import argparse
|
|
634
|
+
import bisect
|
|
520
635
|
import copy
|
|
521
636
|
import importlib.util
|
|
522
637
|
import io
|
|
@@ -828,6 +943,13 @@ _PY_TARGET_LIST_RE = re.compile(
|
|
|
828
943
|
)
|
|
829
944
|
# `def \` or `class \`: the name is on the next line.
|
|
830
945
|
_PY_SPLIT_DECL_RE = re.compile(r"^\s*(?:async\s+def|def|class)\s*\\$")
|
|
946
|
+
# A line that can matter to a name it does not hold: an import (which may go
|
|
947
|
+
# on over the next lines) or a `def \` / `class \` whose name is on the next.
|
|
948
|
+
_PY_NAMELESS_MATCH_RE = re.compile(r"^\s*(?:from|import|async|def|class)\b")
|
|
949
|
+
# The decorator of an `@overload` stub, imported by name or used through its
|
|
950
|
+
# module under any name (`typing`, `typing_extensions`, `t` after
|
|
951
|
+
# `import typing as t`).
|
|
952
|
+
_PY_OVERLOAD_RE = re.compile(r"^\s*@\s*(?:[A-Za-z_]\w*\.)?overload\s*$")
|
|
831
953
|
# Tokens that never begin a statement.
|
|
832
954
|
_PY_LAYOUT_TOKENS = frozenset(
|
|
833
955
|
{
|
|
@@ -861,6 +983,15 @@ _JS_EXPORT_ITEM_RE = re.compile(
|
|
|
861
983
|
rf"^(?:type\s+)?({_JS_NAME_OR_STRING})(?:\s+as\s+({_JS_NAME_OR_STRING}))?$"
|
|
862
984
|
)
|
|
863
985
|
_JS_MODULE_EXPORTS_OBJECT_RE = re.compile(r"^\s*module\.exports\s*=\s*\{(.*)$")
|
|
986
|
+
# A bound value that is an import: `require(...)`, `import(...)`, awaited, in
|
|
987
|
+
# parentheses (`(await import("./m")).X`) or in one helper call
|
|
988
|
+
# (`__importDefault(require("./m"))`).
|
|
989
|
+
_JS_IMPORT_VALUE_RE = re.compile(
|
|
990
|
+
r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?(?:await\s+)?(?:require|import)\s*\("
|
|
991
|
+
)
|
|
992
|
+
# The start of such a value that goes on on the next line: nothing yet, a
|
|
993
|
+
# parenthesis, or a helper call's name and parenthesis.
|
|
994
|
+
_JS_IMPORT_VALUE_OPEN_RE = re.compile(r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?$")
|
|
864
995
|
# A `const`, `let` or `var` statement: its declarator list.
|
|
865
996
|
_JS_VAR_DECL_RE = re.compile(
|
|
866
997
|
rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(?!enum(?![\w$]))(.*)$"
|
|
@@ -870,6 +1001,18 @@ _JS_SPLIT_DECL_RE = re.compile(
|
|
|
870
1001
|
rf"^\s*{_JS_MODIFIERS}(?P<kind>function(?:\s*\*)?|class|interface|type"
|
|
871
1002
|
r"|enum|namespace|let|var|const(?:\s+enum)?)$"
|
|
872
1003
|
)
|
|
1004
|
+
# The keywords every line that opens a statement going on below holds (a
|
|
1005
|
+
# `const` / `let` / `var` list, a split declaration, `export {`,
|
|
1006
|
+
# `module.exports = {`): a line with none of them, and not the name, can
|
|
1007
|
+
# match no rule while no statement is open.
|
|
1008
|
+
_JS_OPENING_WORDS = ("const", "let", "var", "function", "class", "interface", "type", "enum", "namespace",
|
|
1009
|
+
"export", "module")
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
def _js_quiet_lines(code_lines: list[str]) -> list[bool]:
|
|
1013
|
+
"""For each comment-free line, True when it holds none of
|
|
1014
|
+
`_JS_OPENING_WORDS`."""
|
|
1015
|
+
return [not any(word in code for word in _JS_OPENING_WORDS) for code in code_lines]
|
|
873
1016
|
_JS_VAR_KEYWORDS = frozenset({"const", "let", "var"})
|
|
874
1017
|
# How a `const` / `let` / `var` list that did not end on its line goes on
|
|
875
1018
|
# (`_js_declarators`): the next line begins a declarator (this one ended
|
|
@@ -883,6 +1026,11 @@ _DECL_MAYBE = "maybe"
|
|
|
883
1026
|
# What `_js_declarators` gives: None, or (brackets left open, how) the list
|
|
884
1027
|
# goes on with, and the pieces that begin a declarator.
|
|
885
1028
|
_DeclSplit = tuple[tuple[str, str] | None, list[str]]
|
|
1029
|
+
# What `_js_destructured` gives for a `const` / `let` / `var` statement
|
|
1030
|
+
# whose first declarator is a destructuring pattern: the (name, line) of
|
|
1031
|
+
# each name it binds, and the (line, text after the `=` on that line) its
|
|
1032
|
+
# initializer starts at, None when it has none.
|
|
1033
|
+
_PatternDecl = tuple[list[tuple[str, int]], tuple[int, str] | None]
|
|
886
1034
|
# After one of these, `/` begins a regular expression and `<` begins JSX, a
|
|
887
1035
|
# type assertion or a generic arrow function's type parameters.
|
|
888
1036
|
_JS_EXPRESSION_START = frozenset("([{,;=?:&|^%+-*/<>!~")
|
|
@@ -909,6 +1057,21 @@ def _read_lines(path: Path) -> list[str] | None:
|
|
|
909
1057
|
return None
|
|
910
1058
|
|
|
911
1059
|
|
|
1060
|
+
def _read_source(path: Path) -> str:
|
|
1061
|
+
"""A text file's content, decoded as `_read_lines` decodes it. Raises
|
|
1062
|
+
OSError when the file cannot be read."""
|
|
1063
|
+
with path.open("r", encoding="utf-8-sig", errors="replace") as fh:
|
|
1064
|
+
return fh.read()
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def _text_lines(text: str) -> list[str]:
|
|
1068
|
+
"""`_read_source`'s text as the lines `_read_lines` gives for the file."""
|
|
1069
|
+
lines = text.split("\n")
|
|
1070
|
+
if lines[-1] == "":
|
|
1071
|
+
lines.pop()
|
|
1072
|
+
return lines
|
|
1073
|
+
|
|
1074
|
+
|
|
912
1075
|
def _is_column_0(text: str) -> bool:
|
|
913
1076
|
"""True when a line starts at column 0 (no leading whitespace)."""
|
|
914
1077
|
return not text[:1].isspace()
|
|
@@ -984,14 +1147,17 @@ def _js_code_lines(lines: list[str]) -> list[str]:
|
|
|
984
1147
|
|
|
985
1148
|
|
|
986
1149
|
class _SourceCache:
|
|
987
|
-
"""Source lines, Python statement starts
|
|
988
|
-
|
|
989
|
-
|
|
1150
|
+
"""Source lines, Python statement starts, TS/JS comment-free lines and
|
|
1151
|
+
the names TS/JS destructuring patterns bind, read once per file, and
|
|
1152
|
+
TS/JS declarator lists, split once per line (neither depends on the
|
|
1153
|
+
export looked up)."""
|
|
990
1154
|
|
|
991
1155
|
def __init__(self) -> None:
|
|
992
1156
|
self._lines: dict[Path, list[str] | None] = {}
|
|
993
1157
|
self._starts: dict[Path, set[int] | None] = {}
|
|
994
1158
|
self._js_code: dict[Path, list[str]] = {}
|
|
1159
|
+
self._js_quiet: dict[Path, list[bool]] = {}
|
|
1160
|
+
self._js_patterns: dict[Path, dict[int, _PatternDecl]] = {}
|
|
995
1161
|
self._js_decl: dict[tuple[str, str, bool], _DeclSplit] = {}
|
|
996
1162
|
|
|
997
1163
|
def lines(self, path: Path) -> list[str] | None:
|
|
@@ -1009,6 +1175,18 @@ class _SourceCache:
|
|
|
1009
1175
|
self._js_code[path] = _js_code_lines(lines)
|
|
1010
1176
|
return self._js_code[path]
|
|
1011
1177
|
|
|
1178
|
+
def js_quiet_lines(self, path: Path, code_lines: list[str]) -> list[bool]:
|
|
1179
|
+
if path not in self._js_quiet:
|
|
1180
|
+
self._js_quiet[path] = _js_quiet_lines(code_lines)
|
|
1181
|
+
return self._js_quiet[path]
|
|
1182
|
+
|
|
1183
|
+
def js_destructured(
|
|
1184
|
+
self, path: Path, code_lines: list[str]
|
|
1185
|
+
) -> dict[int, _PatternDecl]:
|
|
1186
|
+
if path not in self._js_patterns:
|
|
1187
|
+
self._js_patterns[path] = _js_destructured(code_lines)
|
|
1188
|
+
return self._js_patterns[path]
|
|
1189
|
+
|
|
1012
1190
|
def js_declarators(
|
|
1013
1191
|
self, text: str, opened: str = "", fresh: bool = True
|
|
1014
1192
|
) -> _DeclSplit:
|
|
@@ -1018,8 +1196,12 @@ class _SourceCache:
|
|
|
1018
1196
|
return self._js_decl[key]
|
|
1019
1197
|
|
|
1020
1198
|
|
|
1021
|
-
def _py_import_binds(
|
|
1022
|
-
|
|
1199
|
+
def _py_import_binds(
|
|
1200
|
+
items: str, name: str, from_import: bool, alias_only: bool = False
|
|
1201
|
+
) -> bool:
|
|
1202
|
+
"""True when an import item list binds `name` (with `alias_only`, only
|
|
1203
|
+
through an explicit `as name` that renames: `from m import name as
|
|
1204
|
+
name` and `import a.name as name` keep the imported name)."""
|
|
1023
1205
|
items = items.split("#", 1)[0]
|
|
1024
1206
|
for raw in items.replace("(", " ").replace(")", " ").replace("\\", " ").split(","):
|
|
1025
1207
|
m = _PY_IMPORT_ITEM_RE.match(raw.strip())
|
|
@@ -1027,7 +1209,11 @@ def _py_import_binds(items: str, name: str, from_import: bool) -> bool:
|
|
|
1027
1209
|
continue
|
|
1028
1210
|
target, alias = m.group(1), m.group(2)
|
|
1029
1211
|
if alias is not None:
|
|
1212
|
+
if alias_only and alias == target.rsplit(".", 1)[-1]:
|
|
1213
|
+
continue
|
|
1030
1214
|
bound = alias
|
|
1215
|
+
elif alias_only:
|
|
1216
|
+
continue
|
|
1031
1217
|
elif from_import:
|
|
1032
1218
|
bound = target
|
|
1033
1219
|
else:
|
|
@@ -1050,7 +1236,10 @@ def _py_assigned_names(code: str) -> set[str]:
|
|
|
1050
1236
|
|
|
1051
1237
|
|
|
1052
1238
|
def _python_definition_lines(
|
|
1053
|
-
lines: list[str],
|
|
1239
|
+
lines: list[str],
|
|
1240
|
+
name: str,
|
|
1241
|
+
starts: set[int] | None,
|
|
1242
|
+
declarations_only: bool = False,
|
|
1054
1243
|
) -> list[tuple[int, bool]]:
|
|
1055
1244
|
"""(line, at column 0) for every line that defines `name`.
|
|
1056
1245
|
|
|
@@ -1059,7 +1248,9 @@ def _python_definition_lines(
|
|
|
1059
1248
|
line of a `def \\` or `class \\` whose name is on the next line. A
|
|
1060
1249
|
`from X import *` line counts only when no other line defines `name` (a
|
|
1061
1250
|
re-export the extraction could not trace further is cited at its star
|
|
1062
|
-
import).
|
|
1251
|
+
import). With `declarations_only`, an import counts only when it binds
|
|
1252
|
+
`name` through an explicit `as name` that renames (`_py_import_binds`),
|
|
1253
|
+
and a star import never does.
|
|
1063
1254
|
"""
|
|
1064
1255
|
esc = re.escape(name)
|
|
1065
1256
|
decl = re.compile(rf"^\s*(?:async\s+def|def|class)\s+{esc}(?!\w)")
|
|
@@ -1085,11 +1276,13 @@ def _python_definition_lines(
|
|
|
1085
1276
|
code = stripped[:-1]
|
|
1086
1277
|
else:
|
|
1087
1278
|
cont = None
|
|
1088
|
-
if _py_import_binds(code, name, from_import):
|
|
1279
|
+
if _py_import_binds(code, name, from_import, declarations_only):
|
|
1089
1280
|
found.append((number, top))
|
|
1090
1281
|
continue
|
|
1091
1282
|
if starts is not None and number not in starts:
|
|
1092
1283
|
continue
|
|
1284
|
+
if name not in code and not _PY_NAMELESS_MATCH_RE.match(code):
|
|
1285
|
+
continue # no rule below can match it: spares a scan per name
|
|
1093
1286
|
top = _is_column_0(text)
|
|
1094
1287
|
if (
|
|
1095
1288
|
decl.match(code)
|
|
@@ -1114,27 +1307,103 @@ def _python_definition_lines(
|
|
|
1114
1307
|
continue
|
|
1115
1308
|
rest = m.group(1)
|
|
1116
1309
|
if from_import and rest.strip() == "*":
|
|
1117
|
-
|
|
1310
|
+
if not declarations_only:
|
|
1311
|
+
star_imports.append((number, top))
|
|
1118
1312
|
continue
|
|
1119
1313
|
if from_import and rest.lstrip().startswith("(") and ")" not in rest:
|
|
1120
1314
|
cont = ("paren", top, True)
|
|
1121
1315
|
elif rest.rstrip().endswith("\\"):
|
|
1122
1316
|
cont = ("backslash", top, from_import)
|
|
1123
1317
|
rest = rest.rstrip()[:-1]
|
|
1124
|
-
if _py_import_binds(rest, name, from_import):
|
|
1318
|
+
if _py_import_binds(rest, name, from_import, declarations_only):
|
|
1125
1319
|
found.append((number, top))
|
|
1126
1320
|
return found or star_imports
|
|
1127
1321
|
|
|
1128
1322
|
|
|
1129
|
-
def
|
|
1323
|
+
def _py_overloaded(lines: list[str], number: int, starts: set[int] | None) -> bool:
|
|
1324
|
+
"""True when the `def` on line `number` has `@overload` or
|
|
1325
|
+
`@<module>.overload` (`@typing.overload`, `@t.overload`) in its
|
|
1326
|
+
contiguous decorator block: the decorator lines at its indentation
|
|
1327
|
+
right above it, up to a blank line or a line that is not a decorator.
|
|
1328
|
+
A comment line is passed over, and so, in a file that tokenizes, is a
|
|
1329
|
+
line on which no statement begins (a decorator's arguments over
|
|
1330
|
+
several lines)."""
|
|
1331
|
+
text = lines[number - 1]
|
|
1332
|
+
indent = text[: len(text) - len(text.lstrip())]
|
|
1333
|
+
i = number - 1
|
|
1334
|
+
while i >= 1:
|
|
1335
|
+
text = lines[i - 1]
|
|
1336
|
+
code = text.split("#", 1)[0]
|
|
1337
|
+
if not text.strip():
|
|
1338
|
+
return False
|
|
1339
|
+
if not code.strip() or (starts is not None and i not in starts):
|
|
1340
|
+
i -= 1
|
|
1341
|
+
continue
|
|
1342
|
+
if not code.lstrip().startswith("@") or text[: len(text) - len(text.lstrip())] != indent:
|
|
1343
|
+
return False
|
|
1344
|
+
if _PY_OVERLOAD_RE.match(code):
|
|
1345
|
+
return True
|
|
1346
|
+
i -= 1
|
|
1347
|
+
return False
|
|
1348
|
+
|
|
1349
|
+
|
|
1350
|
+
def _py_enclosing_line(
|
|
1351
|
+
lines: list[str], number: int, indent: str, starts: set[int] | None
|
|
1352
|
+
) -> int | None:
|
|
1353
|
+
"""The line of the block that holds the statement on line `number`,
|
|
1354
|
+
indented by `indent`: the nearest line above it where a statement
|
|
1355
|
+
begins with less indentation. None at module level."""
|
|
1356
|
+
i = number - 1
|
|
1357
|
+
while indent and i >= 1:
|
|
1358
|
+
text = lines[i - 1]
|
|
1359
|
+
if text.split("#", 1)[0].strip() and (starts is None or i in starts):
|
|
1360
|
+
if len(text) - len(text.lstrip()) < len(indent):
|
|
1361
|
+
return i
|
|
1362
|
+
i -= 1
|
|
1363
|
+
return None
|
|
1364
|
+
|
|
1365
|
+
|
|
1366
|
+
def _python_overload_start(
|
|
1367
|
+
lines: list[str], name: str, defs: list[int], starts: set[int] | None
|
|
1368
|
+
) -> int | None:
|
|
1369
|
+
"""The first of `defs` when they are one `@overload` group: two lines
|
|
1370
|
+
or more, each a `def` / `async def NAME` line, all at one indentation
|
|
1371
|
+
in one enclosing block (`_py_enclosing_line`, so two classes'
|
|
1372
|
+
methods never make one group), and each but at most the last
|
|
1373
|
+
decorated as an overload (`_py_overloaded`). None otherwise."""
|
|
1374
|
+
if len(defs) < 2:
|
|
1375
|
+
return None
|
|
1376
|
+
decl = re.compile(rf"^(\s*)(?:async\s+def|def)\s+{re.escape(name)}(?!\w)")
|
|
1377
|
+
indents = set()
|
|
1378
|
+
for number in defs:
|
|
1379
|
+
m = decl.match(lines[number - 1].split("#", 1)[0])
|
|
1380
|
+
if m is None:
|
|
1381
|
+
return None
|
|
1382
|
+
indents.add(m.group(1))
|
|
1383
|
+
if len(indents) > 1:
|
|
1384
|
+
return None
|
|
1385
|
+
(indent,) = indents
|
|
1386
|
+
if len({_py_enclosing_line(lines, number, indent, starts) for number in defs}) > 1:
|
|
1387
|
+
return None
|
|
1388
|
+
if not all(_py_overloaded(lines, number, starts) for number in defs[:-1]):
|
|
1389
|
+
return None
|
|
1390
|
+
return defs[0]
|
|
1391
|
+
|
|
1392
|
+
|
|
1393
|
+
def _js_export_item_binds(item: str, name: str, alias_only: bool = False) -> bool:
|
|
1130
1394
|
"""True when one `export { ... }` item exposes `name`: the name after
|
|
1131
|
-
`as`, else the item's own name, an identifier or
|
|
1395
|
+
`as`, else (unless `alias_only`) the item's own name, an identifier or
|
|
1396
|
+
a string. With `alias_only`, an `as` that keeps the item's own name
|
|
1397
|
+
(`NAME as NAME`) does not count either."""
|
|
1132
1398
|
m = _JS_EXPORT_ITEM_RE.match(item.strip())
|
|
1133
|
-
if not m:
|
|
1399
|
+
if not m or (alias_only and m.group(2) is None):
|
|
1400
|
+
return False
|
|
1401
|
+
own, exposed = (
|
|
1402
|
+
s[1:-1] if s[:1] in ("'", '"') else s
|
|
1403
|
+
for s in (m.group(1), m.group(2) or m.group(1))
|
|
1404
|
+
)
|
|
1405
|
+
if alias_only and exposed == own:
|
|
1134
1406
|
return False
|
|
1135
|
-
exposed = m.group(2) or m.group(1)
|
|
1136
|
-
if exposed[:1] in ("'", '"'):
|
|
1137
|
-
exposed = exposed[1:-1]
|
|
1138
1407
|
return exposed == name
|
|
1139
1408
|
|
|
1140
1409
|
|
|
@@ -1321,46 +1590,364 @@ def _js_paren_tail_is_call(after_open: str) -> bool:
|
|
|
1321
1590
|
return False
|
|
1322
1591
|
|
|
1323
1592
|
|
|
1593
|
+
def _js_pattern_bindings(
|
|
1594
|
+
text: str, start: int
|
|
1595
|
+
) -> tuple[list[tuple[str, int]], int | None]:
|
|
1596
|
+
"""The names the destructuring pattern whose `{` or `[` is at
|
|
1597
|
+
`text[start]` binds, each with its offset in `text`, and the offset
|
|
1598
|
+
just past the pattern's closing bracket.
|
|
1599
|
+
|
|
1600
|
+
An object pattern's property is `...target`, a shorthand key or
|
|
1601
|
+
`key: target` (a key is a name, a string, a number or a computed
|
|
1602
|
+
`[...]`, and only a shorthand key binds itself); an array pattern's
|
|
1603
|
+
element is `...target`, a target or a hole; a target is a name or a
|
|
1604
|
+
nested pattern. A `= default` is skipped to the next comma at the
|
|
1605
|
+
pattern's depth, and so is a property or element of any other shape,
|
|
1606
|
+
which binds nothing. '...', "..." and `...` strings are skipped
|
|
1607
|
+
whole, so no bracket or comma inside one counts. Whitespace, newlines
|
|
1608
|
+
included, may sit anywhere between the parts.
|
|
1609
|
+
|
|
1610
|
+
Returns ([], None) when `text[start]` opens no pattern, the pattern
|
|
1611
|
+
does not close in `text`, a string in it does not end, or a bracket in
|
|
1612
|
+
it closes one it did not open. A pure function of its arguments (no
|
|
1613
|
+
module state), so another script can carry a copy of it.
|
|
1614
|
+
"""
|
|
1615
|
+
n = len(text)
|
|
1616
|
+
names: list[tuple[str, int]] = []
|
|
1617
|
+
|
|
1618
|
+
def space(i: int) -> int:
|
|
1619
|
+
while i < n and text[i].isspace():
|
|
1620
|
+
i += 1
|
|
1621
|
+
return i
|
|
1622
|
+
|
|
1623
|
+
def name_end(i: int) -> int:
|
|
1624
|
+
"""The end of the identifier at `i`, `i` itself when none starts there."""
|
|
1625
|
+
if i < n and (text[i].isalpha() or text[i] in "_$"):
|
|
1626
|
+
i += 1
|
|
1627
|
+
while i < n and (text[i].isalnum() or text[i] in "_$"):
|
|
1628
|
+
i += 1
|
|
1629
|
+
return i
|
|
1630
|
+
|
|
1631
|
+
def string_end(i: int) -> int | None:
|
|
1632
|
+
"""The offset past the string whose quote is at `i`."""
|
|
1633
|
+
quote = text[i]
|
|
1634
|
+
i += 1
|
|
1635
|
+
while i < n:
|
|
1636
|
+
if text[i] == "\\":
|
|
1637
|
+
i += 2
|
|
1638
|
+
elif text[i] == quote:
|
|
1639
|
+
return i + 1
|
|
1640
|
+
else:
|
|
1641
|
+
i += 1
|
|
1642
|
+
return None
|
|
1643
|
+
|
|
1644
|
+
def skip(i: int, to_comma: bool = True) -> int | None:
|
|
1645
|
+
"""From `i`, the offset of the next comma (unless not `to_comma`)
|
|
1646
|
+
or closing bracket at depth 0; with `to_comma` False, `text[i]`
|
|
1647
|
+
opens a bracket and the offset past the one closing it."""
|
|
1648
|
+
depth = 0
|
|
1649
|
+
while i < n:
|
|
1650
|
+
ch = text[i]
|
|
1651
|
+
if ch in "'\"`":
|
|
1652
|
+
end = string_end(i)
|
|
1653
|
+
if end is None:
|
|
1654
|
+
return None
|
|
1655
|
+
i = end
|
|
1656
|
+
continue
|
|
1657
|
+
if ch in "([{":
|
|
1658
|
+
depth += 1
|
|
1659
|
+
elif ch in ")]}":
|
|
1660
|
+
if depth == 0:
|
|
1661
|
+
return i
|
|
1662
|
+
depth -= 1
|
|
1663
|
+
if depth == 0 and not to_comma:
|
|
1664
|
+
return i + 1
|
|
1665
|
+
elif ch == "," and depth == 0 and to_comma:
|
|
1666
|
+
return i
|
|
1667
|
+
i += 1
|
|
1668
|
+
return None
|
|
1669
|
+
|
|
1670
|
+
def default(i: int) -> int | None:
|
|
1671
|
+
"""After a target: past its `= default`, when it has one."""
|
|
1672
|
+
i = space(i)
|
|
1673
|
+
if i < n and text[i] == "=" and text[i + 1 : i + 2] not in ("=", ">"):
|
|
1674
|
+
return skip(i + 1)
|
|
1675
|
+
return i
|
|
1676
|
+
|
|
1677
|
+
def bind(i: int, end: int, close: str) -> int | None:
|
|
1678
|
+
"""The name `text[i:end]` is bound when its property or element
|
|
1679
|
+
ends after it or takes a default (any other text: another shape)."""
|
|
1680
|
+
after = space(end)
|
|
1681
|
+
if after < n and (
|
|
1682
|
+
text[after] in (",", close)
|
|
1683
|
+
or text[after] == "=" and text[after + 1 : after + 2] not in ("=", ">")
|
|
1684
|
+
):
|
|
1685
|
+
names.append((text[i:end], i))
|
|
1686
|
+
return default(after)
|
|
1687
|
+
return after
|
|
1688
|
+
|
|
1689
|
+
def target(i: int, close: str) -> int | None:
|
|
1690
|
+
if i < n and text[i] in "{[":
|
|
1691
|
+
end = pattern(i)
|
|
1692
|
+
return None if end is None else default(end)
|
|
1693
|
+
end = name_end(i)
|
|
1694
|
+
return i if end == i else bind(i, end, close)
|
|
1695
|
+
|
|
1696
|
+
def element(i: int, close: str) -> int | None:
|
|
1697
|
+
"""One property or element at `i`: the offset after it."""
|
|
1698
|
+
if text.startswith("...", i):
|
|
1699
|
+
return target(space(i + 3), close)
|
|
1700
|
+
if close == "]":
|
|
1701
|
+
return target(i, close)
|
|
1702
|
+
if text[i] in "'\"":
|
|
1703
|
+
end = string_end(i)
|
|
1704
|
+
elif text[i] == "[":
|
|
1705
|
+
end = skip(i, to_comma=False) # a computed key
|
|
1706
|
+
else:
|
|
1707
|
+
end = name_end(i)
|
|
1708
|
+
if end > i:
|
|
1709
|
+
after = space(end)
|
|
1710
|
+
if after < n and text[after] == ":":
|
|
1711
|
+
return target(space(after + 1), close)
|
|
1712
|
+
return bind(i, end, close) # a shorthand key
|
|
1713
|
+
while end < n and (text[end].isalnum() or text[end] in "._"):
|
|
1714
|
+
end += 1 # a number key
|
|
1715
|
+
if end is None:
|
|
1716
|
+
return None
|
|
1717
|
+
after = space(end)
|
|
1718
|
+
if end > i and after < n and text[after] == ":":
|
|
1719
|
+
return target(space(after + 1), close)
|
|
1720
|
+
return after
|
|
1721
|
+
|
|
1722
|
+
def pattern(i: int) -> int | None:
|
|
1723
|
+
"""The pattern opening at `i`: the offset past its closing bracket."""
|
|
1724
|
+
close = "}" if text[i] == "{" else "]"
|
|
1725
|
+
i += 1
|
|
1726
|
+
while True:
|
|
1727
|
+
i = space(i)
|
|
1728
|
+
if i >= n:
|
|
1729
|
+
return None
|
|
1730
|
+
if text[i] == close:
|
|
1731
|
+
return i + 1
|
|
1732
|
+
if text[i] in ")]}":
|
|
1733
|
+
return None
|
|
1734
|
+
if text[i] == ",":
|
|
1735
|
+
i += 1 # a hole, or a stray comma
|
|
1736
|
+
continue
|
|
1737
|
+
end = element(i, close)
|
|
1738
|
+
if end is None:
|
|
1739
|
+
return None
|
|
1740
|
+
end = space(end)
|
|
1741
|
+
if end < n and text[end] not in (",", close):
|
|
1742
|
+
end = skip(end) # any other shape binds nothing
|
|
1743
|
+
if end is None:
|
|
1744
|
+
return None
|
|
1745
|
+
i = end + 1 if end < n and text[end] == "," else end
|
|
1746
|
+
|
|
1747
|
+
if not (0 <= start < n and text[start] in "{["):
|
|
1748
|
+
return [], None
|
|
1749
|
+
end = pattern(start)
|
|
1750
|
+
return (names, end) if end is not None else ([], None)
|
|
1751
|
+
|
|
1752
|
+
|
|
1753
|
+
def _js_initializer_start(text: str, end: int) -> int | None:
|
|
1754
|
+
"""The offset just past the `=` that begins the initializer of a
|
|
1755
|
+
declarator whose name or pattern ends at `end`, a type annotation
|
|
1756
|
+
(`: T`, brackets, strings and `=>` in it included) skipped. None when
|
|
1757
|
+
neither `=` nor `:` follows, or when a `,` or `;` outside the
|
|
1758
|
+
annotation's brackets, or the end of `text`, comes first."""
|
|
1759
|
+
n = len(text)
|
|
1760
|
+
i = end
|
|
1761
|
+
while i < n and text[i].isspace():
|
|
1762
|
+
i += 1
|
|
1763
|
+
if i < n and text[i] == "=":
|
|
1764
|
+
return None if text[i + 1 : i + 2] == "=" else i + 1
|
|
1765
|
+
if not (i < n and text[i] == ":"):
|
|
1766
|
+
return None
|
|
1767
|
+
i += 1
|
|
1768
|
+
depth = 0
|
|
1769
|
+
while i < n:
|
|
1770
|
+
ch = text[i]
|
|
1771
|
+
if ch in "'\"`":
|
|
1772
|
+
i += 1
|
|
1773
|
+
while i < n and text[i] != ch:
|
|
1774
|
+
i += 2 if text[i] == "\\" else 1
|
|
1775
|
+
elif text.startswith("=>", i):
|
|
1776
|
+
i += 1
|
|
1777
|
+
elif ch in "([{<":
|
|
1778
|
+
depth += 1
|
|
1779
|
+
elif ch in ")]}>":
|
|
1780
|
+
depth -= 1
|
|
1781
|
+
if depth < 0:
|
|
1782
|
+
return None
|
|
1783
|
+
elif ch in ",;" and depth == 0:
|
|
1784
|
+
return None
|
|
1785
|
+
elif ch == "=" and depth == 0:
|
|
1786
|
+
return None if text[i + 1 : i + 2] == "=" else i + 1
|
|
1787
|
+
i += 1
|
|
1788
|
+
return None
|
|
1789
|
+
|
|
1790
|
+
|
|
1791
|
+
def _js_destructured(code_lines: list[str]) -> dict[int, _PatternDecl]:
|
|
1792
|
+
"""For each line that opens a `const` / `let` / `var` statement whose
|
|
1793
|
+
first declarator is a destructuring pattern (`_JS_VAR_DECL_RE`, the
|
|
1794
|
+
rest starting with `{` or `[`): the names the pattern binds, each with
|
|
1795
|
+
the line it appears on, and where its initializer starts.
|
|
1796
|
+
|
|
1797
|
+
The patterns are read from the comment-free lines joined with `\\n`
|
|
1798
|
+
(`_js_pattern_bindings`), so a pattern may span lines and a name's
|
|
1799
|
+
offset maps back to its line. A pattern that does not close, or that
|
|
1800
|
+
nests deeper than the interpreter's recursion limit, binds nothing and
|
|
1801
|
+
gives no item.
|
|
1802
|
+
"""
|
|
1803
|
+
starts: list[tuple[int, int]] = []
|
|
1804
|
+
for number, code in enumerate(code_lines, start=1):
|
|
1805
|
+
if "{" not in code and "[" not in code:
|
|
1806
|
+
continue
|
|
1807
|
+
m = _JS_VAR_DECL_RE.match(code)
|
|
1808
|
+
if m is not None and m.group(1)[:1] in ("{", "["):
|
|
1809
|
+
starts.append((number, m.start(1)))
|
|
1810
|
+
if not starts:
|
|
1811
|
+
return {}
|
|
1812
|
+
text = "\n".join(code_lines)
|
|
1813
|
+
offsets: list[int] = []
|
|
1814
|
+
pos = 0
|
|
1815
|
+
for code in code_lines:
|
|
1816
|
+
offsets.append(pos)
|
|
1817
|
+
pos += len(code) + 1
|
|
1818
|
+
|
|
1819
|
+
def line_of(offset: int) -> int:
|
|
1820
|
+
return bisect.bisect_right(offsets, offset)
|
|
1821
|
+
|
|
1822
|
+
out: dict[int, _PatternDecl] = {}
|
|
1823
|
+
for number, column in starts:
|
|
1824
|
+
try:
|
|
1825
|
+
names, end = _js_pattern_bindings(text, offsets[number - 1] + column)
|
|
1826
|
+
except RecursionError:
|
|
1827
|
+
continue # patterns nested past the interpreter's limit: none bound
|
|
1828
|
+
if end is None:
|
|
1829
|
+
continue
|
|
1830
|
+
value = _js_initializer_start(text, end)
|
|
1831
|
+
at = None
|
|
1832
|
+
if value is not None:
|
|
1833
|
+
line = line_of(value)
|
|
1834
|
+
at = (line, code_lines[line - 1][value - offsets[line - 1] :])
|
|
1835
|
+
out[number] = ([(name, line_of(offset)) for name, offset in names], at)
|
|
1836
|
+
return out
|
|
1837
|
+
|
|
1838
|
+
|
|
1324
1839
|
def _tsjs_definition_lines(
|
|
1325
1840
|
lines: list[str],
|
|
1326
1841
|
code_lines: list[str],
|
|
1327
1842
|
name: str,
|
|
1328
1843
|
dotted: bool,
|
|
1329
1844
|
declarators: Callable[..., _DeclSplit] = _js_declarators,
|
|
1845
|
+
declarations_only: bool = False,
|
|
1846
|
+
quiet: list[bool] | None = None,
|
|
1847
|
+
destructured: dict[int, _PatternDecl] | None = None,
|
|
1330
1848
|
) -> list[tuple[int, bool]]:
|
|
1331
1849
|
"""(line, at column 0) for every line that defines `name`.
|
|
1332
1850
|
|
|
1333
|
-
`
|
|
1851
|
+
`quiet` is `_js_quiet_lines(code_lines)` when the caller has it (it does
|
|
1852
|
+
not depend on the name): a quiet line without the name is passed over
|
|
1853
|
+
while no statement is open. `code_lines` are `lines` with comments removed (`_js_code_lines`), so a
|
|
1334
1854
|
JSDoc line or a commented-out declaration never matches. A line inside
|
|
1335
1855
|
a multi-line `export { ... }` list or `module.exports = { ... }` object
|
|
1336
1856
|
takes the column of the line the statement starts on, and so do the
|
|
1337
|
-
name line of a declaration whose keyword ends the line above it
|
|
1338
|
-
later declarator on a line of its own
|
|
1857
|
+
name line of a declaration whose keyword ends the line above it, a
|
|
1858
|
+
later declarator on a line of its own and the line of a name that a
|
|
1859
|
+
destructuring pattern binds. `destructured` is
|
|
1860
|
+
`_js_destructured(code_lines)` when the caller has it (it does not
|
|
1861
|
+
depend on the name either): a name its pattern binds counts when the
|
|
1862
|
+
`const` / `let` / `var` line it is keyed by opens a statement here.
|
|
1863
|
+
Class members count only for a
|
|
1339
1864
|
dotted export name. `declarators` splits a declarator list
|
|
1340
1865
|
(`_js_declarators`, or `_SourceCache.js_declarators`, which splits each
|
|
1341
|
-
line once for every export looked up).
|
|
1866
|
+
line once for every export looked up). With `declarations_only`,
|
|
1867
|
+
`export import NAME =`, `export default NAME`, an export-list item
|
|
1868
|
+
without an `as` that renames, a `module.exports` shorthand key and an
|
|
1869
|
+
import-valued binding (a declarator, a destructured name, `exports.NAME =`
|
|
1870
|
+
or a `module.exports` key whose value starts with `require(` or an
|
|
1871
|
+
`(await) import(`, on its line or the next one) never count, nor does
|
|
1872
|
+
a class member.
|
|
1342
1873
|
"""
|
|
1343
1874
|
esc = re.escape(name)
|
|
1344
1875
|
end = r"(?![\w$])"
|
|
1345
|
-
statements =
|
|
1876
|
+
statements = [
|
|
1346
1877
|
re.compile(rf"^\s*{_JS_MODIFIERS}{_JS_KINDS}{esc}{end}"),
|
|
1347
|
-
re.compile(rf"^\s*export\s+import\s+(?:type\s+)?{esc}\s*=(?![=>])"),
|
|
1348
1878
|
re.compile(
|
|
1349
1879
|
rf"^\s*export\s+(?:type\s+)?\*\s+as\s+"
|
|
1350
1880
|
rf"(?:{esc}\s+|([\"']){esc}\1\s*)from{end}"
|
|
1351
1881
|
),
|
|
1352
|
-
re.compile(rf"^\s*export\s+default\s+{esc}\s*;?\s*$"),
|
|
1353
1882
|
re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])"),
|
|
1354
|
-
|
|
1883
|
+
]
|
|
1884
|
+
if not declarations_only:
|
|
1885
|
+
statements += [
|
|
1886
|
+
re.compile(rf"^\s*export\s+import\s+(?:type\s+)?{esc}\s*=(?![=>])"),
|
|
1887
|
+
re.compile(rf"^\s*export\s+default\s+{esc}\s*;?\s*$"),
|
|
1888
|
+
]
|
|
1355
1889
|
# One declarator of a `const` / `let` / `var` list: `NAME = v`,
|
|
1356
1890
|
# `NAME: T = v`, `NAME!: T`, or a bare `NAME`. A destructuring pattern
|
|
1357
|
-
# starts with `{` or `[` and never matches
|
|
1891
|
+
# starts with `{` or `[` and never matches: the names a pattern that
|
|
1892
|
+
# begins the list binds come from `destructured` (`_js_pattern_bindings`).
|
|
1358
1893
|
declarator = re.compile(rf"^\s*{esc}{end}\s*(?:!?\s*:|=|$)")
|
|
1894
|
+
# With declarations_only, the value bound to NAME (the rest of a
|
|
1895
|
+
# declarator, of `exports.NAME =` or of a `module.exports` key), to
|
|
1896
|
+
# test for an import.
|
|
1897
|
+
# (A type annotation holds no `=` but in `=>`: `NAME: () => T = v`.)
|
|
1898
|
+
declarator_value = re.compile(
|
|
1899
|
+
rf"^\s*{esc}{end}\s*(?:!?\s*:(?:[^=]|=>)*)?=(?![=>])(.*)$"
|
|
1900
|
+
)
|
|
1901
|
+
var_statement = re.compile(rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(.*)$")
|
|
1902
|
+
exports_value = re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])(.*)$")
|
|
1903
|
+
key_value = re.compile(rf"^\s*(?:(['\"]){esc}\1|{esc}{end})\s*:(.*)$")
|
|
1904
|
+
|
|
1905
|
+
def imported(value: str, number: int) -> bool:
|
|
1906
|
+
"""True when a bound value is an import: `require(...)`, or an
|
|
1907
|
+
`import(...)` awaited or not, alone or in one helper call. A value
|
|
1908
|
+
that goes on on the next line (`NAME =`, `NAME = (` or
|
|
1909
|
+
`NAME = helper(` ends line `number`) is read there too."""
|
|
1910
|
+
if _JS_IMPORT_VALUE_OPEN_RE.match(value):
|
|
1911
|
+
value += next((c for c in code_lines[number:] if c.strip()), "")
|
|
1912
|
+
return _JS_IMPORT_VALUE_RE.match(value) is not None
|
|
1913
|
+
|
|
1914
|
+
def declares(piece: str, number: int) -> bool:
|
|
1915
|
+
if declarator.match(piece) is None:
|
|
1916
|
+
return False
|
|
1917
|
+
if not declarations_only:
|
|
1918
|
+
return True
|
|
1919
|
+
m = declarator_value.match(piece)
|
|
1920
|
+
return m is None or not imported(m.group(1), number)
|
|
1921
|
+
|
|
1922
|
+
def is_statement(code: str, number: int) -> bool:
|
|
1923
|
+
if not any(p.match(code) for p in statements):
|
|
1924
|
+
return False
|
|
1925
|
+
if not declarations_only:
|
|
1926
|
+
return True
|
|
1927
|
+
m = var_statement.match(code)
|
|
1928
|
+
if m is not None:
|
|
1929
|
+
m = declarator_value.match(m.group(1))
|
|
1930
|
+
else:
|
|
1931
|
+
m = exports_value.match(code)
|
|
1932
|
+
return m is None or not imported(m.group(1), number)
|
|
1933
|
+
|
|
1359
1934
|
leading_name = re.compile(rf"^\s*{esc}{end}")
|
|
1935
|
+
# A key with a value, a method, or (unless declarations_only) a shorthand.
|
|
1936
|
+
key_end = r"[:(]" if declarations_only else r"[:(]|$"
|
|
1360
1937
|
object_key = re.compile(
|
|
1361
1938
|
rf"^\s*(?:(?:async|get|set)\s+|\*\s*)*(?:(['\"]){esc}\1|{esc}{end})"
|
|
1362
|
-
rf"\s*(?:
|
|
1939
|
+
rf"\s*(?:{key_end})"
|
|
1363
1940
|
)
|
|
1941
|
+
|
|
1942
|
+
def binds_key(piece: str, number: int) -> bool:
|
|
1943
|
+
if object_key.match(piece) is None:
|
|
1944
|
+
return False
|
|
1945
|
+
if not declarations_only:
|
|
1946
|
+
return True
|
|
1947
|
+
m = key_value.match(piece)
|
|
1948
|
+
return m is None or not imported(m.group(2), number)
|
|
1949
|
+
|
|
1950
|
+
dotted = dotted and not declarations_only
|
|
1364
1951
|
member = re.compile(
|
|
1365
1952
|
rf"^\s+(?P<mods>{_JS_MEMBER_MODIFIERS})(?:\*\s*)?{esc}{end}\s*[?!]?\s*"
|
|
1366
1953
|
rf"(?:<[^>]*>\s*)?(?P<tail>[(:=;]|$)"
|
|
@@ -1375,7 +1962,27 @@ def _tsjs_definition_lines(
|
|
|
1375
1962
|
# nested in another's initializer included: (column 0, brackets left
|
|
1376
1963
|
# open, how, as `_js_declarators` gives them).
|
|
1377
1964
|
decls: list[tuple[bool, str, str]] = []
|
|
1965
|
+
if quiet is None:
|
|
1966
|
+
quiet = _js_quiet_lines(code_lines)
|
|
1967
|
+
if destructured is None:
|
|
1968
|
+
destructured = _js_destructured(code_lines)
|
|
1969
|
+
|
|
1970
|
+
def pattern_binds(number: int) -> list[int]:
|
|
1971
|
+
"""The lines on which the pattern that begins the `const` / `let` /
|
|
1972
|
+
`var` list of line `number` binds `name` (none with
|
|
1973
|
+
`declarations_only` when its value is an import)."""
|
|
1974
|
+
decl = destructured.get(number)
|
|
1975
|
+
if decl is None:
|
|
1976
|
+
return []
|
|
1977
|
+
bound, value = decl
|
|
1978
|
+
at = [line for bound_name, line in bound if bound_name == name]
|
|
1979
|
+
if at and declarations_only and value is not None and imported(value[1], value[0]):
|
|
1980
|
+
return []
|
|
1981
|
+
return at
|
|
1982
|
+
|
|
1378
1983
|
for number, (text, code) in enumerate(zip(lines, code_lines), start=1):
|
|
1984
|
+
if quiet[number - 1] and not (decls or split or cont) and name not in code:
|
|
1985
|
+
continue # no rule below can match it: spares a scan per name
|
|
1379
1986
|
# A line of a list carried from above still goes through the rules
|
|
1380
1987
|
# below: it may hold a class member or a local of an arrow function.
|
|
1381
1988
|
if decls and code.strip():
|
|
@@ -1387,7 +1994,7 @@ def _tsjs_definition_lines(
|
|
|
1387
1994
|
carry, pieces = declarators(code, opened, how == _DECL_NEXT)
|
|
1388
1995
|
if carry is not None:
|
|
1389
1996
|
carried.append((decl_top, *carry))
|
|
1390
|
-
if any(
|
|
1997
|
+
if any(declares(p, number) for p in pieces):
|
|
1391
1998
|
# a later declarator on a line of its own, as Prettier
|
|
1392
1999
|
# writes it: `export const a = 1,` then ` NAME = 2;`
|
|
1393
2000
|
found.append((number, decl_top))
|
|
@@ -1399,7 +2006,7 @@ def _tsjs_definition_lines(
|
|
|
1399
2006
|
carry, pieces = declarators(code)
|
|
1400
2007
|
if carry is not None:
|
|
1401
2008
|
decls.append((top, *carry))
|
|
1402
|
-
hit = any(
|
|
2009
|
+
hit = any(declares(p, number) for p in pieces)
|
|
1403
2010
|
else:
|
|
1404
2011
|
hit = leading_name.match(code) is not None
|
|
1405
2012
|
if hit:
|
|
@@ -1411,14 +2018,19 @@ def _tsjs_definition_lines(
|
|
|
1411
2018
|
depth, pieces = _js_depth1_pieces(code, depth)
|
|
1412
2019
|
cont = (kind, top, depth) if depth > 0 else None
|
|
1413
2020
|
if kind == "list":
|
|
1414
|
-
hit = any(
|
|
2021
|
+
hit = any(
|
|
2022
|
+
_js_export_item_binds(p, name, declarations_only) for p in pieces
|
|
2023
|
+
)
|
|
1415
2024
|
else:
|
|
1416
|
-
hit = any(
|
|
2025
|
+
hit = any(binds_key(p, number) for p in pieces)
|
|
1417
2026
|
if hit:
|
|
1418
2027
|
found.append((number, top))
|
|
1419
2028
|
continue
|
|
1420
2029
|
top = _is_column_0(text)
|
|
1421
|
-
|
|
2030
|
+
# every rule that matches a line needs the name in it, but for a
|
|
2031
|
+
# destructuring pattern, whose names may sit on the lines below
|
|
2032
|
+
named = name in code
|
|
2033
|
+
if named and is_statement(code, number):
|
|
1422
2034
|
found.append((number, top))
|
|
1423
2035
|
continue
|
|
1424
2036
|
m = _JS_VAR_DECL_RE.match(code)
|
|
@@ -1426,7 +2038,10 @@ def _tsjs_definition_lines(
|
|
|
1426
2038
|
carry, pieces = declarators(m.group(1))
|
|
1427
2039
|
if carry is not None:
|
|
1428
2040
|
decls.append((top, *carry))
|
|
1429
|
-
|
|
2041
|
+
# `export const { a, b: NAME } = obj;`, on one line or several:
|
|
2042
|
+
# each name's line, at the statement's column
|
|
2043
|
+
found.extend((line, top) for line in pattern_binds(number))
|
|
2044
|
+
if named and any(declares(p, number) for p in pieces):
|
|
1430
2045
|
# a later declarator: `export const a = 1, NAME = 2;`
|
|
1431
2046
|
found.append((number, top))
|
|
1432
2047
|
continue
|
|
@@ -1439,7 +2054,7 @@ def _tsjs_definition_lines(
|
|
|
1439
2054
|
depth, pieces = _js_depth1_pieces(m.group(1), 1)
|
|
1440
2055
|
if depth > 0:
|
|
1441
2056
|
cont = ("list", top, depth)
|
|
1442
|
-
if any(_js_export_item_binds(p, name) for p in pieces):
|
|
2057
|
+
if any(_js_export_item_binds(p, name, declarations_only) for p in pieces):
|
|
1443
2058
|
found.append((number, top))
|
|
1444
2059
|
continue
|
|
1445
2060
|
m = _JS_MODULE_EXPORTS_OBJECT_RE.match(code)
|
|
@@ -1447,7 +2062,7 @@ def _tsjs_definition_lines(
|
|
|
1447
2062
|
depth, pieces = _js_depth1_pieces(m.group(1), 1)
|
|
1448
2063
|
if depth > 0:
|
|
1449
2064
|
cont = ("object", top, depth)
|
|
1450
|
-
if any(
|
|
2065
|
+
if any(binds_key(p, number) for p in pieces):
|
|
1451
2066
|
found.append((number, top))
|
|
1452
2067
|
continue
|
|
1453
2068
|
if not dotted:
|
|
@@ -1494,12 +2109,19 @@ def _definition_matches(
|
|
|
1494
2109
|
export_name: str,
|
|
1495
2110
|
source_root: Path,
|
|
1496
2111
|
cache: _SourceCache | None = None,
|
|
2112
|
+
declarations_only: bool = False,
|
|
1497
2113
|
) -> tuple[list[int], set[int]] | None:
|
|
1498
2114
|
"""(definition lines to report, every matching line) for an export.
|
|
1499
2115
|
|
|
1500
2116
|
The first item applies the column-0 rule; the second keeps the indented
|
|
1501
2117
|
matches that rule dropped. None when the file's extension has no
|
|
1502
2118
|
definition rule or the file cannot be read.
|
|
2119
|
+
|
|
2120
|
+
`declarations_only` (classify-stale's `defined_at` lookup) keeps the
|
|
2121
|
+
lines that declare the name at column 0, both items alike: no plain
|
|
2122
|
+
import, star import or re-export (`_python_definition_lines` and
|
|
2123
|
+
`_tsjs_definition_lines` say which), and nothing in a Python file that
|
|
2124
|
+
does not tokenize.
|
|
1503
2125
|
"""
|
|
1504
2126
|
rel = source_file.strip().replace("\\", "/")
|
|
1505
2127
|
suffix = Path(rel).suffix.lower()
|
|
@@ -1510,18 +2132,39 @@ def _definition_matches(
|
|
|
1510
2132
|
lines = cache.lines(path)
|
|
1511
2133
|
if lines is None:
|
|
1512
2134
|
return None
|
|
2135
|
+
return _line_matches(suffix, path, lines, export_name, cache, declarations_only)
|
|
2136
|
+
|
|
2137
|
+
|
|
2138
|
+
def _line_matches(
|
|
2139
|
+
suffix: str,
|
|
2140
|
+
path: Path,
|
|
2141
|
+
lines: list[str],
|
|
2142
|
+
export_name: str,
|
|
2143
|
+
cache: _SourceCache,
|
|
2144
|
+
declarations_only: bool = False,
|
|
2145
|
+
) -> tuple[list[int], set[int]]:
|
|
2146
|
+
"""`_definition_matches` over the lines of a Python or TS/JS file already
|
|
2147
|
+
read: `suffix`, the cited file's extension, picks the rules, and `path`
|
|
2148
|
+
keys `cache`."""
|
|
1513
2149
|
name = definition_name(export_name)
|
|
1514
2150
|
if not name:
|
|
1515
2151
|
return [], set()
|
|
1516
2152
|
dotted = "." in export_name
|
|
1517
2153
|
if suffix in PYTHON_EXTENSIONS:
|
|
1518
2154
|
starts = cache.py_statement_starts(path, lines)
|
|
1519
|
-
|
|
2155
|
+
if declarations_only and starts is None:
|
|
2156
|
+
# every line would be a candidate, a docstring line included
|
|
2157
|
+
return [], set()
|
|
2158
|
+
found = _python_definition_lines(lines, name, starts, declarations_only)
|
|
1520
2159
|
else:
|
|
1521
2160
|
code_lines = cache.js_code_lines(path, lines)
|
|
1522
2161
|
found = _tsjs_definition_lines(
|
|
1523
|
-
lines, code_lines, name, dotted, cache.js_declarators
|
|
2162
|
+
lines, code_lines, name, dotted, cache.js_declarators, declarations_only,
|
|
2163
|
+
cache.js_quiet_lines(path, code_lines), cache.js_destructured(path, code_lines),
|
|
1524
2164
|
)
|
|
2165
|
+
if declarations_only:
|
|
2166
|
+
top = {n for n, at_column_0 in found if at_column_0}
|
|
2167
|
+
return sorted(top), top
|
|
1525
2168
|
return _prefer_module_level(found, dotted), {n for n, _ in found}
|
|
1526
2169
|
|
|
1527
2170
|
|
|
@@ -1541,6 +2184,39 @@ def find_definition_lines(
|
|
|
1541
2184
|
return None if matches is None else matches[0]
|
|
1542
2185
|
|
|
1543
2186
|
|
|
2187
|
+
def _missed_definition_lines(
|
|
2188
|
+
source_file: str,
|
|
2189
|
+
export_name: str,
|
|
2190
|
+
source_root: Path,
|
|
2191
|
+
cache: _SourceCache,
|
|
2192
|
+
line: int,
|
|
2193
|
+
defs: list[int],
|
|
2194
|
+
every: set[int],
|
|
2195
|
+
) -> list[int]:
|
|
2196
|
+
"""The `definition_lines` a finding lists for a recorded `line` that is
|
|
2197
|
+
not one of `defs` (`_definition_matches` over the same `cache`).
|
|
2198
|
+
|
|
2199
|
+
A recorded line that is an indented match the column-0 rule dropped
|
|
2200
|
+
joins them, so the finding carries several lines and nobody moves it.
|
|
2201
|
+
Otherwise a Python file's `@overload` group gives its first line alone
|
|
2202
|
+
(`_python_overload_start`), the line to move the entry to, and any
|
|
2203
|
+
other answer gives `defs` as they are.
|
|
2204
|
+
"""
|
|
2205
|
+
if line in every:
|
|
2206
|
+
return sorted(set(defs) | {line})
|
|
2207
|
+
rel = source_file.strip().replace("\\", "/")
|
|
2208
|
+
if Path(rel).suffix.lower() not in PYTHON_EXTENSIONS or len(defs) < 2:
|
|
2209
|
+
return defs
|
|
2210
|
+
path = (source_root / rel).resolve()
|
|
2211
|
+
lines = cache.lines(path)
|
|
2212
|
+
if lines is None:
|
|
2213
|
+
return defs
|
|
2214
|
+
first = _python_overload_start(
|
|
2215
|
+
lines, definition_name(export_name), defs, cache.py_statement_starts(path, lines)
|
|
2216
|
+
)
|
|
2217
|
+
return defs if first is None else [first]
|
|
2218
|
+
|
|
2219
|
+
|
|
1544
2220
|
def definition_lines_report(
|
|
1545
2221
|
source_file: str,
|
|
1546
2222
|
export_name: str,
|
|
@@ -1553,9 +2229,11 @@ def definition_lines_report(
|
|
|
1553
2229
|
|
|
1554
2230
|
`line_check` is `file-missing`, `skipped-export-type` (a `module` or
|
|
1555
2231
|
`package`, no definition line), `skipped-language` (no rule for the
|
|
1556
|
-
extension) or `checked`. With `line`, a recorded line that is
|
|
1557
|
-
|
|
1558
|
-
`
|
|
2232
|
+
extension) or `checked`. With `line`, a recorded line that is not a
|
|
2233
|
+
definition line gets the lines `verify` would list
|
|
2234
|
+
(`_missed_definition_lines`): an indented match the column-0 rule
|
|
2235
|
+
dropped joins `definition_lines`, so the answer carries several lines,
|
|
2236
|
+
and a Python `@overload` group gives its first line alone.
|
|
1559
2237
|
"""
|
|
1560
2238
|
report: dict = {
|
|
1561
2239
|
"source_file": source_file,
|
|
@@ -1574,14 +2252,17 @@ def definition_lines_report(
|
|
|
1574
2252
|
covered = suffix in PYTHON_EXTENSIONS or suffix in TSJS_EXTENSIONS
|
|
1575
2253
|
report["line_check"] = "skipped-export-type" if covered else "skipped-language"
|
|
1576
2254
|
return report
|
|
1577
|
-
|
|
2255
|
+
cache = _SourceCache()
|
|
2256
|
+
matches = _definition_matches(source_file, export_name, source_root, cache)
|
|
1578
2257
|
if matches is None:
|
|
1579
2258
|
raise ValueError(f"failed to read source file {source_root / rel}")
|
|
1580
2259
|
defs, every = matches
|
|
1581
2260
|
if line is not None:
|
|
1582
2261
|
report["line_is_definition"] = line in defs
|
|
1583
|
-
if line not in defs
|
|
1584
|
-
defs =
|
|
2262
|
+
if line not in defs:
|
|
2263
|
+
defs = _missed_definition_lines(
|
|
2264
|
+
source_file, export_name, source_root, cache, line, defs, every
|
|
2265
|
+
)
|
|
1585
2266
|
report["definition_lines"] = defs
|
|
1586
2267
|
return report
|
|
1587
2268
|
|
|
@@ -1647,14 +2328,657 @@ def _stale_line_check(
|
|
|
1647
2328
|
return report["line_check"], report["definition_lines"]
|
|
1648
2329
|
|
|
1649
2330
|
|
|
1650
|
-
|
|
2331
|
+
# The folders and files the `defined_at` walk leaves out, judged
|
|
2332
|
+
# case-insensitively on the path below its walk root: generated, built and
|
|
2333
|
+
# installed output, a vendored copy of another package, tests and their
|
|
2334
|
+
# configuration, docs, examples and benchmarks.
|
|
2335
|
+
# Keep identical to EXCLUDED_DIR_NAMES in skf-resolve-authoritative-files.py.
|
|
2336
|
+
EXCLUDED_DIR_NAMES = {
|
|
2337
|
+
"node_modules", "__pycache__", "dist", "build", ".webpack",
|
|
2338
|
+
"target", ".next", ".nuxt", "out", "coverage", ".git",
|
|
2339
|
+
".venv", "venv", ".tox", ".mypy_cache", ".pytest_cache",
|
|
2340
|
+
".ruff_cache", ".gradle", ".idea", ".vscode",
|
|
2341
|
+
}
|
|
2342
|
+
# Keep identical to _VENDORED_SEGMENTS in skf-detect-language.py.
|
|
2343
|
+
_VENDORED_SEGMENTS = frozenset(
|
|
2344
|
+
{"vendor", "vendored", "third_party", "third-party", "thirdparty"}
|
|
2345
|
+
)
|
|
2346
|
+
# Keep identical to TEST_FOLDERS in skf-disqualify-candidates.py.
|
|
2347
|
+
TEST_FOLDERS = frozenset({"test", "tests", "__tests__", "spec", "specs"})
|
|
2348
|
+
# Keep identical to TEST_CONFIG_NAMES in skf-disqualify-candidates.py.
|
|
2349
|
+
TEST_CONFIG_NAMES = frozenset({
|
|
2350
|
+
"conftest.py", "pytest.ini", "phpunit.xml", "phpunit.xml.dist", ".rspec",
|
|
2351
|
+
})
|
|
2352
|
+
# Keep identical to TEST_CONFIG_PREFIXES in skf-disqualify-candidates.py.
|
|
2353
|
+
TEST_CONFIG_PREFIXES = (
|
|
2354
|
+
"jest.config.", "vitest.config.", "karma.conf.", "playwright.config.",
|
|
2355
|
+
"cypress.config.",
|
|
2356
|
+
)
|
|
2357
|
+
# Keep identical to TEST_FILE_RE in skf-disqualify-candidates.py.
|
|
2358
|
+
TEST_FILE_RE = re.compile(
|
|
2359
|
+
r"(\.(test|spec)\.[^./]+$)|(_test\.[^./]+$)|(^test_.+\.py$)|(_spec\.rb$)"
|
|
2360
|
+
)
|
|
2361
|
+
# Keep identical to PY_NON_CORE_TOPS in skf-extract-public-api.py.
|
|
2362
|
+
PY_NON_CORE_TOPS = frozenset({"docs", "examples", "benchmarks"})
|
|
2363
|
+
# The walk's own additions: pip's and setuptools' `_vendor`, test helpers
|
|
2364
|
+
# (`testing`, `fixtures`, `__mocks__`), installed packages (`site-packages`,
|
|
2365
|
+
# a virtualenv named `env`, `bower_components`) and the ESM, CommonJS and
|
|
2366
|
+
# UMD builds a JS package commits.
|
|
2367
|
+
LOOKUP_EXTRA_FOLDERS = frozenset({
|
|
2368
|
+
"_vendor", "testing", "fixtures", "__mocks__", "site-packages", "env",
|
|
2369
|
+
"bower_components", "esm", "cjs", "umd",
|
|
2370
|
+
})
|
|
2371
|
+
LOOKUP_SKIPPED_FOLDERS = frozenset(
|
|
2372
|
+
EXCLUDED_DIR_NAMES | _VENDORED_SEGMENTS | TEST_FOLDERS | PY_NON_CORE_TOPS
|
|
2373
|
+
| LOOKUP_EXTRA_FOLDERS
|
|
2374
|
+
)
|
|
2375
|
+
# The test folders the walk alone skips wherever they lie (end-to-end tests,
|
|
2376
|
+
# Jest's `__fixtures__` beside the code): a file the map cites in one is
|
|
2377
|
+
# still read.
|
|
2378
|
+
LOOKUP_WALK_FOLDERS = frozenset({"e2e", "__fixtures__"})
|
|
2379
|
+
# The folders the walk alone skips as the first folder below its walk root
|
|
2380
|
+
# or below a package root it passes through (`_package_root`'s roots):
|
|
2381
|
+
# examples and demos, samples, scripts, playgrounds, benchmarks and
|
|
2382
|
+
# migrations, which hold code that uses or changes the package rather than
|
|
2383
|
+
# its API. Deeper in a package such a folder can be live package code
|
|
2384
|
+
# (cognee's `cognee/modules/migrations/`), so it is walked there; one that
|
|
2385
|
+
# holds a file the map cites is walked too, and that file is still read.
|
|
2386
|
+
LOOKUP_TOP_FOLDERS = frozenset({
|
|
2387
|
+
"example", "demo", "demos", "sample", "samples", "scripts", "playground",
|
|
2388
|
+
"bench", "benchmark", "migrations",
|
|
2389
|
+
})
|
|
2390
|
+
# The kinds of a documented inventory (validate-inventory.py's `kind`) that a
|
|
2391
|
+
# module cannot be: such a name takes no line 1 from a module so named. A
|
|
2392
|
+
# `method` is left out: reconcile-coverage.py never lists a method as stale,
|
|
2393
|
+
# and a module may share a method's name.
|
|
2394
|
+
NO_MODULE_KINDS = frozenset({"function", "hook", "class", "interface", "enum", "type"})
|
|
2395
|
+
# A TS/JS file that is itself the export: its module so named declares the
|
|
2396
|
+
# name whatever its kind.
|
|
2397
|
+
_SELF_EXPORT_RE = re.compile(r"^(?:export\s+default\b|module\.exports\s*=(?!=))", re.M)
|
|
2398
|
+
# A minified bundle: built output, whatever folder holds it.
|
|
2399
|
+
LOOKUP_SKIPPED_SUFFIXES = (".min.js",)
|
|
2400
|
+
# The errors a lookup gives up on: every `defined_at` is then null.
|
|
2401
|
+
_LOOKUP_ERRORS = (OSError, ValueError, RuntimeError)
|
|
2402
|
+
|
|
2403
|
+
|
|
2404
|
+
def _rule_family(path: str) -> str | None:
|
|
2405
|
+
"""The definition rules a file's extension picks: `python`, `tsjs` or
|
|
2406
|
+
None."""
|
|
2407
|
+
suffix = posixpath.splitext(path)[1].lower()
|
|
2408
|
+
if suffix in PYTHON_EXTENSIONS:
|
|
2409
|
+
return "python"
|
|
2410
|
+
return "tsjs" if suffix in TSJS_EXTENSIONS else None
|
|
2411
|
+
|
|
2412
|
+
|
|
2413
|
+
def _lookup_skips_folder(segment: str) -> bool:
|
|
2414
|
+
"""True when the `defined_at` walk never goes into this folder."""
|
|
2415
|
+
segment = segment.lower()
|
|
2416
|
+
return segment.startswith(".") or segment in LOOKUP_SKIPPED_FOLDERS
|
|
2417
|
+
|
|
2418
|
+
|
|
2419
|
+
def _lookup_skips_file(filename: str) -> bool:
|
|
2420
|
+
"""True when the `defined_at` lookup never reads this file: a test file,
|
|
2421
|
+
as skf-disqualify-candidates.py's `_is_test_file` judges its name (a
|
|
2422
|
+
test configuration file included), or a minified bundle."""
|
|
2423
|
+
name = filename.lower()
|
|
2424
|
+
return (
|
|
2425
|
+
name in TEST_CONFIG_NAMES
|
|
2426
|
+
or name.startswith(TEST_CONFIG_PREFIXES)
|
|
2427
|
+
or TEST_FILE_RE.search(name) is not None
|
|
2428
|
+
or name.endswith(LOOKUP_SKIPPED_SUFFIXES)
|
|
2429
|
+
)
|
|
2430
|
+
|
|
2431
|
+
|
|
2432
|
+
def _symlinked(root: Path, parts: tuple[str, ...]) -> bool:
|
|
2433
|
+
"""True when a path below `root` goes through a symlink, itself
|
|
2434
|
+
included."""
|
|
2435
|
+
return any(root.joinpath(*parts[:i]).is_symlink() for i in range(1, len(parts) + 1))
|
|
2436
|
+
|
|
2437
|
+
|
|
2438
|
+
def _readable_below(root: Path, parts: tuple[str, ...], real_root: Path | None = None) -> bool:
|
|
2439
|
+
"""True when the `defined_at` lookup may read the file at `parts` below
|
|
2440
|
+
`root`: not a file it skips (`_lookup_skips_file`), no symlink on its
|
|
2441
|
+
path, and its real path a file inside the root (`real_root`, the root
|
|
2442
|
+
resolved)."""
|
|
2443
|
+
if _lookup_skips_file(parts[-1]) or _symlinked(root, parts):
|
|
2444
|
+
return False
|
|
2445
|
+
real = root.joinpath(*parts).resolve()
|
|
2446
|
+
return real.is_relative_to(real_root or root.resolve()) and real.is_file()
|
|
2447
|
+
|
|
2448
|
+
|
|
2449
|
+
def _cited_parts(source_file: str) -> tuple[str, ...] | None:
|
|
2450
|
+
"""A recorded `source_file` as path segments below the source root, None
|
|
2451
|
+
when it is absolute or climbs out of the root."""
|
|
2452
|
+
path = normalize_path(source_file)
|
|
2453
|
+
if not path or path == "." or posixpath.isabs(path):
|
|
2454
|
+
return None
|
|
2455
|
+
parts = tuple(path.split("/"))
|
|
2456
|
+
return None if parts[0] == ".." or ":" in parts[0] else parts
|
|
2457
|
+
|
|
2458
|
+
|
|
2459
|
+
# The file whose folder is a package, by rule family.
|
|
2460
|
+
_PACKAGE_MARKERS = {"python": "__init__.py", "tsjs": "package.json"}
|
|
2461
|
+
# Type stubs: a declaration file beside the module it types.
|
|
2462
|
+
_STUB_SUFFIXES = {".pyi": "python", ".d.ts": "tsjs", ".d.mts": "tsjs", ".d.cts": "tsjs"}
|
|
2463
|
+
|
|
2464
|
+
|
|
2465
|
+
def _stub_of(path: str) -> tuple[str, str] | None:
|
|
2466
|
+
"""(the path without its suffix, family) of a type stub (`NAME.pyi`,
|
|
2467
|
+
`NAME.d.ts`), None for any other file."""
|
|
2468
|
+
lower = path.lower()
|
|
2469
|
+
for suffix, family in _STUB_SUFFIXES.items():
|
|
2470
|
+
if lower.endswith(suffix):
|
|
2471
|
+
return path[: -len(suffix)], family
|
|
2472
|
+
return None
|
|
2473
|
+
|
|
2474
|
+
|
|
2475
|
+
def _without_stubs(hits: list[str]) -> list[str]:
|
|
2476
|
+
"""`file:line` items less each stub whose implementation (`NAME.py`
|
|
2477
|
+
beside `NAME.pyi`, a TS/JS `NAME.<ext>` beside `NAME.d.ts`) is an item
|
|
2478
|
+
too: the two are one declaring file, the implementation."""
|
|
2479
|
+
files = [h.rsplit(":", 1)[0] for h in hits]
|
|
2480
|
+
implementations = {
|
|
2481
|
+
(posixpath.splitext(f)[0], _rule_family(f))
|
|
2482
|
+
for f in files
|
|
2483
|
+
if _stub_of(f) is None and _rule_family(f) is not None
|
|
2484
|
+
}
|
|
2485
|
+
return [h for h, f in zip(hits, files) if _stub_of(f) not in implementations]
|
|
2486
|
+
|
|
2487
|
+
|
|
2488
|
+
def _module_name(parts: tuple[str, ...]) -> str | None:
|
|
2489
|
+
"""The name a file is the module or package of, which it declares at
|
|
2490
|
+
its line 1: `NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
|
|
2491
|
+
`NAME/index.<ext>`. None for any other file, and for a package file
|
|
2492
|
+
with no folder above it below the source root."""
|
|
2493
|
+
stem, ext = posixpath.splitext(parts[-1])
|
|
2494
|
+
ext = ext.lower()
|
|
2495
|
+
if ext == ".py":
|
|
2496
|
+
package = "__init__"
|
|
2497
|
+
elif ext in TSJS_EXTENSIONS:
|
|
2498
|
+
package = "index"
|
|
2499
|
+
else:
|
|
2500
|
+
return None
|
|
2501
|
+
if stem != package:
|
|
2502
|
+
return stem
|
|
2503
|
+
return parts[-2] if len(parts) > 1 else None
|
|
2504
|
+
|
|
2505
|
+
|
|
2506
|
+
def _exports_itself(parts: tuple[str, ...], text: str) -> bool:
|
|
2507
|
+
"""True for a TS/JS file that is itself the export, with a column-0
|
|
2508
|
+
`export default` or `module.exports =`: its module so named declares
|
|
2509
|
+
the name whatever kind the inventory gives it."""
|
|
2510
|
+
return _rule_family(parts[-1]) == "tsjs" and _SELF_EXPORT_RE.search(text) is not None
|
|
2511
|
+
|
|
2512
|
+
|
|
2513
|
+
def declared_line(
|
|
2514
|
+
source_file: str | tuple[str, ...],
|
|
2515
|
+
name: str,
|
|
2516
|
+
source_root: Path | None = None,
|
|
2517
|
+
*,
|
|
2518
|
+
lines: list[str] | None = None,
|
|
2519
|
+
cache: _SourceCache | None = None,
|
|
2520
|
+
module_fallback: bool = True,
|
|
2521
|
+
) -> int | None:
|
|
2522
|
+
"""The line at which one file declares `name`, by classify-stale's
|
|
2523
|
+
`defined_at` rule (see the module docstring): its first line at column
|
|
2524
|
+
0 that the definition-line rules match with imports left out (a `def`,
|
|
2525
|
+
`class`, assignment, annotation, type alias, TS/JS declaration, or an
|
|
2526
|
+
alias that renames, such as `from X import Y as NAME`; a plain or star
|
|
2527
|
+
import, an `as` that keeps the imported name, a re-export and an
|
|
2528
|
+
import-valued binding never count), else, with `module_fallback`, its
|
|
2529
|
+
line 1 when it is the module or package so named (`_module_name`).
|
|
2530
|
+
|
|
2531
|
+
`source_file` is the file's path below the source root, as recorded
|
|
2532
|
+
(one that is absolute or climbs out of the root gives None) or as its
|
|
2533
|
+
segments. `lines` are its lines when the caller read them already;
|
|
2534
|
+
otherwise the file is read below `source_root` under the lookup's
|
|
2535
|
+
guards (`_readable_below`: no symlink on its path, its real path
|
|
2536
|
+
inside the root, not a file the lookup skips). Pass one `cache` for
|
|
2537
|
+
the names of one source root, and each file is read and parsed once.
|
|
2538
|
+
None when no line declares the name, the name is empty or dotted, the
|
|
2539
|
+
file is not a Python or TS/JS one, or it cannot be read.
|
|
2540
|
+
"""
|
|
2541
|
+
parts = _cited_parts(source_file) if isinstance(source_file, str) else tuple(source_file)
|
|
2542
|
+
if not name or "." in name or not parts or _rule_family(parts[-1]) is None:
|
|
2543
|
+
return None
|
|
2544
|
+
cache = cache if cache is not None else _SourceCache()
|
|
2545
|
+
if lines is None:
|
|
2546
|
+
if source_root is None:
|
|
2547
|
+
raise ValueError("declared_line reads the file below source_root when no lines are given")
|
|
2548
|
+
if not _readable_below(source_root, parts):
|
|
2549
|
+
return None
|
|
2550
|
+
lines = cache.lines(source_root.joinpath(*parts))
|
|
2551
|
+
if lines is None:
|
|
2552
|
+
return None
|
|
2553
|
+
path = Path(*parts)
|
|
2554
|
+
top, _ = _line_matches(path.suffix.lower(), path, lines, name, cache, True)
|
|
2555
|
+
if top:
|
|
2556
|
+
return top[0]
|
|
2557
|
+
return 1 if module_fallback and _module_name(parts) == name else None
|
|
2558
|
+
|
|
2559
|
+
|
|
2560
|
+
# declared_line's `cache`, public for a script that loads this one.
|
|
2561
|
+
SourceCache = _SourceCache
|
|
2562
|
+
|
|
2563
|
+
|
|
2564
|
+
def _baseline_path(path: str) -> str:
|
|
2565
|
+
"""A cited path as an extraction writes it: forward slashes, no
|
|
2566
|
+
leading `./`."""
|
|
2567
|
+
rel = path.strip().replace("\\", "/")
|
|
2568
|
+
while rel.startswith("./"):
|
|
2569
|
+
rel = rel[2:]
|
|
2570
|
+
return rel
|
|
2571
|
+
|
|
2572
|
+
|
|
2573
|
+
def _declaration_refused(source_root: Path, file: str) -> bool:
|
|
2574
|
+
"""True when `declared_line` would not read a file that is there: a
|
|
2575
|
+
path outside the source root, a file the lookup skips (a test file, a
|
|
2576
|
+
minified bundle) or one with a symlink on its path. A file that is gone
|
|
2577
|
+
is not refused: its entries are deleted."""
|
|
2578
|
+
parts = _cited_parts(file)
|
|
2579
|
+
if parts is None:
|
|
2580
|
+
return True
|
|
2581
|
+
path = source_root.joinpath(*parts)
|
|
2582
|
+
return os.path.lexists(path) and not _readable_below(source_root, parts)
|
|
2583
|
+
|
|
2584
|
+
|
|
2585
|
+
def baseline_gaps(
|
|
2586
|
+
provenance: dict,
|
|
2587
|
+
source_root: Path,
|
|
2588
|
+
held: set[tuple[str, str]],
|
|
2589
|
+
files: set[str],
|
|
2590
|
+
listed: set[tuple] | frozenset = frozenset(),
|
|
2591
|
+
*,
|
|
2592
|
+
cache: _SourceCache | None = None,
|
|
2593
|
+
) -> tuple[list[dict], list[dict]]:
|
|
2594
|
+
"""The provenance entries an extraction left out whose cited file still
|
|
2595
|
+
declares them: the recipe runner never reports a module, an alias that
|
|
2596
|
+
renames, a dunder such as `__version__` or another Python or Rust
|
|
2597
|
+
underscore name, so a diff would read each as removed.
|
|
2598
|
+
|
|
2599
|
+
`held` holds the (name, file) of every export the extraction found,
|
|
2600
|
+
`files` the files whose entries are checked (the ones read), and
|
|
2601
|
+
`listed` the (name, file) of the gaps already listed elsewhere (the
|
|
2602
|
+
runner's own extraction gaps), each path with forward slashes. An
|
|
2603
|
+
entry is left out when its name is dotted (a member) and it is no
|
|
2604
|
+
`module` or `package` entry, when its file is not in `files`, when the
|
|
2605
|
+
extraction holds the name or its re-export target (`reexport_map`,
|
|
2606
|
+
else the entry's `reexported_as`) at that file, or when it is listed
|
|
2607
|
+
already. Each name and file is taken once.
|
|
2608
|
+
|
|
2609
|
+
Returns (gaps, unchecked). A gap is {"name", "file", "line", "entry":
|
|
2610
|
+
null}, plus the entry's `export_type` when it has one, at the line
|
|
2611
|
+
`declared_line` gives (only a `module` or `package` entry takes the
|
|
2612
|
+
module rule); an entry whose file declares it no more is no gap.
|
|
2613
|
+
`unchecked` holds {"name", "file", "export_type"} for each entry no
|
|
2614
|
+
rule can check, to read by eye: one whose file is neither Python nor
|
|
2615
|
+
TS/JS, one whose file the lookup will not read (outside the source
|
|
2616
|
+
root, a test file or a minified bundle, a symlink on its path), and a
|
|
2617
|
+
dotted `module` or `package` entry (`api.v1`). The entries are read
|
|
2618
|
+
file by file over one cache, so each file is read and parsed once."""
|
|
2619
|
+
checkable = PYTHON_EXTENSIONS | TSJS_EXTENSIONS
|
|
2620
|
+
renamed = extract_reexport_map(provenance)
|
|
2621
|
+
seen = set(listed)
|
|
2622
|
+
by_file: dict[str, list[tuple[str, object]]] = {}
|
|
2623
|
+
unchecked: list[dict] = []
|
|
2624
|
+
entries = provenance.get("entries")
|
|
2625
|
+
for entry in entries if isinstance(entries, list) else []:
|
|
2626
|
+
if not isinstance(entry, dict):
|
|
2627
|
+
continue
|
|
2628
|
+
name, cited = entry.get("export_name"), entry.get("source_file")
|
|
2629
|
+
if not isinstance(name, str) or not isinstance(cited, str):
|
|
2630
|
+
continue
|
|
2631
|
+
name, file, kind = name.strip(), _baseline_path(cited), entry.get("export_type")
|
|
2632
|
+
module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
|
|
2633
|
+
if not name or ("." in name and not module) or (name, file) in seen:
|
|
2634
|
+
continue
|
|
2635
|
+
if cited.replace("\\", "/") not in files and file not in files:
|
|
2636
|
+
continue
|
|
2637
|
+
if (name, file) in held or (renamed.get(name), file) in held:
|
|
2638
|
+
continue
|
|
2639
|
+
seen.add((name, file))
|
|
2640
|
+
if "." in name or Path(file).suffix.lower() not in checkable or _declaration_refused(source_root, file):
|
|
2641
|
+
unchecked.append({"name": name, "file": file, "export_type": kind})
|
|
2642
|
+
continue
|
|
2643
|
+
by_file.setdefault(file, []).append((name, kind))
|
|
2644
|
+
cache = cache if cache is not None else _SourceCache()
|
|
2645
|
+
gaps = []
|
|
2646
|
+
for file, named in by_file.items():
|
|
2647
|
+
for name, kind in named:
|
|
2648
|
+
module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
|
|
2649
|
+
line = declared_line(file, name, source_root, cache=cache, module_fallback=module)
|
|
2650
|
+
if line is None:
|
|
2651
|
+
continue
|
|
2652
|
+
gap = {"name": name, "file": file, "line": line, "entry": None}
|
|
2653
|
+
if kind is not None:
|
|
2654
|
+
gap["export_type"] = kind
|
|
2655
|
+
gaps.append(gap)
|
|
2656
|
+
return gaps, unchecked
|
|
2657
|
+
|
|
2658
|
+
|
|
2659
|
+
class _DeclarationLookup:
|
|
2660
|
+
"""Where the source declares the stale names (classify-stale's
|
|
2661
|
+
`defined_at`): a column-0 declaration, an alias that renames included, a
|
|
2662
|
+
plain or star import and a re-export never counting.
|
|
2663
|
+
|
|
2664
|
+
A module or package named after the name (`_module_name`) declares it,
|
|
2665
|
+
at its line 1 when no line of it does, unless `find` takes the name as
|
|
2666
|
+
`strict` and the file is not a TS/JS one that is itself the export
|
|
2667
|
+
(`_exports_itself`). A file the name's entries cite that declares it
|
|
2668
|
+
gives its first such line. Otherwise the walk lists every file that
|
|
2669
|
+
declares it (`declared_in`), and exactly one must: the walk reads,
|
|
2670
|
+
sorted, the files below the walk root (`_walk_root`), leaving out the
|
|
2671
|
+
folders `_lookup_skips_folder` names, the `LOOKUP_WALK_FOLDERS`, the
|
|
2672
|
+
`LOOKUP_TOP_FOLDERS` right below the walk root and a package root
|
|
2673
|
+
(`_is_package_root`) that hold no cited file, the files
|
|
2674
|
+
`_lookup_skips_file` names and every symlink. Only the rule families
|
|
2675
|
+
(Python, TS/JS) the entries cite are read (both when no entry cites a
|
|
2676
|
+
file). A cited file is read inside the source root only, with no symlink
|
|
2677
|
+
on its path and the same skips but the walk-only ones, judged below the
|
|
2678
|
+
walk root when it lies there. The walk runs once for all the names no
|
|
2679
|
+
cited file settles: each file is read once, and only a file whose text
|
|
2680
|
+
holds a name goes through the rules. Any error of the filesystem raises
|
|
2681
|
+
(`_LOOKUP_ERRORS`)."""
|
|
2682
|
+
|
|
2683
|
+
def __init__(self, entries: list[dict], source_root: Path) -> None:
|
|
2684
|
+
self.root = source_root
|
|
2685
|
+
self.real_root = source_root.resolve()
|
|
2686
|
+
recorded = [f for f in map(_entry_file, entries) if f is not None]
|
|
2687
|
+
# a map that cites no file names no language: both families
|
|
2688
|
+
families = {_rule_family(f) for f in recorded}
|
|
2689
|
+
self.families = families - {None} if recorded else {"python", "tsjs"}
|
|
2690
|
+
self.folder = self._walk_root(recorded)
|
|
2691
|
+
# the folders that hold a cited file: a LOOKUP_TOP_FOLDERS one is walked
|
|
2692
|
+
self._cited_folders = {
|
|
2693
|
+
parts[:i]
|
|
2694
|
+
for parts in filter(None, map(_cited_parts, recorded))
|
|
2695
|
+
for i in range(1, len(parts))
|
|
2696
|
+
}
|
|
2697
|
+
# the cited files read, kept for the walk: a few files at most
|
|
2698
|
+
self._cited: dict[tuple[str, ...], list[str] | None] = {}
|
|
2699
|
+
self._cited_cache = _SourceCache()
|
|
2700
|
+
|
|
2701
|
+
# -- the walk root ------------------------------------------------------
|
|
2702
|
+
|
|
2703
|
+
def _symlinked(self, parts: tuple[str, ...]) -> bool:
|
|
2704
|
+
"""True when a path below the source root goes through a symlink,
|
|
2705
|
+
itself included."""
|
|
2706
|
+
return _symlinked(self.root, parts)
|
|
2707
|
+
|
|
2708
|
+
def _real_folder(self, parts: tuple[str, ...]) -> tuple[str, ...]:
|
|
2709
|
+
"""The deepest folder of a path below the source root that is one,
|
|
2710
|
+
with no symlink on its path."""
|
|
2711
|
+
while parts and (
|
|
2712
|
+
self._symlinked(parts) or not self.root.joinpath(*parts).is_dir()
|
|
2713
|
+
):
|
|
2714
|
+
parts = parts[:-1]
|
|
2715
|
+
return parts
|
|
2716
|
+
|
|
2717
|
+
def _holds(self, folder: tuple[str, ...], filename: str) -> bool:
|
|
2718
|
+
return (self.root.joinpath(*folder) / filename).is_file()
|
|
2719
|
+
|
|
2720
|
+
def _package_root(
|
|
2721
|
+
self, folder: tuple[str, ...], family: str
|
|
2722
|
+
) -> tuple[str, ...] | None:
|
|
2723
|
+
"""The root of the package a folder below the source root lies in,
|
|
2724
|
+
None when it lies in none. Python: the nearest folder at or above it
|
|
2725
|
+
that holds `__init__.py`, then up while the parent holds one, never
|
|
2726
|
+
the source root itself (a stray `__init__.py` there would make the
|
|
2727
|
+
whole tree one package). TS/JS: the nearest folder at or above it
|
|
2728
|
+
that holds `package.json`."""
|
|
2729
|
+
marker = _PACKAGE_MARKERS[family]
|
|
2730
|
+
python = family == "python"
|
|
2731
|
+
here = folder
|
|
2732
|
+
while not (self._holds(here, marker) and (here or not python)):
|
|
2733
|
+
if not here:
|
|
2734
|
+
return None
|
|
2735
|
+
here = here[:-1]
|
|
2736
|
+
if python:
|
|
2737
|
+
while len(here) > 1 and self._holds(here[:-1], marker):
|
|
2738
|
+
here = here[:-1]
|
|
2739
|
+
return here
|
|
2740
|
+
|
|
2741
|
+
def _walk_root(self, recorded: list[str]) -> tuple[str, ...]:
|
|
2742
|
+
"""The folder the walk reads: the deepest folder the entries' files
|
|
2743
|
+
share, widened to its package root, leaving out an entry outside
|
|
2744
|
+
every package (the source root when no entry lies in one)."""
|
|
2745
|
+
folders: list[tuple[str, ...]] = []
|
|
2746
|
+
widen: set[str] = set()
|
|
2747
|
+
for source_file in recorded:
|
|
2748
|
+
parts = _cited_parts(source_file)
|
|
2749
|
+
family = _rule_family(source_file) if parts else None
|
|
2750
|
+
if family is None:
|
|
2751
|
+
continue
|
|
2752
|
+
if self._package_root(self._real_folder(parts[:-1]), family) is None:
|
|
2753
|
+
continue # outside every package: a script, a docs page
|
|
2754
|
+
folders.append(parts[:-1])
|
|
2755
|
+
widen.add(family)
|
|
2756
|
+
if not folders:
|
|
2757
|
+
return ()
|
|
2758
|
+
shared = self._real_folder(tuple(os.path.commonprefix(folders)))
|
|
2759
|
+
roots = [self._package_root(shared, family) for family in sorted(widen)]
|
|
2760
|
+
return min((r for r in roots if r is not None), key=len, default=shared)
|
|
2761
|
+
|
|
2762
|
+
# -- reading ------------------------------------------------------------
|
|
2763
|
+
|
|
2764
|
+
def _below_walk_root(self, parts: tuple[str, ...]) -> tuple[str, ...]:
|
|
2765
|
+
depth = len(self.folder)
|
|
2766
|
+
return parts[depth:] if parts[:depth] == self.folder else parts
|
|
2767
|
+
|
|
2768
|
+
def _reads_cited(self, parts: tuple[str, ...]) -> bool:
|
|
2769
|
+
"""True when the lookup reads a cited file: one of a family the
|
|
2770
|
+
entries cite, in no skipped place, with no symlink on its path and
|
|
2771
|
+
inside the source root."""
|
|
2772
|
+
if _rule_family(parts[-1]) not in self.families:
|
|
2773
|
+
return False
|
|
2774
|
+
below = self._below_walk_root(parts)
|
|
2775
|
+
if any(_lookup_skips_folder(s) for s in below[:-1]):
|
|
2776
|
+
return False
|
|
2777
|
+
return _readable_below(self.root, parts, self.real_root)
|
|
2778
|
+
|
|
2779
|
+
def _cited_lines(self, parts: tuple[str, ...]) -> list[str] | None:
|
|
2780
|
+
if parts not in self._cited:
|
|
2781
|
+
readable = self._reads_cited(parts)
|
|
2782
|
+
path = self.root.joinpath(*parts)
|
|
2783
|
+
self._cited[parts] = _text_lines(_read_source(path)) if readable else None
|
|
2784
|
+
return self._cited[parts]
|
|
2785
|
+
|
|
2786
|
+
def _is_package_root(self, folder: tuple[str, ...], filenames: list[str]) -> bool:
|
|
2787
|
+
"""True when a walked folder (its `filenames`) is a package root of a
|
|
2788
|
+
family the walk reads, as `_package_root` finds one: TS/JS, a folder
|
|
2789
|
+
that holds `package.json`; Python, the top of an `__init__.py`
|
|
2790
|
+
chain, never the source root."""
|
|
2791
|
+
if "tsjs" in self.families and "package.json" in filenames:
|
|
2792
|
+
return True
|
|
2793
|
+
return (
|
|
2794
|
+
"python" in self.families and bool(folder) and "__init__.py" in filenames
|
|
2795
|
+
and (len(folder) == 1 or not self._holds(folder[:-1], "__init__.py"))
|
|
2796
|
+
)
|
|
2797
|
+
|
|
2798
|
+
def _tree(self) -> list[tuple[str, ...]]:
|
|
2799
|
+
"""The files the walk reads, as segments below the source root,
|
|
2800
|
+
sorted."""
|
|
2801
|
+
top = self.root.joinpath(*self.folder)
|
|
2802
|
+
found: list[tuple[str, ...]] = []
|
|
2803
|
+
|
|
2804
|
+
def fail(exc: OSError) -> None:
|
|
2805
|
+
raise exc
|
|
2806
|
+
|
|
2807
|
+
for dirpath, dirnames, filenames in os.walk(top, onerror=fail):
|
|
2808
|
+
here = Path(dirpath)
|
|
2809
|
+
inner = here.relative_to(top).parts
|
|
2810
|
+
below = self.folder + inner
|
|
2811
|
+
# a LOOKUP_TOP_FOLDERS name is skipped right below the walk root
|
|
2812
|
+
# and a package root, unless it holds a cited file
|
|
2813
|
+
root = not inner or self._is_package_root(below, filenames)
|
|
2814
|
+
dirnames[:] = [
|
|
2815
|
+
d for d in dirnames
|
|
2816
|
+
if not _lookup_skips_folder(d)
|
|
2817
|
+
and d.lower() not in LOOKUP_WALK_FOLDERS
|
|
2818
|
+
and not (
|
|
2819
|
+
root and d.lower() in LOOKUP_TOP_FOLDERS
|
|
2820
|
+
and (*below, d) not in self._cited_folders
|
|
2821
|
+
)
|
|
2822
|
+
and not (here / d).is_symlink()
|
|
2823
|
+
]
|
|
2824
|
+
for filename in filenames:
|
|
2825
|
+
path = here / filename
|
|
2826
|
+
if (
|
|
2827
|
+
_rule_family(filename) in self.families
|
|
2828
|
+
and not _lookup_skips_file(filename)
|
|
2829
|
+
and not path.is_symlink()
|
|
2830
|
+
and path.is_file()
|
|
2831
|
+
):
|
|
2832
|
+
found.append((*below, filename))
|
|
2833
|
+
return sorted(found)
|
|
2834
|
+
|
|
2835
|
+
@staticmethod
|
|
2836
|
+
def _first(
|
|
2837
|
+
parts: tuple[str, ...],
|
|
2838
|
+
lines: list[str],
|
|
2839
|
+
name: str,
|
|
2840
|
+
cache: _SourceCache,
|
|
2841
|
+
module_fallback: bool = True,
|
|
2842
|
+
) -> str | None:
|
|
2843
|
+
"""`file:line` of the first line of a file that declares `name`, else,
|
|
2844
|
+
with `module_fallback`, its line 1 when it is the module or package
|
|
2845
|
+
so named (`declared_line`)."""
|
|
2846
|
+
line = declared_line(
|
|
2847
|
+
parts, name, lines=lines, cache=cache, module_fallback=module_fallback
|
|
2848
|
+
)
|
|
2849
|
+
return f"{'/'.join(parts)}:{line}" if line is not None else None
|
|
2850
|
+
|
|
2851
|
+
# -- the answer ---------------------------------------------------------
|
|
2852
|
+
|
|
2853
|
+
def find(
|
|
2854
|
+
self, cited: dict[str, list[str]], strict: frozenset[str] = frozenset()
|
|
2855
|
+
) -> tuple[dict[str, str], dict[str, list[str]]]:
|
|
2856
|
+
"""For the names of `cited` (name -> the source_file values its
|
|
2857
|
+
entries record): name -> `file:line` for each one the source
|
|
2858
|
+
declares, and name -> the sorted `file:line` of every walked file
|
|
2859
|
+
that declares it, for each name the walk looked for (the names no
|
|
2860
|
+
cited file settles). A dotted name is never looked up, and a name of
|
|
2861
|
+
`strict` takes no line 1 from a module or package so named that is
|
|
2862
|
+
not itself the export (`_exports_itself`), in a cited file or in
|
|
2863
|
+
the walk."""
|
|
2864
|
+
found: dict[str, str] = {}
|
|
2865
|
+
if not self.families:
|
|
2866
|
+
return found, {}
|
|
2867
|
+
pending: list[str] = []
|
|
2868
|
+
for name, files in cited.items():
|
|
2869
|
+
if not name or "." in name:
|
|
2870
|
+
continue
|
|
2871
|
+
for parts in dict.fromkeys(filter(None, map(_cited_parts, files))):
|
|
2872
|
+
lines = self._cited_lines(parts)
|
|
2873
|
+
hit = None if lines is None else self._first(
|
|
2874
|
+
parts, lines, name, self._cited_cache,
|
|
2875
|
+
name not in strict or _exports_itself(parts, "\n".join(lines)),
|
|
2876
|
+
)
|
|
2877
|
+
if hit is not None:
|
|
2878
|
+
found[name] = hit
|
|
2879
|
+
break
|
|
2880
|
+
else:
|
|
2881
|
+
pending.append(name)
|
|
2882
|
+
declared, unsure = self._walk(pending, strict) if pending else ({}, set())
|
|
2883
|
+
# exactly one walked file must declare a name, and none it holds
|
|
2884
|
+
# in an untokenizable Python file may
|
|
2885
|
+
found.update({
|
|
2886
|
+
name: h[0] for name, h in declared.items()
|
|
2887
|
+
if len(h) == 1 and name not in unsure
|
|
2888
|
+
})
|
|
2889
|
+
return found, declared
|
|
2890
|
+
|
|
2891
|
+
def _walk(
|
|
2892
|
+
self, names: list[str], strict: frozenset[str] = frozenset()
|
|
2893
|
+
) -> tuple[dict[str, list[str]], set[str]]:
|
|
2894
|
+
"""name -> the sorted `file:line` of each walked file that declares
|
|
2895
|
+
it, at the file's first declaration (a stub beside its
|
|
2896
|
+
implementation left out: `_without_stubs`; a module so named
|
|
2897
|
+
declaring no name of `strict` unless it is the export), and the
|
|
2898
|
+
names a Python file that
|
|
2899
|
+
does not tokenize holds, which it may declare."""
|
|
2900
|
+
hits: dict[str, list[str]] = {name: [] for name in names}
|
|
2901
|
+
unsure: set[str] = set()
|
|
2902
|
+
for parts in self._tree():
|
|
2903
|
+
module = _module_name(parts)
|
|
2904
|
+
lines = self._cited.get(parts)
|
|
2905
|
+
text = (
|
|
2906
|
+
_read_source(self.root.joinpath(*parts))
|
|
2907
|
+
if lines is None
|
|
2908
|
+
else "\n".join(lines) # a cited file, read already
|
|
2909
|
+
)
|
|
2910
|
+
# a strict name's module so named counts when it is the export
|
|
2911
|
+
fallback = module not in strict or _exports_itself(parts, text)
|
|
2912
|
+
# the substring test spares the rules most files and names
|
|
2913
|
+
here = sorted(
|
|
2914
|
+
n for n in names if n in text or (n == module and fallback)
|
|
2915
|
+
)
|
|
2916
|
+
if not here:
|
|
2917
|
+
continue
|
|
2918
|
+
if lines is None:
|
|
2919
|
+
lines = _text_lines(text)
|
|
2920
|
+
cache = _SourceCache() # this file's rules only: nothing kept
|
|
2921
|
+
path = Path(*parts)
|
|
2922
|
+
if path.suffix.lower() in PYTHON_EXTENSIONS and (
|
|
2923
|
+
cache.py_statement_starts(path, lines) is None
|
|
2924
|
+
):
|
|
2925
|
+
unsure.update(n for n in here if n in text)
|
|
2926
|
+
for name in here:
|
|
2927
|
+
hit = self._first(parts, lines, name, cache, fallback)
|
|
2928
|
+
if hit is not None:
|
|
2929
|
+
hits[name].append(hit)
|
|
2930
|
+
return {name: sorted(_without_stubs(h)) for name, h in hits.items()}, unsure
|
|
2931
|
+
|
|
2932
|
+
|
|
2933
|
+
def inventory_kinds(data: object, path: str) -> dict[str, frozenset[str]]:
|
|
2934
|
+
"""name -> the kinds a documented inventory gives it (`--inventory`):
|
|
2935
|
+
validate-inventory.py's `--output` file, `{"exports": [...]}`, or its
|
|
2936
|
+
whole result, the inventory under `inventory`. An item without a string
|
|
2937
|
+
`name` and `kind` is left out, and a kind is kept in lower case. Raises
|
|
2938
|
+
ValueError when no `exports` list is there."""
|
|
2939
|
+
if isinstance(data, dict) and isinstance(data.get("inventory"), dict):
|
|
2940
|
+
data = data["inventory"] # validate-inventory.py's whole result
|
|
2941
|
+
if not isinstance(data, dict) or not isinstance(data.get("exports"), list):
|
|
2942
|
+
raise ValueError(f"--inventory {path} holds no `exports` list")
|
|
2943
|
+
kinds: dict[str, set[str]] = {}
|
|
2944
|
+
for item in data["exports"]:
|
|
2945
|
+
if not isinstance(item, dict):
|
|
2946
|
+
continue
|
|
2947
|
+
name, kind = item.get("name"), item.get("kind")
|
|
2948
|
+
if isinstance(name, str) and isinstance(kind, str):
|
|
2949
|
+
kinds.setdefault(name, set()).add(kind.strip().lower())
|
|
2950
|
+
return {name: frozenset(k) for name, k in kinds.items()}
|
|
2951
|
+
|
|
2952
|
+
|
|
2953
|
+
def _module_entry(entry: dict) -> bool:
|
|
2954
|
+
"""True for a `module` or `package` entry."""
|
|
2955
|
+
kind = entry.get("export_type")
|
|
2956
|
+
return isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
|
|
2957
|
+
|
|
2958
|
+
|
|
2959
|
+
def classify_stale(
|
|
2960
|
+
names: list[str],
|
|
2961
|
+
prov: dict,
|
|
2962
|
+
source_root: Path,
|
|
2963
|
+
kinds: dict[str, frozenset[str]] | None = None,
|
|
2964
|
+
) -> list[dict]:
|
|
1651
2965
|
"""The fabricated-signature test for each stale name: the definition-lines
|
|
1652
|
-
rules at every provenance entry whose raw `export_name` is the name
|
|
1653
|
-
|
|
1654
|
-
`
|
|
2966
|
+
rules at every provenance entry whose raw `export_name` is the name, and
|
|
2967
|
+
for a name that is not fabricated, where the source declares it
|
|
2968
|
+
(`defined_at`, and `declared_in` when the walk looked for it). The
|
|
2969
|
+
classify-stale section of the module docstring gives each reason, what
|
|
2970
|
+
a `no-file` entry means, which entry gives `source` and how
|
|
2971
|
+
`defined_at` and `declared_in` are found. `kinds` (`inventory_kinds`,
|
|
2972
|
+
None without `--inventory`) keeps a name of a `NO_MODULE_KINDS` kind
|
|
2973
|
+
from a module or package so named, unless an entry of the name is a
|
|
2974
|
+
`module` or `package` one.
|
|
1655
2975
|
"""
|
|
1656
2976
|
entries = [e for e in prov.get("entries") or [] if isinstance(e, dict)]
|
|
1657
2977
|
results: list[dict] = []
|
|
2978
|
+
# the names to look up (never a fabricated one): name -> its cited files
|
|
2979
|
+
cited: dict[str, list[str]] = {}
|
|
2980
|
+
# the names a module so named does not declare
|
|
2981
|
+
strict: set[str] = set()
|
|
1658
2982
|
for name in names:
|
|
1659
2983
|
mine = [e for e in entries if e.get("export_name") == name]
|
|
1660
2984
|
checks: list[dict] = []
|
|
@@ -1683,15 +3007,33 @@ def classify_stale(names: list[str], prov: dict, source_root: Path) -> list[dict
|
|
|
1683
3007
|
else:
|
|
1684
3008
|
reason = STALE_UNCHECKED
|
|
1685
3009
|
source = next(filter(None, map(_entry_citation, mine)), None)
|
|
3010
|
+
fabricated = reason == STALE_NOT_DEFINED
|
|
3011
|
+
if not fabricated:
|
|
3012
|
+
cited[name] = [f for f in map(_entry_file, mine) if f is not None]
|
|
3013
|
+
documented = (kinds or {}).get(name, frozenset())
|
|
3014
|
+
if documented & NO_MODULE_KINDS and not any(map(_module_entry, mine)):
|
|
3015
|
+
strict.add(name)
|
|
1686
3016
|
results.append(
|
|
1687
3017
|
{
|
|
1688
3018
|
"name": name,
|
|
1689
|
-
"fabricated":
|
|
3019
|
+
"fabricated": fabricated,
|
|
1690
3020
|
"source": source,
|
|
3021
|
+
"defined_at": None,
|
|
3022
|
+
"declared_in": None,
|
|
1691
3023
|
"reason": reason,
|
|
1692
3024
|
"entries": checks,
|
|
1693
3025
|
}
|
|
1694
3026
|
)
|
|
3027
|
+
try:
|
|
3028
|
+
found, declared = _DeclarationLookup(entries, source_root).find(
|
|
3029
|
+
cited, frozenset(strict)
|
|
3030
|
+
)
|
|
3031
|
+
except _LOOKUP_ERRORS:
|
|
3032
|
+
# a lookup that fails proves nothing: every name stays null
|
|
3033
|
+
found, declared = {}, {}
|
|
3034
|
+
for item in results:
|
|
3035
|
+
item["defined_at"] = found.get(item["name"])
|
|
3036
|
+
item["declared_in"] = declared.get(item["name"])
|
|
1695
3037
|
return results
|
|
1696
3038
|
|
|
1697
3039
|
|
|
@@ -2381,13 +3723,13 @@ def verify(
|
|
|
2381
3723
|
continue
|
|
2382
3724
|
defs, every = matches
|
|
2383
3725
|
if line not in defs:
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
# lists several lines and nobody auto-moves it.
|
|
2388
|
-
defs = sorted(set(defs) | {line})
|
|
3726
|
+
# An indented match the column-0 rule dropped joins
|
|
3727
|
+
# the lines, so nobody auto-moves it; an @overload
|
|
3728
|
+
# group gives its first line alone, the move target.
|
|
2389
3729
|
item["reason"] = STALE_LINE_NOT_DEFINITION
|
|
2390
|
-
item["definition_lines"] =
|
|
3730
|
+
item["definition_lines"] = _missed_definition_lines(
|
|
3731
|
+
sf, name, source_root, cache, line, defs, every
|
|
3732
|
+
)
|
|
2391
3733
|
stale.append(item)
|
|
2392
3734
|
stale.sort(key=lambda s: (s["export_name"], s.get("source_file") or ""))
|
|
2393
3735
|
|
|
@@ -2898,8 +4240,12 @@ def _cmd_definition_lines(args: argparse.Namespace) -> int:
|
|
|
2898
4240
|
|
|
2899
4241
|
def _cmd_classify_stale(args: argparse.Namespace) -> int:
|
|
2900
4242
|
names_path, prov_path = Path(args.names), Path(args.provenance)
|
|
4243
|
+
inv_path = Path(args.inventory) if args.inventory is not None else None
|
|
2901
4244
|
root = Path(args.source_root)
|
|
2902
|
-
|
|
4245
|
+
inputs = [("names", names_path), ("provenance map", prov_path)]
|
|
4246
|
+
if inv_path is not None:
|
|
4247
|
+
inputs.append(("inventory", inv_path))
|
|
4248
|
+
for label, path in inputs:
|
|
2903
4249
|
if not path.is_file():
|
|
2904
4250
|
print(f"error: {label} not found: {path}", file=sys.stderr)
|
|
2905
4251
|
return 2
|
|
@@ -2919,11 +4265,14 @@ def _cmd_classify_stale(args: argparse.Namespace) -> int:
|
|
|
2919
4265
|
try:
|
|
2920
4266
|
names = stale_names(data)
|
|
2921
4267
|
prov = load_json_object(prov_path, "provenance map")
|
|
4268
|
+
kinds = None if inv_path is None else inventory_kinds(
|
|
4269
|
+
load_json_object(inv_path, "inventory"), args.inventory
|
|
4270
|
+
)
|
|
2922
4271
|
except ValueError as exc:
|
|
2923
4272
|
print(f"error: {exc}", file=sys.stderr)
|
|
2924
4273
|
return 2
|
|
2925
4274
|
try:
|
|
2926
|
-
result = classify_stale(names, prov, root)
|
|
4275
|
+
result = classify_stale(names, prov, root, kinds)
|
|
2927
4276
|
except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
|
|
2928
4277
|
return _unexpected("stale-name classification", exc)
|
|
2929
4278
|
return 0 if _emit(result, args.output) else 2
|
|
@@ -3225,8 +4574,8 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
3225
4574
|
p = sub.add_parser(
|
|
3226
4575
|
"classify-stale",
|
|
3227
4576
|
help=(
|
|
3228
|
-
"decide which stale names are fabricated signatures
|
|
3229
|
-
"file defines them"
|
|
4577
|
+
"decide which stale names are fabricated signatures (no cited "
|
|
4578
|
+
"file defines them) and where the source declares the others"
|
|
3230
4579
|
),
|
|
3231
4580
|
)
|
|
3232
4581
|
p.add_argument(
|
|
@@ -3245,6 +4594,17 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
3245
4594
|
required=True,
|
|
3246
4595
|
help="source tree root the entries' source_file paths are under",
|
|
3247
4596
|
)
|
|
4597
|
+
p.add_argument(
|
|
4598
|
+
"--inventory",
|
|
4599
|
+
default=None,
|
|
4600
|
+
help=(
|
|
4601
|
+
"validate-inventory.py's documented inventory: a name it gives a "
|
|
4602
|
+
"function, hook, class, interface, enum or type kind takes no "
|
|
4603
|
+
"line 1 from a module so named, unless a provenance map entry of "
|
|
4604
|
+
"the name is a module or package one or the module is a TS/JS "
|
|
4605
|
+
"file that is itself the export"
|
|
4606
|
+
),
|
|
4607
|
+
)
|
|
3248
4608
|
p.add_argument(
|
|
3249
4609
|
"-o",
|
|
3250
4610
|
"--output",
|