bmad-module-skill-forge 3.0.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +3 -3
  3. package/docs/_data/pinned.yaml +19 -13
  4. package/docs/architecture.md +2 -2
  5. package/docs/concepts.md +4 -4
  6. package/docs/examples.md +33 -26
  7. package/docs/getting-started.md +3 -3
  8. package/docs/how-it-works.md +2 -2
  9. package/docs/index.md +1 -1
  10. package/docs/skill-model.md +41 -39
  11. package/docs/troubleshooting.md +8 -0
  12. package/docs/verifying-a-skill.md +46 -21
  13. package/docs/why-skf.md +1 -1
  14. package/docs/workflows.md +5 -5
  15. package/package.json +1 -1
  16. package/src/knowledge/qmd-registry.md +1 -1
  17. package/src/knowledge/version-paths.md +1 -1
  18. package/src/shared/health-check.md +2 -50
  19. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
  20. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +12 -4
  21. package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
  22. package/src/shared/scripts/skf-atomic-write.py +212 -59
  23. package/src/shared/scripts/skf-build-change-manifest.py +1437 -178
  24. package/src/shared/scripts/skf-classify-changed-files.py +78 -0
  25. package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
  26. package/src/shared/scripts/skf-count-tokens.py +54 -10
  27. package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
  28. package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
  29. package/src/shared/scripts/skf-detect-workspaces.py +67 -0
  30. package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
  31. package/src/shared/scripts/skf-extract-public-api.py +1673 -189
  32. package/src/shared/scripts/skf-extraction-inventory.py +584 -46
  33. package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
  34. package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
  35. package/src/shared/scripts/skf-github-probe.py +47 -8
  36. package/src/shared/scripts/skf-hash-content.py +14 -8
  37. package/src/shared/scripts/skf-load-provenance.py +3 -1
  38. package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
  39. package/src/shared/scripts/skf-parse-gaps.py +135 -26
  40. package/src/shared/scripts/skf-preflight.py +15 -1
  41. package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
  42. package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
  43. package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
  44. package/src/shared/scripts/skf-resolve-customization.py +197 -0
  45. package/src/shared/scripts/skf-shard-body.py +239 -41
  46. package/src/shared/scripts/skf-structural-diff.py +133 -18
  47. package/src/shared/scripts/skf-update-active-symlink.py +115 -62
  48. package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
  49. package/src/shared/scripts/skf-validate-output.py +12 -3
  50. package/src/shared/scripts/skf-verify-provenance-completeness.py +1439 -79
  51. package/src/shared/scripts/skf-write-skill-brief.py +41 -8
  52. package/src/shared/tool-requirements.yaml +8 -2
  53. package/src/skf-analyze-source/SKILL.md +2 -2
  54. package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
  55. package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
  56. package/src/skf-audit-skill/SKILL.md +2 -2
  57. package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
  58. package/src/skf-audit-skill/references/headless-contract.md +1 -1
  59. package/src/skf-audit-skill/references/health-check.md +2 -0
  60. package/src/skf-audit-skill/references/init.md +2 -1
  61. package/src/skf-audit-skill/references/re-index.md +14 -9
  62. package/src/skf-audit-skill/references/report.md +2 -2
  63. package/src/skf-audit-skill/references/semantic-diff.md +15 -15
  64. package/src/skf-audit-skill/references/structural-diff.md +10 -5
  65. package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
  66. package/src/skf-brief-skill/SKILL.md +3 -3
  67. package/src/skf-brief-skill/assets/scope-templates.md +1 -1
  68. package/src/skf-brief-skill/assets/skill-brief-schema.md +4 -11
  69. package/src/skf-brief-skill/references/analyze-target.md +6 -6
  70. package/src/skf-brief-skill/references/confirm-brief.md +1 -1
  71. package/src/skf-brief-skill/references/scope-definition.md +4 -4
  72. package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
  73. package/src/skf-brief-skill/references/version-resolution.md +5 -9
  74. package/src/skf-campaign/SKILL.md +2 -2
  75. package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
  76. package/src/skf-create-skill/SKILL.md +2 -2
  77. package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
  78. package/src/skf-create-skill/assets/skill-sections.md +7 -7
  79. package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
  80. package/src/skf-create-skill/references/compile.md +31 -60
  81. package/src/skf-create-skill/references/component-extraction.md +3 -3
  82. package/src/skf-create-skill/references/enrich.md +2 -2
  83. package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
  84. package/src/skf-create-skill/references/extract.md +23 -19
  85. package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
  86. package/src/skf-create-skill/references/extraction-patterns.md +10 -11
  87. package/src/skf-create-skill/references/generate-artifacts.md +1 -1
  88. package/src/skf-create-skill/references/report.md +21 -5
  89. package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
  90. package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
  91. package/src/skf-create-stack-skill/SKILL.md +2 -2
  92. package/src/skf-create-stack-skill/references/validate.md +1 -1
  93. package/src/skf-drop-skill/SKILL.md +2 -2
  94. package/src/skf-drop-skill/references/execute.md +5 -1
  95. package/src/skf-drop-skill/references/health-check.md +2 -0
  96. package/src/skf-drop-skill/references/invocation-contract.md +1 -1
  97. package/src/skf-drop-skill/references/report.md +2 -2
  98. package/src/skf-drop-skill/references/select.md +7 -3
  99. package/src/skf-export-skill/SKILL.md +2 -2
  100. package/src/skf-export-skill/assets/snippet-format.md +5 -5
  101. package/src/skf-export-skill/references/generate-snippet.md +22 -15
  102. package/src/skf-export-skill/references/load-skill.md +1 -1
  103. package/src/skf-export-skill/references/token-report.md +29 -9
  104. package/src/skf-forger/SKILL.md +2 -2
  105. package/src/skf-quick-skill/SKILL.md +2 -2
  106. package/src/skf-refine-architecture/SKILL.md +6 -8
  107. package/src/skf-rename-skill/SKILL.md +2 -2
  108. package/src/skf-setup/SKILL.md +2 -2
  109. package/src/skf-test-skill/SKILL.md +2 -2
  110. package/src/skf-test-skill/assets/output-section-formats.md +6 -4
  111. package/src/skf-test-skill/references/coherence-check.md +5 -4
  112. package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
  113. package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
  114. package/src/skf-test-skill/references/coverage-check.md +69 -111
  115. package/src/skf-test-skill/references/init.md +5 -6
  116. package/src/skf-test-skill/references/migration-section-rules.md +26 -14
  117. package/src/skf-test-skill/references/report.md +11 -9
  118. package/src/skf-test-skill/references/scoring-rules.md +8 -6
  119. package/src/skf-test-skill/references/source-access-protocol.md +15 -18
  120. package/src/skf-test-skill/scripts/build-result-context.py +114 -21
  121. package/src/skf-test-skill/scripts/gap-ledger.py +580 -80
  122. package/src/skf-test-skill/scripts/load-coverage-inputs.py +646 -107
  123. package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
  124. package/src/skf-test-skill/scripts/reconcile-coverage.py +450 -23
  125. package/src/skf-test-skill/scripts/score-signatures.py +356 -61
  126. package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
  127. package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
  128. package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
  129. package/src/skf-test-skill/templates/test-report-template.md +1 -0
  130. package/src/skf-update-skill/SKILL.md +2 -2
  131. package/src/skf-update-skill/references/detect-changes.md +32 -13
  132. package/src/skf-update-skill/references/gap-driven.md +37 -22
  133. package/src/skf-update-skill/references/init.md +6 -2
  134. package/src/skf-update-skill/references/invocation-contract.md +1 -1
  135. package/src/skf-update-skill/references/merge.md +17 -12
  136. package/src/skf-update-skill/references/re-extract.md +4 -2
  137. package/src/skf-update-skill/references/report.md +7 -4
  138. package/src/skf-update-skill/references/write.md +77 -36
  139. package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
  140. package/src/skf-verify-stack/SKILL.md +2 -2
  141. package/tools/cli/lib/tool-check.js +158 -2
@@ -33,8 +33,9 @@ entries are per-citation, so the two sets measure different surfaces:
33
33
  Four more subcommands act on the same rules: `fix` applies the citation
34
34
  prefix and source-line fixes a `verify` result has one answer for,
35
35
  `definition-lines` gives the lines that define one export,
36
- `classify-stale` decides which stale names are fabricated signatures, and
37
- `kind-at` gives the node kind of the recipe that matches a source line.
36
+ `classify-stale` decides which stale names are fabricated signatures and
37
+ where the source declares the others, and `kind-at` gives the node kind
38
+ of the recipe that matches a source line.
38
39
 
39
40
  An LLM set-diff can silently pass a dropped or orphaned entry, and eyeballing
40
41
  whether a cited line still exists is not something the model can do reliably.
@@ -88,7 +89,19 @@ name (`App.update`) it is the last segment. The rules follow the
88
89
  a keyword argument or a parameter never matches; a file that does not
89
90
  tokenize is matched line by line. A `from X import *` line counts only
90
91
  when no other line of the file defines NAME (a re-export the
91
- extraction could not trace is cited at its star import).
92
+ extraction could not trace is cited at its star import). An
93
+ `@overload` group, where every line found is a `def` / `async def
94
+ NAME` line at one indentation in one enclosing block (one class, not
95
+ two) and every one but at most the last has `@overload` or
96
+ `@<module>.overload` (`@typing.overload`,
97
+ `@typing_extensions.overload`, `@t.overload` after
98
+ `import typing as t`) in its contiguous decorator block, defines NAME
99
+ at each of its lines, the stubs and the implementation. A comment line
100
+ may sit in the decorator block, and so may a decorator's arguments on
101
+ lines of their own, unless the file does not tokenize. A recorded line
102
+ that misses the group gives its first line alone, which is its first
103
+ stub: the public-API runner, its merge rule and the structural diff
104
+ all record that line, so `fix` can move the entry there.
92
105
  - TS/JS (`.ts`, `.tsx`, `.mts`, `.cts`, `.js`, `.jsx`, `.mjs`, `.cjs`): a
93
106
  `function` (or `function*`), `class`, `interface`, `type`, `enum`,
94
107
  `namespace`, `const`, `let` or `var` declaration of NAME, with any of
@@ -97,7 +110,11 @@ name (`App.update`) it is the last segment. The rules follow the
97
110
  and `class`; a name inside a decorator's arguments is not declared);
98
111
  a later declarator of a `const`, `let` or `var` list
99
112
  (`export const a = 1, NAME = 2;`, or `export const a = 1,` then
100
- ` NAME = 2;` as Prettier writes it: see Declarator lists below); NAME
113
+ ` NAME = 2;` as Prettier writes it: see Declarator lists below);
114
+ NAME bound by a destructuring pattern that begins such a list
115
+ (`export const { a, b: NAME } = obj;`, `const [NAME] = list`, the
116
+ pattern on one line or several), on the line where NAME appears (see
117
+ Declarator lists below); NAME
101
118
  on the line right after a line that ends in its declaration keyword
102
119
  (`export function`, `export interface`, `export const` and so on),
103
120
  which is the line cited;
@@ -124,14 +141,23 @@ name (`App.update`) it is the last segment. The rules follow the
124
141
  Declarator lists: a `const`, `let` or `var` list is split at its
125
142
  top-level commas, outside strings, regular expression literals and
126
143
  brackets, a type-argument list included (`Map<K, V>`: a `<` right after
127
- a name), up to a top-level `;`, and a destructuring pattern declares
128
- nothing. The list goes on to the next line when a line ends inside a
129
- bracket, after a top-level comma or after an operator waiting for its
130
- operand, or when the next line starts with an operator (a member chain or
131
- a ternary broken over lines), so a declarator on a line of its own
132
- counts. A `<` where an expression begins (JSX, a type assertion, a generic
144
+ a name), up to a top-level `;`. The list goes on to the next line when a
145
+ line ends inside a bracket, after a top-level comma or after an operator
146
+ waiting for its operand, or when the next line starts with an operator (a
147
+ member chain or a ternary broken over lines), so a declarator on a line
148
+ of its own counts. A `<` where an expression begins (JSX, a type assertion, a generic
133
149
  arrow function's type parameters) is not followed: no later declarator of
134
- the list counts after it.
150
+ the list counts after it. A destructuring pattern that begins the list,
151
+ on the keyword's line, declares the names it binds: a shorthand key, the
152
+ target after `key:`, an array element, a rest `...NAME` and the names of
153
+ a nested pattern, each on the line where it appears (the pattern may
154
+ span lines), with its statement's column. A renamed or computed key, a
155
+ default value, the type annotation after the pattern and anything inside
156
+ a string or a comment bind nothing, and neither does a pattern that does
157
+ not close. The later declarators after a pattern count as any others. A
158
+ pattern that is a later declarator (`export const a = 1, { b } = c`) or
159
+ that sits on the line after a split keyword (`export const`, then
160
+ ` { b } = c;`) is not covered: it declares nothing.
135
161
 
136
162
  Indentation: for a dotted name every match counts. For an undotted name,
137
163
  matches at any indentation count, but when the file has a column-0 match
@@ -139,7 +165,8 @@ the indented ones are dropped, so a local shadow never outranks the
139
165
  module-level definition. A line inside a multi-line import, export list
140
166
  or `module.exports` object takes the column of the line its statement
141
167
  starts on, and so do the name line of a declaration split after its
142
- keyword and a later declarator on a line of its own. When the recorded
168
+ keyword, a later declarator on a line of its own and the line of a name
169
+ a destructuring pattern binds. When the recorded
143
170
  line is itself an indented match that this rule dropped (a method
144
171
  recorded under an undotted name beside a module-level function of the
145
172
  same name), the finding lists the recorded line too, so it carries
@@ -148,7 +175,11 @@ several lines and no caller auto-moves it.
148
175
  Entries in other files, and entries whose `export_type` is `module` or
149
176
  `package` (case-insensitive; neither has a definition line), are skipped
150
177
  and counted in `summary.line_check_skipped`. A `line-not-definition` item
151
- lists every defining line in the file in `definition_lines`. An empty list
178
+ lists every defining line in the file in `definition_lines`, the recorded
179
+ line included when it is an indented match the column-0 rule dropped (see
180
+ Indentation). The one exception is a Python `@overload` group: it lists
181
+ the group's first line alone, unless the recorded line is such a dropped
182
+ indented match, which still gives several lines. An empty list
152
183
  means the rules found no definition line, which may be a shape they do not
153
184
  cover: the entry is unverified, not gone, and no caller auto-fixes it.
154
185
 
@@ -386,23 +417,30 @@ Subcommands:
386
417
  }
387
418
 
388
419
  With `--line`, a recorded line that is an indented match the column-0
389
- rule dropped joins `definition_lines`, as in `verify`.
420
+ rule dropped joins `definition_lines`, and one that misses a Python
421
+ `@overload` group gives its first line alone, as in `verify`.
390
422
 
391
423
  classify-stale --names <coverage.json> --provenance <provenance-map.json>
392
- --source-root <path> [-o <out.json>]
424
+ --source-root <path> [--inventory <inventory.json>]
425
+ [-o <out.json>]
393
426
 
394
427
  test-skill's fabricated-signature test, for each stale name (a
395
428
  documented name the enumerated source surface lacks): the
396
429
  definition-lines rules above, run for every provenance entry whose raw
397
430
  `export_name` is the name, at its `source_file` and `source_line`, with
398
431
  its `export_type`. `--names` holds reconcile-coverage.py's result (its
399
- `stale` names) or a JSON array of names.
432
+ `stale` names) or a JSON array of names. `--inventory` holds the
433
+ documented inventory, validate-inventory.py's `--output` file or its
434
+ whole result: the `kind` it gives each name decides the module rule
435
+ below.
400
436
 
401
437
  Emit JSON, one item per name, in the order given (a repeated name once):
402
438
  [
403
439
  {"name": "<as given>",
404
440
  "fabricated": <bool>,
405
441
  "source": "<source_file>:<source_line>" | "<source_file>" | null,
442
+ "defined_at": "<path>:<line>" | null,
443
+ "declared_in": ["<path>:<line>", ...] | null,
406
444
  "reason": "not-defined" | "defined" | "unchecked" | "no-entry",
407
445
  "entries": [{"source_file": "<as recorded>",
408
446
  "source_line": <as recorded>,
@@ -428,6 +466,81 @@ Subcommands:
428
466
  entry with a `source_file` (the file alone when its line is not a
429
467
  number), null when no entry has one.
430
468
 
469
+ `defined_at` is where the source declares a name that is not
470
+ fabricated, so test-skill can tell a documented extra (a name outside
471
+ the surface that the source still declares) from stale documentation;
472
+ a fabricated name is not looked up (null). A declaration is a line at
473
+ column 0 that the definition-line rules match with imports left out:
474
+ a `def`, `class`, assignment, annotation, type alias, TS/JS
475
+ declaration, `exports.NAME =`, or an alias that renames
476
+ (`import X as NAME`, `from X import Y as NAME`,
477
+ `export { Y as NAME }`, `export * as NAME from`), or a
478
+ `module.exports` key with a value or a method. A plain or star import,
479
+ an `as` that keeps the imported name (`from m import NAME as NAME`,
480
+ `export { NAME as NAME }`), `export import NAME =`,
481
+ `export default NAME`, an export-list item without `as`, a
482
+ `module.exports` shorthand key, a binding whose value is an import
483
+ (`require(...)` or `(await) import(...)`, on its line or the next:
484
+ `const NAME = require(...)`, `const { NAME } = require(...)`,
485
+ `exports.NAME = require(...)`, `{ NAME: require(...) }`), an
486
+ indented line and any line of a Python
487
+ file that does not tokenize never count. A module or package named
488
+ NAME (`NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
489
+ `NAME/index.<ext>`) declares NAME too: at its line 1 when no line of it
490
+ declares NAME, unless `--inventory` gives NAME a `function`, `hook`,
491
+ `class`, `interface`, `enum` or `type` kind (any of its kinds, when it
492
+ gives several; a `method` kind is not read), no entry of NAME is a
493
+ `module` or `package` one, and the file is not a TS/JS one that is
494
+ itself the export (a column-0 `export default` or `module.exports =`),
495
+ so that a removed function named after its file is no declaration
496
+ (the inventory has no `module` kind). `declared_line` gives this
497
+ answer for one file, the module rule optional, to a script that loads
498
+ this one, and
499
+ `baseline_gaps` applies it to the map's entries an extraction left
500
+ out (skf-extraction-snapshot.py's baseline gaps and
501
+ skf-build-change-manifest.py's `baseline-gaps`, which apply the module
502
+ rule only to a `module` or `package` entry).
503
+
504
+ The files: when a file the name's entries cite declares it,
505
+ `defined_at` is that file's first declaration (the files in entry
506
+ order), and `declared_in` is null. Otherwise `declared_in` lists, as
507
+ sorted `<path>:<line>`, every file of the walk that declares the name,
508
+ at its first declaration (a type stub, `NAME.pyi` or `NAME.d.ts`,
509
+ beside its implementation is one file with it: the implementation),
510
+ and `defined_at` is the one item when the list has exactly one and no
511
+ walked Python file that does not tokenize holds the name (a homonym,
512
+ two files or more, gives null: test-skill decides it by the files the
513
+ skill cites). The walk reads, sorted by path, every file below the
514
+ walk root: the deepest folder that the files of the map's entries
515
+ share, widened to its package root (Python: the nearest folder at or
516
+ above it that holds `__init__.py`, then up while the parent holds
517
+ one, never the source root itself; TS/JS: the nearest folder at or
518
+ above it that holds `package.json`), leaving out the entries whose
519
+ file lies outside every package; the source root when none lies in
520
+ one. Only the rule families (Python, TS/JS) the map's entries cite are
521
+ read (both when no entry cites a file). The walk skips symlinks, hidden folders, test
522
+ files and test configuration files (`conftest.py`, `jest.config.*`),
523
+ `*.min.js`, and the folders generated, built, installed, vendored or
524
+ test output lives in (`node_modules`, `dist`, `build`, `vendor`,
525
+ `_vendor`, `tests`, `testing`, `fixtures`, `__mocks__`,
526
+ `site-packages`, `env`, `bower_components`, `esm`, `cjs`, `umd` and
527
+ the like), `docs`, `examples` and `benchmarks`, each judged on the
528
+ path below the walk root, the end-to-end test folders `e2e` and
529
+ `__fixtures__` wherever they lie, and, as the first folder below the
530
+ walk root or below a package root inside it (a folder that holds
531
+ `package.json`, the top of an `__init__.py` chain), `example`, `demo`,
532
+ `demos`, `sample`, `samples`, `scripts`, `playground`, `bench`,
533
+ `benchmark` and `migrations` unless it holds a file an entry cites
534
+ (deeper in a package such a folder can be package code). A cited file
535
+ is read only inside the source root (a path that is absolute or climbs
536
+ out of it is not read), with no symlink on its path and under the same
537
+ skips, those walk-only ones aside. The path is relative to the source
538
+ root. A dotted name, or a map whose entries cite files
539
+ but no Python or TS/JS one, is not looked up (`defined_at` and
540
+ `declared_in` null, as for a fabricated name), and a filesystem error
541
+ during the lookup gives every name null in both (the exit code stays
542
+ 0).
543
+
431
544
  kind-at --file <path> --line <n> --recipes <recipes>
432
545
  [--language <lang>] [--name <export_name>]
433
546
  [--source-root <path>] [-o <out.json>]
@@ -483,7 +596,8 @@ CLI examples:
483
596
  uv run skf-verify-provenance-completeness.py classify-stale \\
484
597
  --names {run_dir}/coverage.json \\
485
598
  --provenance {forge_version}/provenance-map.json \\
486
- --source-root {source_root} -o {run_dir}/stale.json
599
+ --source-root {source_root} --inventory {run_dir}/inventory.json \\
600
+ -o {run_dir}/stale.json
487
601
  uv run skf-verify-provenance-completeness.py kind-at \\
488
602
  --source-root {source_root} --file src/api.ts --line 12 --name Store \\
489
603
  --recipes {extractionPatternsData}
@@ -507,16 +621,17 @@ Exit codes:
507
621
  2 error, for every subcommand (an input file or folder not found,
508
622
  --skill-dir without a SKILL.md, malformed JSON or YAML, a verify
509
623
  result without stale[] and citations[], --names without a list of
510
- names, no recipe in --recipes, no PyYAML for kind-at, an atomic
511
- write that failed, an -o file that cannot be written, or any
512
- unexpected failure, such as an unreadable
513
- file; once fix has started writing, stderr names the files already
514
- written); one line on stderr and no JSON
624
+ names, --inventory without an `exports` list, no recipe in
625
+ --recipes, no PyYAML for kind-at, an atomic write that failed, an -o
626
+ file that cannot be written, or any unexpected failure, such as an
627
+ unreadable file; once fix has started writing, stderr names the files
628
+ already written); one line on stderr and no JSON
515
629
  """
516
630
 
517
631
  from __future__ import annotations
518
632
 
519
633
  import argparse
634
+ import bisect
520
635
  import copy
521
636
  import importlib.util
522
637
  import io
@@ -828,6 +943,13 @@ _PY_TARGET_LIST_RE = re.compile(
828
943
  )
829
944
  # `def \` or `class \`: the name is on the next line.
830
945
  _PY_SPLIT_DECL_RE = re.compile(r"^\s*(?:async\s+def|def|class)\s*\\$")
946
+ # A line that can matter to a name it does not hold: an import (which may go
947
+ # on over the next lines) or a `def \` / `class \` whose name is on the next.
948
+ _PY_NAMELESS_MATCH_RE = re.compile(r"^\s*(?:from|import|async|def|class)\b")
949
+ # The decorator of an `@overload` stub, imported by name or used through its
950
+ # module under any name (`typing`, `typing_extensions`, `t` after
951
+ # `import typing as t`).
952
+ _PY_OVERLOAD_RE = re.compile(r"^\s*@\s*(?:[A-Za-z_]\w*\.)?overload\s*$")
831
953
  # Tokens that never begin a statement.
832
954
  _PY_LAYOUT_TOKENS = frozenset(
833
955
  {
@@ -861,6 +983,15 @@ _JS_EXPORT_ITEM_RE = re.compile(
861
983
  rf"^(?:type\s+)?({_JS_NAME_OR_STRING})(?:\s+as\s+({_JS_NAME_OR_STRING}))?$"
862
984
  )
863
985
  _JS_MODULE_EXPORTS_OBJECT_RE = re.compile(r"^\s*module\.exports\s*=\s*\{(.*)$")
986
+ # A bound value that is an import: `require(...)`, `import(...)`, awaited, in
987
+ # parentheses (`(await import("./m")).X`) or in one helper call
988
+ # (`__importDefault(require("./m"))`).
989
+ _JS_IMPORT_VALUE_RE = re.compile(
990
+ r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?(?:await\s+)?(?:require|import)\s*\("
991
+ )
992
+ # The start of such a value that goes on on the next line: nothing yet, a
993
+ # parenthesis, or a helper call's name and parenthesis.
994
+ _JS_IMPORT_VALUE_OPEN_RE = re.compile(r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?$")
864
995
  # A `const`, `let` or `var` statement: its declarator list.
865
996
  _JS_VAR_DECL_RE = re.compile(
866
997
  rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(?!enum(?![\w$]))(.*)$"
@@ -870,6 +1001,18 @@ _JS_SPLIT_DECL_RE = re.compile(
870
1001
  rf"^\s*{_JS_MODIFIERS}(?P<kind>function(?:\s*\*)?|class|interface|type"
871
1002
  r"|enum|namespace|let|var|const(?:\s+enum)?)$"
872
1003
  )
1004
+ # The keywords every line that opens a statement going on below holds (a
1005
+ # `const` / `let` / `var` list, a split declaration, `export {`,
1006
+ # `module.exports = {`): a line with none of them, and not the name, can
1007
+ # match no rule while no statement is open.
1008
+ _JS_OPENING_WORDS = ("const", "let", "var", "function", "class", "interface", "type", "enum", "namespace",
1009
+ "export", "module")
1010
+
1011
+
1012
+ def _js_quiet_lines(code_lines: list[str]) -> list[bool]:
1013
+ """For each comment-free line, True when it holds none of
1014
+ `_JS_OPENING_WORDS`."""
1015
+ return [not any(word in code for word in _JS_OPENING_WORDS) for code in code_lines]
873
1016
  _JS_VAR_KEYWORDS = frozenset({"const", "let", "var"})
874
1017
  # How a `const` / `let` / `var` list that did not end on its line goes on
875
1018
  # (`_js_declarators`): the next line begins a declarator (this one ended
@@ -883,6 +1026,11 @@ _DECL_MAYBE = "maybe"
883
1026
  # What `_js_declarators` gives: None, or (brackets left open, how) the list
884
1027
  # goes on with, and the pieces that begin a declarator.
885
1028
  _DeclSplit = tuple[tuple[str, str] | None, list[str]]
1029
+ # What `_js_destructured` gives for a `const` / `let` / `var` statement
1030
+ # whose first declarator is a destructuring pattern: the (name, line) of
1031
+ # each name it binds, and the (line, text after the `=` on that line) its
1032
+ # initializer starts at, None when it has none.
1033
+ _PatternDecl = tuple[list[tuple[str, int]], tuple[int, str] | None]
886
1034
  # After one of these, `/` begins a regular expression and `<` begins JSX, a
887
1035
  # type assertion or a generic arrow function's type parameters.
888
1036
  _JS_EXPRESSION_START = frozenset("([{,;=?:&|^%+-*/<>!~")
@@ -909,6 +1057,21 @@ def _read_lines(path: Path) -> list[str] | None:
909
1057
  return None
910
1058
 
911
1059
 
1060
+ def _read_source(path: Path) -> str:
1061
+ """A text file's content, decoded as `_read_lines` decodes it. Raises
1062
+ OSError when the file cannot be read."""
1063
+ with path.open("r", encoding="utf-8-sig", errors="replace") as fh:
1064
+ return fh.read()
1065
+
1066
+
1067
+ def _text_lines(text: str) -> list[str]:
1068
+ """`_read_source`'s text as the lines `_read_lines` gives for the file."""
1069
+ lines = text.split("\n")
1070
+ if lines[-1] == "":
1071
+ lines.pop()
1072
+ return lines
1073
+
1074
+
912
1075
  def _is_column_0(text: str) -> bool:
913
1076
  """True when a line starts at column 0 (no leading whitespace)."""
914
1077
  return not text[:1].isspace()
@@ -984,14 +1147,17 @@ def _js_code_lines(lines: list[str]) -> list[str]:
984
1147
 
985
1148
 
986
1149
  class _SourceCache:
987
- """Source lines, Python statement starts and TS/JS comment-free lines,
988
- read once per file, and TS/JS declarator lists, split once per line
989
- (the split does not depend on the export looked up)."""
1150
+ """Source lines, Python statement starts, TS/JS comment-free lines and
1151
+ the names TS/JS destructuring patterns bind, read once per file, and
1152
+ TS/JS declarator lists, split once per line (neither depends on the
1153
+ export looked up)."""
990
1154
 
991
1155
  def __init__(self) -> None:
992
1156
  self._lines: dict[Path, list[str] | None] = {}
993
1157
  self._starts: dict[Path, set[int] | None] = {}
994
1158
  self._js_code: dict[Path, list[str]] = {}
1159
+ self._js_quiet: dict[Path, list[bool]] = {}
1160
+ self._js_patterns: dict[Path, dict[int, _PatternDecl]] = {}
995
1161
  self._js_decl: dict[tuple[str, str, bool], _DeclSplit] = {}
996
1162
 
997
1163
  def lines(self, path: Path) -> list[str] | None:
@@ -1009,6 +1175,18 @@ class _SourceCache:
1009
1175
  self._js_code[path] = _js_code_lines(lines)
1010
1176
  return self._js_code[path]
1011
1177
 
1178
+ def js_quiet_lines(self, path: Path, code_lines: list[str]) -> list[bool]:
1179
+ if path not in self._js_quiet:
1180
+ self._js_quiet[path] = _js_quiet_lines(code_lines)
1181
+ return self._js_quiet[path]
1182
+
1183
+ def js_destructured(
1184
+ self, path: Path, code_lines: list[str]
1185
+ ) -> dict[int, _PatternDecl]:
1186
+ if path not in self._js_patterns:
1187
+ self._js_patterns[path] = _js_destructured(code_lines)
1188
+ return self._js_patterns[path]
1189
+
1012
1190
  def js_declarators(
1013
1191
  self, text: str, opened: str = "", fresh: bool = True
1014
1192
  ) -> _DeclSplit:
@@ -1018,8 +1196,12 @@ class _SourceCache:
1018
1196
  return self._js_decl[key]
1019
1197
 
1020
1198
 
1021
- def _py_import_binds(items: str, name: str, from_import: bool) -> bool:
1022
- """True when an import item list binds `name`."""
1199
+ def _py_import_binds(
1200
+ items: str, name: str, from_import: bool, alias_only: bool = False
1201
+ ) -> bool:
1202
+ """True when an import item list binds `name` (with `alias_only`, only
1203
+ through an explicit `as name` that renames: `from m import name as
1204
+ name` and `import a.name as name` keep the imported name)."""
1023
1205
  items = items.split("#", 1)[0]
1024
1206
  for raw in items.replace("(", " ").replace(")", " ").replace("\\", " ").split(","):
1025
1207
  m = _PY_IMPORT_ITEM_RE.match(raw.strip())
@@ -1027,7 +1209,11 @@ def _py_import_binds(items: str, name: str, from_import: bool) -> bool:
1027
1209
  continue
1028
1210
  target, alias = m.group(1), m.group(2)
1029
1211
  if alias is not None:
1212
+ if alias_only and alias == target.rsplit(".", 1)[-1]:
1213
+ continue
1030
1214
  bound = alias
1215
+ elif alias_only:
1216
+ continue
1031
1217
  elif from_import:
1032
1218
  bound = target
1033
1219
  else:
@@ -1050,7 +1236,10 @@ def _py_assigned_names(code: str) -> set[str]:
1050
1236
 
1051
1237
 
1052
1238
  def _python_definition_lines(
1053
- lines: list[str], name: str, starts: set[int] | None
1239
+ lines: list[str],
1240
+ name: str,
1241
+ starts: set[int] | None,
1242
+ declarations_only: bool = False,
1054
1243
  ) -> list[tuple[int, bool]]:
1055
1244
  """(line, at column 0) for every line that defines `name`.
1056
1245
 
@@ -1059,7 +1248,9 @@ def _python_definition_lines(
1059
1248
  line of a `def \\` or `class \\` whose name is on the next line. A
1060
1249
  `from X import *` line counts only when no other line defines `name` (a
1061
1250
  re-export the extraction could not trace further is cited at its star
1062
- import).
1251
+ import). With `declarations_only`, an import counts only when it binds
1252
+ `name` through an explicit `as name` that renames (`_py_import_binds`),
1253
+ and a star import never does.
1063
1254
  """
1064
1255
  esc = re.escape(name)
1065
1256
  decl = re.compile(rf"^\s*(?:async\s+def|def|class)\s+{esc}(?!\w)")
@@ -1085,11 +1276,13 @@ def _python_definition_lines(
1085
1276
  code = stripped[:-1]
1086
1277
  else:
1087
1278
  cont = None
1088
- if _py_import_binds(code, name, from_import):
1279
+ if _py_import_binds(code, name, from_import, declarations_only):
1089
1280
  found.append((number, top))
1090
1281
  continue
1091
1282
  if starts is not None and number not in starts:
1092
1283
  continue
1284
+ if name not in code and not _PY_NAMELESS_MATCH_RE.match(code):
1285
+ continue # no rule below can match it: spares a scan per name
1093
1286
  top = _is_column_0(text)
1094
1287
  if (
1095
1288
  decl.match(code)
@@ -1114,27 +1307,103 @@ def _python_definition_lines(
1114
1307
  continue
1115
1308
  rest = m.group(1)
1116
1309
  if from_import and rest.strip() == "*":
1117
- star_imports.append((number, top))
1310
+ if not declarations_only:
1311
+ star_imports.append((number, top))
1118
1312
  continue
1119
1313
  if from_import and rest.lstrip().startswith("(") and ")" not in rest:
1120
1314
  cont = ("paren", top, True)
1121
1315
  elif rest.rstrip().endswith("\\"):
1122
1316
  cont = ("backslash", top, from_import)
1123
1317
  rest = rest.rstrip()[:-1]
1124
- if _py_import_binds(rest, name, from_import):
1318
+ if _py_import_binds(rest, name, from_import, declarations_only):
1125
1319
  found.append((number, top))
1126
1320
  return found or star_imports
1127
1321
 
1128
1322
 
1129
- def _js_export_item_binds(item: str, name: str) -> bool:
1323
+ def _py_overloaded(lines: list[str], number: int, starts: set[int] | None) -> bool:
1324
+ """True when the `def` on line `number` has `@overload` or
1325
+ `@<module>.overload` (`@typing.overload`, `@t.overload`) in its
1326
+ contiguous decorator block: the decorator lines at its indentation
1327
+ right above it, up to a blank line or a line that is not a decorator.
1328
+ A comment line is passed over, and so, in a file that tokenizes, is a
1329
+ line on which no statement begins (a decorator's arguments over
1330
+ several lines)."""
1331
+ text = lines[number - 1]
1332
+ indent = text[: len(text) - len(text.lstrip())]
1333
+ i = number - 1
1334
+ while i >= 1:
1335
+ text = lines[i - 1]
1336
+ code = text.split("#", 1)[0]
1337
+ if not text.strip():
1338
+ return False
1339
+ if not code.strip() or (starts is not None and i not in starts):
1340
+ i -= 1
1341
+ continue
1342
+ if not code.lstrip().startswith("@") or text[: len(text) - len(text.lstrip())] != indent:
1343
+ return False
1344
+ if _PY_OVERLOAD_RE.match(code):
1345
+ return True
1346
+ i -= 1
1347
+ return False
1348
+
1349
+
1350
+ def _py_enclosing_line(
1351
+ lines: list[str], number: int, indent: str, starts: set[int] | None
1352
+ ) -> int | None:
1353
+ """The line of the block that holds the statement on line `number`,
1354
+ indented by `indent`: the nearest line above it where a statement
1355
+ begins with less indentation. None at module level."""
1356
+ i = number - 1
1357
+ while indent and i >= 1:
1358
+ text = lines[i - 1]
1359
+ if text.split("#", 1)[0].strip() and (starts is None or i in starts):
1360
+ if len(text) - len(text.lstrip()) < len(indent):
1361
+ return i
1362
+ i -= 1
1363
+ return None
1364
+
1365
+
1366
+ def _python_overload_start(
1367
+ lines: list[str], name: str, defs: list[int], starts: set[int] | None
1368
+ ) -> int | None:
1369
+ """The first of `defs` when they are one `@overload` group: two lines
1370
+ or more, each a `def` / `async def NAME` line, all at one indentation
1371
+ in one enclosing block (`_py_enclosing_line`, so two classes'
1372
+ methods never make one group), and each but at most the last
1373
+ decorated as an overload (`_py_overloaded`). None otherwise."""
1374
+ if len(defs) < 2:
1375
+ return None
1376
+ decl = re.compile(rf"^(\s*)(?:async\s+def|def)\s+{re.escape(name)}(?!\w)")
1377
+ indents = set()
1378
+ for number in defs:
1379
+ m = decl.match(lines[number - 1].split("#", 1)[0])
1380
+ if m is None:
1381
+ return None
1382
+ indents.add(m.group(1))
1383
+ if len(indents) > 1:
1384
+ return None
1385
+ (indent,) = indents
1386
+ if len({_py_enclosing_line(lines, number, indent, starts) for number in defs}) > 1:
1387
+ return None
1388
+ if not all(_py_overloaded(lines, number, starts) for number in defs[:-1]):
1389
+ return None
1390
+ return defs[0]
1391
+
1392
+
1393
+ def _js_export_item_binds(item: str, name: str, alias_only: bool = False) -> bool:
1130
1394
  """True when one `export { ... }` item exposes `name`: the name after
1131
- `as`, else the item's own name, an identifier or a string."""
1395
+ `as`, else (unless `alias_only`) the item's own name, an identifier or
1396
+ a string. With `alias_only`, an `as` that keeps the item's own name
1397
+ (`NAME as NAME`) does not count either."""
1132
1398
  m = _JS_EXPORT_ITEM_RE.match(item.strip())
1133
- if not m:
1399
+ if not m or (alias_only and m.group(2) is None):
1400
+ return False
1401
+ own, exposed = (
1402
+ s[1:-1] if s[:1] in ("'", '"') else s
1403
+ for s in (m.group(1), m.group(2) or m.group(1))
1404
+ )
1405
+ if alias_only and exposed == own:
1134
1406
  return False
1135
- exposed = m.group(2) or m.group(1)
1136
- if exposed[:1] in ("'", '"'):
1137
- exposed = exposed[1:-1]
1138
1407
  return exposed == name
1139
1408
 
1140
1409
 
@@ -1321,46 +1590,364 @@ def _js_paren_tail_is_call(after_open: str) -> bool:
1321
1590
  return False
1322
1591
 
1323
1592
 
1593
+ def _js_pattern_bindings(
1594
+ text: str, start: int
1595
+ ) -> tuple[list[tuple[str, int]], int | None]:
1596
+ """The names the destructuring pattern whose `{` or `[` is at
1597
+ `text[start]` binds, each with its offset in `text`, and the offset
1598
+ just past the pattern's closing bracket.
1599
+
1600
+ An object pattern's property is `...target`, a shorthand key or
1601
+ `key: target` (a key is a name, a string, a number or a computed
1602
+ `[...]`, and only a shorthand key binds itself); an array pattern's
1603
+ element is `...target`, a target or a hole; a target is a name or a
1604
+ nested pattern. A `= default` is skipped to the next comma at the
1605
+ pattern's depth, and so is a property or element of any other shape,
1606
+ which binds nothing. '...', "..." and `...` strings are skipped
1607
+ whole, so no bracket or comma inside one counts. Whitespace, newlines
1608
+ included, may sit anywhere between the parts.
1609
+
1610
+ Returns ([], None) when `text[start]` opens no pattern, the pattern
1611
+ does not close in `text`, a string in it does not end, or a bracket in
1612
+ it closes one it did not open. A pure function of its arguments (no
1613
+ module state), so another script can carry a copy of it.
1614
+ """
1615
+ n = len(text)
1616
+ names: list[tuple[str, int]] = []
1617
+
1618
+ def space(i: int) -> int:
1619
+ while i < n and text[i].isspace():
1620
+ i += 1
1621
+ return i
1622
+
1623
+ def name_end(i: int) -> int:
1624
+ """The end of the identifier at `i`, `i` itself when none starts there."""
1625
+ if i < n and (text[i].isalpha() or text[i] in "_$"):
1626
+ i += 1
1627
+ while i < n and (text[i].isalnum() or text[i] in "_$"):
1628
+ i += 1
1629
+ return i
1630
+
1631
+ def string_end(i: int) -> int | None:
1632
+ """The offset past the string whose quote is at `i`."""
1633
+ quote = text[i]
1634
+ i += 1
1635
+ while i < n:
1636
+ if text[i] == "\\":
1637
+ i += 2
1638
+ elif text[i] == quote:
1639
+ return i + 1
1640
+ else:
1641
+ i += 1
1642
+ return None
1643
+
1644
+ def skip(i: int, to_comma: bool = True) -> int | None:
1645
+ """From `i`, the offset of the next comma (unless not `to_comma`)
1646
+ or closing bracket at depth 0; with `to_comma` False, `text[i]`
1647
+ opens a bracket and the offset past the one closing it."""
1648
+ depth = 0
1649
+ while i < n:
1650
+ ch = text[i]
1651
+ if ch in "'\"`":
1652
+ end = string_end(i)
1653
+ if end is None:
1654
+ return None
1655
+ i = end
1656
+ continue
1657
+ if ch in "([{":
1658
+ depth += 1
1659
+ elif ch in ")]}":
1660
+ if depth == 0:
1661
+ return i
1662
+ depth -= 1
1663
+ if depth == 0 and not to_comma:
1664
+ return i + 1
1665
+ elif ch == "," and depth == 0 and to_comma:
1666
+ return i
1667
+ i += 1
1668
+ return None
1669
+
1670
+ def default(i: int) -> int | None:
1671
+ """After a target: past its `= default`, when it has one."""
1672
+ i = space(i)
1673
+ if i < n and text[i] == "=" and text[i + 1 : i + 2] not in ("=", ">"):
1674
+ return skip(i + 1)
1675
+ return i
1676
+
1677
+ def bind(i: int, end: int, close: str) -> int | None:
1678
+ """The name `text[i:end]` is bound when its property or element
1679
+ ends after it or takes a default (any other text: another shape)."""
1680
+ after = space(end)
1681
+ if after < n and (
1682
+ text[after] in (",", close)
1683
+ or text[after] == "=" and text[after + 1 : after + 2] not in ("=", ">")
1684
+ ):
1685
+ names.append((text[i:end], i))
1686
+ return default(after)
1687
+ return after
1688
+
1689
+ def target(i: int, close: str) -> int | None:
1690
+ if i < n and text[i] in "{[":
1691
+ end = pattern(i)
1692
+ return None if end is None else default(end)
1693
+ end = name_end(i)
1694
+ return i if end == i else bind(i, end, close)
1695
+
1696
+ def element(i: int, close: str) -> int | None:
1697
+ """One property or element at `i`: the offset after it."""
1698
+ if text.startswith("...", i):
1699
+ return target(space(i + 3), close)
1700
+ if close == "]":
1701
+ return target(i, close)
1702
+ if text[i] in "'\"":
1703
+ end = string_end(i)
1704
+ elif text[i] == "[":
1705
+ end = skip(i, to_comma=False) # a computed key
1706
+ else:
1707
+ end = name_end(i)
1708
+ if end > i:
1709
+ after = space(end)
1710
+ if after < n and text[after] == ":":
1711
+ return target(space(after + 1), close)
1712
+ return bind(i, end, close) # a shorthand key
1713
+ while end < n and (text[end].isalnum() or text[end] in "._"):
1714
+ end += 1 # a number key
1715
+ if end is None:
1716
+ return None
1717
+ after = space(end)
1718
+ if end > i and after < n and text[after] == ":":
1719
+ return target(space(after + 1), close)
1720
+ return after
1721
+
1722
+ def pattern(i: int) -> int | None:
1723
+ """The pattern opening at `i`: the offset past its closing bracket."""
1724
+ close = "}" if text[i] == "{" else "]"
1725
+ i += 1
1726
+ while True:
1727
+ i = space(i)
1728
+ if i >= n:
1729
+ return None
1730
+ if text[i] == close:
1731
+ return i + 1
1732
+ if text[i] in ")]}":
1733
+ return None
1734
+ if text[i] == ",":
1735
+ i += 1 # a hole, or a stray comma
1736
+ continue
1737
+ end = element(i, close)
1738
+ if end is None:
1739
+ return None
1740
+ end = space(end)
1741
+ if end < n and text[end] not in (",", close):
1742
+ end = skip(end) # any other shape binds nothing
1743
+ if end is None:
1744
+ return None
1745
+ i = end + 1 if end < n and text[end] == "," else end
1746
+
1747
+ if not (0 <= start < n and text[start] in "{["):
1748
+ return [], None
1749
+ end = pattern(start)
1750
+ return (names, end) if end is not None else ([], None)
1751
+
1752
+
1753
+ def _js_initializer_start(text: str, end: int) -> int | None:
1754
+ """The offset just past the `=` that begins the initializer of a
1755
+ declarator whose name or pattern ends at `end`, a type annotation
1756
+ (`: T`, brackets, strings and `=>` in it included) skipped. None when
1757
+ neither `=` nor `:` follows, or when a `,` or `;` outside the
1758
+ annotation's brackets, or the end of `text`, comes first."""
1759
+ n = len(text)
1760
+ i = end
1761
+ while i < n and text[i].isspace():
1762
+ i += 1
1763
+ if i < n and text[i] == "=":
1764
+ return None if text[i + 1 : i + 2] == "=" else i + 1
1765
+ if not (i < n and text[i] == ":"):
1766
+ return None
1767
+ i += 1
1768
+ depth = 0
1769
+ while i < n:
1770
+ ch = text[i]
1771
+ if ch in "'\"`":
1772
+ i += 1
1773
+ while i < n and text[i] != ch:
1774
+ i += 2 if text[i] == "\\" else 1
1775
+ elif text.startswith("=>", i):
1776
+ i += 1
1777
+ elif ch in "([{<":
1778
+ depth += 1
1779
+ elif ch in ")]}>":
1780
+ depth -= 1
1781
+ if depth < 0:
1782
+ return None
1783
+ elif ch in ",;" and depth == 0:
1784
+ return None
1785
+ elif ch == "=" and depth == 0:
1786
+ return None if text[i + 1 : i + 2] == "=" else i + 1
1787
+ i += 1
1788
+ return None
1789
+
1790
+
1791
+ def _js_destructured(code_lines: list[str]) -> dict[int, _PatternDecl]:
1792
+ """For each line that opens a `const` / `let` / `var` statement whose
1793
+ first declarator is a destructuring pattern (`_JS_VAR_DECL_RE`, the
1794
+ rest starting with `{` or `[`): the names the pattern binds, each with
1795
+ the line it appears on, and where its initializer starts.
1796
+
1797
+ The patterns are read from the comment-free lines joined with `\\n`
1798
+ (`_js_pattern_bindings`), so a pattern may span lines and a name's
1799
+ offset maps back to its line. A pattern that does not close, or that
1800
+ nests deeper than the interpreter's recursion limit, binds nothing and
1801
+ gives no item.
1802
+ """
1803
+ starts: list[tuple[int, int]] = []
1804
+ for number, code in enumerate(code_lines, start=1):
1805
+ if "{" not in code and "[" not in code:
1806
+ continue
1807
+ m = _JS_VAR_DECL_RE.match(code)
1808
+ if m is not None and m.group(1)[:1] in ("{", "["):
1809
+ starts.append((number, m.start(1)))
1810
+ if not starts:
1811
+ return {}
1812
+ text = "\n".join(code_lines)
1813
+ offsets: list[int] = []
1814
+ pos = 0
1815
+ for code in code_lines:
1816
+ offsets.append(pos)
1817
+ pos += len(code) + 1
1818
+
1819
+ def line_of(offset: int) -> int:
1820
+ return bisect.bisect_right(offsets, offset)
1821
+
1822
+ out: dict[int, _PatternDecl] = {}
1823
+ for number, column in starts:
1824
+ try:
1825
+ names, end = _js_pattern_bindings(text, offsets[number - 1] + column)
1826
+ except RecursionError:
1827
+ continue # patterns nested past the interpreter's limit: none bound
1828
+ if end is None:
1829
+ continue
1830
+ value = _js_initializer_start(text, end)
1831
+ at = None
1832
+ if value is not None:
1833
+ line = line_of(value)
1834
+ at = (line, code_lines[line - 1][value - offsets[line - 1] :])
1835
+ out[number] = ([(name, line_of(offset)) for name, offset in names], at)
1836
+ return out
1837
+
1838
+
1324
1839
  def _tsjs_definition_lines(
1325
1840
  lines: list[str],
1326
1841
  code_lines: list[str],
1327
1842
  name: str,
1328
1843
  dotted: bool,
1329
1844
  declarators: Callable[..., _DeclSplit] = _js_declarators,
1845
+ declarations_only: bool = False,
1846
+ quiet: list[bool] | None = None,
1847
+ destructured: dict[int, _PatternDecl] | None = None,
1330
1848
  ) -> list[tuple[int, bool]]:
1331
1849
  """(line, at column 0) for every line that defines `name`.
1332
1850
 
1333
- `code_lines` are `lines` with comments removed (`_js_code_lines`), so a
1851
+ `quiet` is `_js_quiet_lines(code_lines)` when the caller has it (it does
1852
+ not depend on the name): a quiet line without the name is passed over
1853
+ while no statement is open. `code_lines` are `lines` with comments removed (`_js_code_lines`), so a
1334
1854
  JSDoc line or a commented-out declaration never matches. A line inside
1335
1855
  a multi-line `export { ... }` list or `module.exports = { ... }` object
1336
1856
  takes the column of the line the statement starts on, and so do the
1337
- name line of a declaration whose keyword ends the line above it and a
1338
- later declarator on a line of its own. Class members count only for a
1857
+ name line of a declaration whose keyword ends the line above it, a
1858
+ later declarator on a line of its own and the line of a name that a
1859
+ destructuring pattern binds. `destructured` is
1860
+ `_js_destructured(code_lines)` when the caller has it (it does not
1861
+ depend on the name either): a name its pattern binds counts when the
1862
+ `const` / `let` / `var` line it is keyed by opens a statement here.
1863
+ Class members count only for a
1339
1864
  dotted export name. `declarators` splits a declarator list
1340
1865
  (`_js_declarators`, or `_SourceCache.js_declarators`, which splits each
1341
- line once for every export looked up).
1866
+ line once for every export looked up). With `declarations_only`,
1867
+ `export import NAME =`, `export default NAME`, an export-list item
1868
+ without an `as` that renames, a `module.exports` shorthand key and an
1869
+ import-valued binding (a declarator, a destructured name, `exports.NAME =`
1870
+ or a `module.exports` key whose value starts with `require(` or an
1871
+ `(await) import(`, on its line or the next one) never count, nor does
1872
+ a class member.
1342
1873
  """
1343
1874
  esc = re.escape(name)
1344
1875
  end = r"(?![\w$])"
1345
- statements = (
1876
+ statements = [
1346
1877
  re.compile(rf"^\s*{_JS_MODIFIERS}{_JS_KINDS}{esc}{end}"),
1347
- re.compile(rf"^\s*export\s+import\s+(?:type\s+)?{esc}\s*=(?![=>])"),
1348
1878
  re.compile(
1349
1879
  rf"^\s*export\s+(?:type\s+)?\*\s+as\s+"
1350
1880
  rf"(?:{esc}\s+|([\"']){esc}\1\s*)from{end}"
1351
1881
  ),
1352
- re.compile(rf"^\s*export\s+default\s+{esc}\s*;?\s*$"),
1353
1882
  re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])"),
1354
- )
1883
+ ]
1884
+ if not declarations_only:
1885
+ statements += [
1886
+ re.compile(rf"^\s*export\s+import\s+(?:type\s+)?{esc}\s*=(?![=>])"),
1887
+ re.compile(rf"^\s*export\s+default\s+{esc}\s*;?\s*$"),
1888
+ ]
1355
1889
  # One declarator of a `const` / `let` / `var` list: `NAME = v`,
1356
1890
  # `NAME: T = v`, `NAME!: T`, or a bare `NAME`. A destructuring pattern
1357
- # starts with `{` or `[` and never matches.
1891
+ # starts with `{` or `[` and never matches: the names a pattern that
1892
+ # begins the list binds come from `destructured` (`_js_pattern_bindings`).
1358
1893
  declarator = re.compile(rf"^\s*{esc}{end}\s*(?:!?\s*:|=|$)")
1894
+ # With declarations_only, the value bound to NAME (the rest of a
1895
+ # declarator, of `exports.NAME =` or of a `module.exports` key), to
1896
+ # test for an import.
1897
+ # (A type annotation holds no `=` but in `=>`: `NAME: () => T = v`.)
1898
+ declarator_value = re.compile(
1899
+ rf"^\s*{esc}{end}\s*(?:!?\s*:(?:[^=]|=>)*)?=(?![=>])(.*)$"
1900
+ )
1901
+ var_statement = re.compile(rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(.*)$")
1902
+ exports_value = re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])(.*)$")
1903
+ key_value = re.compile(rf"^\s*(?:(['\"]){esc}\1|{esc}{end})\s*:(.*)$")
1904
+
1905
+ def imported(value: str, number: int) -> bool:
1906
+ """True when a bound value is an import: `require(...)`, or an
1907
+ `import(...)` awaited or not, alone or in one helper call. A value
1908
+ that goes on on the next line (`NAME =`, `NAME = (` or
1909
+ `NAME = helper(` ends line `number`) is read there too."""
1910
+ if _JS_IMPORT_VALUE_OPEN_RE.match(value):
1911
+ value += next((c for c in code_lines[number:] if c.strip()), "")
1912
+ return _JS_IMPORT_VALUE_RE.match(value) is not None
1913
+
1914
+ def declares(piece: str, number: int) -> bool:
1915
+ if declarator.match(piece) is None:
1916
+ return False
1917
+ if not declarations_only:
1918
+ return True
1919
+ m = declarator_value.match(piece)
1920
+ return m is None or not imported(m.group(1), number)
1921
+
1922
+ def is_statement(code: str, number: int) -> bool:
1923
+ if not any(p.match(code) for p in statements):
1924
+ return False
1925
+ if not declarations_only:
1926
+ return True
1927
+ m = var_statement.match(code)
1928
+ if m is not None:
1929
+ m = declarator_value.match(m.group(1))
1930
+ else:
1931
+ m = exports_value.match(code)
1932
+ return m is None or not imported(m.group(1), number)
1933
+
1359
1934
  leading_name = re.compile(rf"^\s*{esc}{end}")
1935
+ # A key with a value, a method, or (unless declarations_only) a shorthand.
1936
+ key_end = r"[:(]" if declarations_only else r"[:(]|$"
1360
1937
  object_key = re.compile(
1361
1938
  rf"^\s*(?:(?:async|get|set)\s+|\*\s*)*(?:(['\"]){esc}\1|{esc}{end})"
1362
- rf"\s*(?:[:(]|$)"
1939
+ rf"\s*(?:{key_end})"
1363
1940
  )
1941
+
1942
+ def binds_key(piece: str, number: int) -> bool:
1943
+ if object_key.match(piece) is None:
1944
+ return False
1945
+ if not declarations_only:
1946
+ return True
1947
+ m = key_value.match(piece)
1948
+ return m is None or not imported(m.group(2), number)
1949
+
1950
+ dotted = dotted and not declarations_only
1364
1951
  member = re.compile(
1365
1952
  rf"^\s+(?P<mods>{_JS_MEMBER_MODIFIERS})(?:\*\s*)?{esc}{end}\s*[?!]?\s*"
1366
1953
  rf"(?:<[^>]*>\s*)?(?P<tail>[(:=;]|$)"
@@ -1375,7 +1962,27 @@ def _tsjs_definition_lines(
1375
1962
  # nested in another's initializer included: (column 0, brackets left
1376
1963
  # open, how, as `_js_declarators` gives them).
1377
1964
  decls: list[tuple[bool, str, str]] = []
1965
+ if quiet is None:
1966
+ quiet = _js_quiet_lines(code_lines)
1967
+ if destructured is None:
1968
+ destructured = _js_destructured(code_lines)
1969
+
1970
+ def pattern_binds(number: int) -> list[int]:
1971
+ """The lines on which the pattern that begins the `const` / `let` /
1972
+ `var` list of line `number` binds `name` (none with
1973
+ `declarations_only` when its value is an import)."""
1974
+ decl = destructured.get(number)
1975
+ if decl is None:
1976
+ return []
1977
+ bound, value = decl
1978
+ at = [line for bound_name, line in bound if bound_name == name]
1979
+ if at and declarations_only and value is not None and imported(value[1], value[0]):
1980
+ return []
1981
+ return at
1982
+
1378
1983
  for number, (text, code) in enumerate(zip(lines, code_lines), start=1):
1984
+ if quiet[number - 1] and not (decls or split or cont) and name not in code:
1985
+ continue # no rule below can match it: spares a scan per name
1379
1986
  # A line of a list carried from above still goes through the rules
1380
1987
  # below: it may hold a class member or a local of an arrow function.
1381
1988
  if decls and code.strip():
@@ -1387,7 +1994,7 @@ def _tsjs_definition_lines(
1387
1994
  carry, pieces = declarators(code, opened, how == _DECL_NEXT)
1388
1995
  if carry is not None:
1389
1996
  carried.append((decl_top, *carry))
1390
- if any(declarator.match(p) for p in pieces):
1997
+ if any(declares(p, number) for p in pieces):
1391
1998
  # a later declarator on a line of its own, as Prettier
1392
1999
  # writes it: `export const a = 1,` then ` NAME = 2;`
1393
2000
  found.append((number, decl_top))
@@ -1399,7 +2006,7 @@ def _tsjs_definition_lines(
1399
2006
  carry, pieces = declarators(code)
1400
2007
  if carry is not None:
1401
2008
  decls.append((top, *carry))
1402
- hit = any(declarator.match(p) for p in pieces)
2009
+ hit = any(declares(p, number) for p in pieces)
1403
2010
  else:
1404
2011
  hit = leading_name.match(code) is not None
1405
2012
  if hit:
@@ -1411,14 +2018,19 @@ def _tsjs_definition_lines(
1411
2018
  depth, pieces = _js_depth1_pieces(code, depth)
1412
2019
  cont = (kind, top, depth) if depth > 0 else None
1413
2020
  if kind == "list":
1414
- hit = any(_js_export_item_binds(p, name) for p in pieces)
2021
+ hit = any(
2022
+ _js_export_item_binds(p, name, declarations_only) for p in pieces
2023
+ )
1415
2024
  else:
1416
- hit = any(object_key.match(p) for p in pieces)
2025
+ hit = any(binds_key(p, number) for p in pieces)
1417
2026
  if hit:
1418
2027
  found.append((number, top))
1419
2028
  continue
1420
2029
  top = _is_column_0(text)
1421
- if any(p.match(code) for p in statements):
2030
+ # every rule that matches a line needs the name in it, but for a
2031
+ # destructuring pattern, whose names may sit on the lines below
2032
+ named = name in code
2033
+ if named and is_statement(code, number):
1422
2034
  found.append((number, top))
1423
2035
  continue
1424
2036
  m = _JS_VAR_DECL_RE.match(code)
@@ -1426,7 +2038,10 @@ def _tsjs_definition_lines(
1426
2038
  carry, pieces = declarators(m.group(1))
1427
2039
  if carry is not None:
1428
2040
  decls.append((top, *carry))
1429
- if any(declarator.match(p) for p in pieces):
2041
+ # `export const { a, b: NAME } = obj;`, on one line or several:
2042
+ # each name's line, at the statement's column
2043
+ found.extend((line, top) for line in pattern_binds(number))
2044
+ if named and any(declares(p, number) for p in pieces):
1430
2045
  # a later declarator: `export const a = 1, NAME = 2;`
1431
2046
  found.append((number, top))
1432
2047
  continue
@@ -1439,7 +2054,7 @@ def _tsjs_definition_lines(
1439
2054
  depth, pieces = _js_depth1_pieces(m.group(1), 1)
1440
2055
  if depth > 0:
1441
2056
  cont = ("list", top, depth)
1442
- if any(_js_export_item_binds(p, name) for p in pieces):
2057
+ if any(_js_export_item_binds(p, name, declarations_only) for p in pieces):
1443
2058
  found.append((number, top))
1444
2059
  continue
1445
2060
  m = _JS_MODULE_EXPORTS_OBJECT_RE.match(code)
@@ -1447,7 +2062,7 @@ def _tsjs_definition_lines(
1447
2062
  depth, pieces = _js_depth1_pieces(m.group(1), 1)
1448
2063
  if depth > 0:
1449
2064
  cont = ("object", top, depth)
1450
- if any(object_key.match(p) for p in pieces):
2065
+ if any(binds_key(p, number) for p in pieces):
1451
2066
  found.append((number, top))
1452
2067
  continue
1453
2068
  if not dotted:
@@ -1494,12 +2109,19 @@ def _definition_matches(
1494
2109
  export_name: str,
1495
2110
  source_root: Path,
1496
2111
  cache: _SourceCache | None = None,
2112
+ declarations_only: bool = False,
1497
2113
  ) -> tuple[list[int], set[int]] | None:
1498
2114
  """(definition lines to report, every matching line) for an export.
1499
2115
 
1500
2116
  The first item applies the column-0 rule; the second keeps the indented
1501
2117
  matches that rule dropped. None when the file's extension has no
1502
2118
  definition rule or the file cannot be read.
2119
+
2120
+ `declarations_only` (classify-stale's `defined_at` lookup) keeps the
2121
+ lines that declare the name at column 0, both items alike: no plain
2122
+ import, star import or re-export (`_python_definition_lines` and
2123
+ `_tsjs_definition_lines` say which), and nothing in a Python file that
2124
+ does not tokenize.
1503
2125
  """
1504
2126
  rel = source_file.strip().replace("\\", "/")
1505
2127
  suffix = Path(rel).suffix.lower()
@@ -1510,18 +2132,39 @@ def _definition_matches(
1510
2132
  lines = cache.lines(path)
1511
2133
  if lines is None:
1512
2134
  return None
2135
+ return _line_matches(suffix, path, lines, export_name, cache, declarations_only)
2136
+
2137
+
2138
+ def _line_matches(
2139
+ suffix: str,
2140
+ path: Path,
2141
+ lines: list[str],
2142
+ export_name: str,
2143
+ cache: _SourceCache,
2144
+ declarations_only: bool = False,
2145
+ ) -> tuple[list[int], set[int]]:
2146
+ """`_definition_matches` over the lines of a Python or TS/JS file already
2147
+ read: `suffix`, the cited file's extension, picks the rules, and `path`
2148
+ keys `cache`."""
1513
2149
  name = definition_name(export_name)
1514
2150
  if not name:
1515
2151
  return [], set()
1516
2152
  dotted = "." in export_name
1517
2153
  if suffix in PYTHON_EXTENSIONS:
1518
2154
  starts = cache.py_statement_starts(path, lines)
1519
- found = _python_definition_lines(lines, name, starts)
2155
+ if declarations_only and starts is None:
2156
+ # every line would be a candidate, a docstring line included
2157
+ return [], set()
2158
+ found = _python_definition_lines(lines, name, starts, declarations_only)
1520
2159
  else:
1521
2160
  code_lines = cache.js_code_lines(path, lines)
1522
2161
  found = _tsjs_definition_lines(
1523
- lines, code_lines, name, dotted, cache.js_declarators
2162
+ lines, code_lines, name, dotted, cache.js_declarators, declarations_only,
2163
+ cache.js_quiet_lines(path, code_lines), cache.js_destructured(path, code_lines),
1524
2164
  )
2165
+ if declarations_only:
2166
+ top = {n for n, at_column_0 in found if at_column_0}
2167
+ return sorted(top), top
1525
2168
  return _prefer_module_level(found, dotted), {n for n, _ in found}
1526
2169
 
1527
2170
 
@@ -1541,6 +2184,39 @@ def find_definition_lines(
1541
2184
  return None if matches is None else matches[0]
1542
2185
 
1543
2186
 
2187
+ def _missed_definition_lines(
2188
+ source_file: str,
2189
+ export_name: str,
2190
+ source_root: Path,
2191
+ cache: _SourceCache,
2192
+ line: int,
2193
+ defs: list[int],
2194
+ every: set[int],
2195
+ ) -> list[int]:
2196
+ """The `definition_lines` a finding lists for a recorded `line` that is
2197
+ not one of `defs` (`_definition_matches` over the same `cache`).
2198
+
2199
+ A recorded line that is an indented match the column-0 rule dropped
2200
+ joins them, so the finding carries several lines and nobody moves it.
2201
+ Otherwise a Python file's `@overload` group gives its first line alone
2202
+ (`_python_overload_start`), the line to move the entry to, and any
2203
+ other answer gives `defs` as they are.
2204
+ """
2205
+ if line in every:
2206
+ return sorted(set(defs) | {line})
2207
+ rel = source_file.strip().replace("\\", "/")
2208
+ if Path(rel).suffix.lower() not in PYTHON_EXTENSIONS or len(defs) < 2:
2209
+ return defs
2210
+ path = (source_root / rel).resolve()
2211
+ lines = cache.lines(path)
2212
+ if lines is None:
2213
+ return defs
2214
+ first = _python_overload_start(
2215
+ lines, definition_name(export_name), defs, cache.py_statement_starts(path, lines)
2216
+ )
2217
+ return defs if first is None else [first]
2218
+
2219
+
1544
2220
  def definition_lines_report(
1545
2221
  source_file: str,
1546
2222
  export_name: str,
@@ -1553,9 +2229,11 @@ def definition_lines_report(
1553
2229
 
1554
2230
  `line_check` is `file-missing`, `skipped-export-type` (a `module` or
1555
2231
  `package`, no definition line), `skipped-language` (no rule for the
1556
- extension) or `checked`. With `line`, a recorded line that is an
1557
- indented match the column-0 rule dropped joins `definition_lines`, as in
1558
- `verify`, so the answer carries several lines.
2232
+ extension) or `checked`. With `line`, a recorded line that is not a
2233
+ definition line gets the lines `verify` would list
2234
+ (`_missed_definition_lines`): an indented match the column-0 rule
2235
+ dropped joins `definition_lines`, so the answer carries several lines,
2236
+ and a Python `@overload` group gives its first line alone.
1559
2237
  """
1560
2238
  report: dict = {
1561
2239
  "source_file": source_file,
@@ -1574,14 +2252,17 @@ def definition_lines_report(
1574
2252
  covered = suffix in PYTHON_EXTENSIONS or suffix in TSJS_EXTENSIONS
1575
2253
  report["line_check"] = "skipped-export-type" if covered else "skipped-language"
1576
2254
  return report
1577
- matches = _definition_matches(source_file, export_name, source_root)
2255
+ cache = _SourceCache()
2256
+ matches = _definition_matches(source_file, export_name, source_root, cache)
1578
2257
  if matches is None:
1579
2258
  raise ValueError(f"failed to read source file {source_root / rel}")
1580
2259
  defs, every = matches
1581
2260
  if line is not None:
1582
2261
  report["line_is_definition"] = line in defs
1583
- if line not in defs and line in every:
1584
- defs = sorted(set(defs) | {line})
2262
+ if line not in defs:
2263
+ defs = _missed_definition_lines(
2264
+ source_file, export_name, source_root, cache, line, defs, every
2265
+ )
1585
2266
  report["definition_lines"] = defs
1586
2267
  return report
1587
2268
 
@@ -1647,14 +2328,657 @@ def _stale_line_check(
1647
2328
  return report["line_check"], report["definition_lines"]
1648
2329
 
1649
2330
 
1650
- def classify_stale(names: list[str], prov: dict, source_root: Path) -> list[dict]:
2331
+ # The folders and files the `defined_at` walk leaves out, judged
2332
+ # case-insensitively on the path below its walk root: generated, built and
2333
+ # installed output, a vendored copy of another package, tests and their
2334
+ # configuration, docs, examples and benchmarks.
2335
+ # Keep identical to EXCLUDED_DIR_NAMES in skf-resolve-authoritative-files.py.
2336
+ EXCLUDED_DIR_NAMES = {
2337
+ "node_modules", "__pycache__", "dist", "build", ".webpack",
2338
+ "target", ".next", ".nuxt", "out", "coverage", ".git",
2339
+ ".venv", "venv", ".tox", ".mypy_cache", ".pytest_cache",
2340
+ ".ruff_cache", ".gradle", ".idea", ".vscode",
2341
+ }
2342
+ # Keep identical to _VENDORED_SEGMENTS in skf-detect-language.py.
2343
+ _VENDORED_SEGMENTS = frozenset(
2344
+ {"vendor", "vendored", "third_party", "third-party", "thirdparty"}
2345
+ )
2346
+ # Keep identical to TEST_FOLDERS in skf-disqualify-candidates.py.
2347
+ TEST_FOLDERS = frozenset({"test", "tests", "__tests__", "spec", "specs"})
2348
+ # Keep identical to TEST_CONFIG_NAMES in skf-disqualify-candidates.py.
2349
+ TEST_CONFIG_NAMES = frozenset({
2350
+ "conftest.py", "pytest.ini", "phpunit.xml", "phpunit.xml.dist", ".rspec",
2351
+ })
2352
+ # Keep identical to TEST_CONFIG_PREFIXES in skf-disqualify-candidates.py.
2353
+ TEST_CONFIG_PREFIXES = (
2354
+ "jest.config.", "vitest.config.", "karma.conf.", "playwright.config.",
2355
+ "cypress.config.",
2356
+ )
2357
+ # Keep identical to TEST_FILE_RE in skf-disqualify-candidates.py.
2358
+ TEST_FILE_RE = re.compile(
2359
+ r"(\.(test|spec)\.[^./]+$)|(_test\.[^./]+$)|(^test_.+\.py$)|(_spec\.rb$)"
2360
+ )
2361
+ # Keep identical to PY_NON_CORE_TOPS in skf-extract-public-api.py.
2362
+ PY_NON_CORE_TOPS = frozenset({"docs", "examples", "benchmarks"})
2363
+ # The walk's own additions: pip's and setuptools' `_vendor`, test helpers
2364
+ # (`testing`, `fixtures`, `__mocks__`), installed packages (`site-packages`,
2365
+ # a virtualenv named `env`, `bower_components`) and the ESM, CommonJS and
2366
+ # UMD builds a JS package commits.
2367
+ LOOKUP_EXTRA_FOLDERS = frozenset({
2368
+ "_vendor", "testing", "fixtures", "__mocks__", "site-packages", "env",
2369
+ "bower_components", "esm", "cjs", "umd",
2370
+ })
2371
+ LOOKUP_SKIPPED_FOLDERS = frozenset(
2372
+ EXCLUDED_DIR_NAMES | _VENDORED_SEGMENTS | TEST_FOLDERS | PY_NON_CORE_TOPS
2373
+ | LOOKUP_EXTRA_FOLDERS
2374
+ )
2375
+ # The test folders the walk alone skips wherever they lie (end-to-end tests,
2376
+ # Jest's `__fixtures__` beside the code): a file the map cites in one is
2377
+ # still read.
2378
+ LOOKUP_WALK_FOLDERS = frozenset({"e2e", "__fixtures__"})
2379
+ # The folders the walk alone skips as the first folder below its walk root
2380
+ # or below a package root it passes through (`_package_root`'s roots):
2381
+ # examples and demos, samples, scripts, playgrounds, benchmarks and
2382
+ # migrations, which hold code that uses or changes the package rather than
2383
+ # its API. Deeper in a package such a folder can be live package code
2384
+ # (cognee's `cognee/modules/migrations/`), so it is walked there; one that
2385
+ # holds a file the map cites is walked too, and that file is still read.
2386
+ LOOKUP_TOP_FOLDERS = frozenset({
2387
+ "example", "demo", "demos", "sample", "samples", "scripts", "playground",
2388
+ "bench", "benchmark", "migrations",
2389
+ })
2390
+ # The kinds of a documented inventory (validate-inventory.py's `kind`) that a
2391
+ # module cannot be: such a name takes no line 1 from a module so named. A
2392
+ # `method` is left out: reconcile-coverage.py never lists a method as stale,
2393
+ # and a module may share a method's name.
2394
+ NO_MODULE_KINDS = frozenset({"function", "hook", "class", "interface", "enum", "type"})
2395
+ # A TS/JS file that is itself the export: its module so named declares the
2396
+ # name whatever its kind.
2397
+ _SELF_EXPORT_RE = re.compile(r"^(?:export\s+default\b|module\.exports\s*=(?!=))", re.M)
2398
+ # A minified bundle: built output, whatever folder holds it.
2399
+ LOOKUP_SKIPPED_SUFFIXES = (".min.js",)
2400
+ # The errors a lookup gives up on: every `defined_at` is then null.
2401
+ _LOOKUP_ERRORS = (OSError, ValueError, RuntimeError)
2402
+
2403
+
2404
+ def _rule_family(path: str) -> str | None:
2405
+ """The definition rules a file's extension picks: `python`, `tsjs` or
2406
+ None."""
2407
+ suffix = posixpath.splitext(path)[1].lower()
2408
+ if suffix in PYTHON_EXTENSIONS:
2409
+ return "python"
2410
+ return "tsjs" if suffix in TSJS_EXTENSIONS else None
2411
+
2412
+
2413
+ def _lookup_skips_folder(segment: str) -> bool:
2414
+ """True when the `defined_at` walk never goes into this folder."""
2415
+ segment = segment.lower()
2416
+ return segment.startswith(".") or segment in LOOKUP_SKIPPED_FOLDERS
2417
+
2418
+
2419
+ def _lookup_skips_file(filename: str) -> bool:
2420
+ """True when the `defined_at` lookup never reads this file: a test file,
2421
+ as skf-disqualify-candidates.py's `_is_test_file` judges its name (a
2422
+ test configuration file included), or a minified bundle."""
2423
+ name = filename.lower()
2424
+ return (
2425
+ name in TEST_CONFIG_NAMES
2426
+ or name.startswith(TEST_CONFIG_PREFIXES)
2427
+ or TEST_FILE_RE.search(name) is not None
2428
+ or name.endswith(LOOKUP_SKIPPED_SUFFIXES)
2429
+ )
2430
+
2431
+
2432
+ def _symlinked(root: Path, parts: tuple[str, ...]) -> bool:
2433
+ """True when a path below `root` goes through a symlink, itself
2434
+ included."""
2435
+ return any(root.joinpath(*parts[:i]).is_symlink() for i in range(1, len(parts) + 1))
2436
+
2437
+
2438
+ def _readable_below(root: Path, parts: tuple[str, ...], real_root: Path | None = None) -> bool:
2439
+ """True when the `defined_at` lookup may read the file at `parts` below
2440
+ `root`: not a file it skips (`_lookup_skips_file`), no symlink on its
2441
+ path, and its real path a file inside the root (`real_root`, the root
2442
+ resolved)."""
2443
+ if _lookup_skips_file(parts[-1]) or _symlinked(root, parts):
2444
+ return False
2445
+ real = root.joinpath(*parts).resolve()
2446
+ return real.is_relative_to(real_root or root.resolve()) and real.is_file()
2447
+
2448
+
2449
+ def _cited_parts(source_file: str) -> tuple[str, ...] | None:
2450
+ """A recorded `source_file` as path segments below the source root, None
2451
+ when it is absolute or climbs out of the root."""
2452
+ path = normalize_path(source_file)
2453
+ if not path or path == "." or posixpath.isabs(path):
2454
+ return None
2455
+ parts = tuple(path.split("/"))
2456
+ return None if parts[0] == ".." or ":" in parts[0] else parts
2457
+
2458
+
2459
+ # The file whose folder is a package, by rule family.
2460
+ _PACKAGE_MARKERS = {"python": "__init__.py", "tsjs": "package.json"}
2461
+ # Type stubs: a declaration file beside the module it types.
2462
+ _STUB_SUFFIXES = {".pyi": "python", ".d.ts": "tsjs", ".d.mts": "tsjs", ".d.cts": "tsjs"}
2463
+
2464
+
2465
+ def _stub_of(path: str) -> tuple[str, str] | None:
2466
+ """(the path without its suffix, family) of a type stub (`NAME.pyi`,
2467
+ `NAME.d.ts`), None for any other file."""
2468
+ lower = path.lower()
2469
+ for suffix, family in _STUB_SUFFIXES.items():
2470
+ if lower.endswith(suffix):
2471
+ return path[: -len(suffix)], family
2472
+ return None
2473
+
2474
+
2475
+ def _without_stubs(hits: list[str]) -> list[str]:
2476
+ """`file:line` items less each stub whose implementation (`NAME.py`
2477
+ beside `NAME.pyi`, a TS/JS `NAME.<ext>` beside `NAME.d.ts`) is an item
2478
+ too: the two are one declaring file, the implementation."""
2479
+ files = [h.rsplit(":", 1)[0] for h in hits]
2480
+ implementations = {
2481
+ (posixpath.splitext(f)[0], _rule_family(f))
2482
+ for f in files
2483
+ if _stub_of(f) is None and _rule_family(f) is not None
2484
+ }
2485
+ return [h for h, f in zip(hits, files) if _stub_of(f) not in implementations]
2486
+
2487
+
2488
+ def _module_name(parts: tuple[str, ...]) -> str | None:
2489
+ """The name a file is the module or package of, which it declares at
2490
+ its line 1: `NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
2491
+ `NAME/index.<ext>`. None for any other file, and for a package file
2492
+ with no folder above it below the source root."""
2493
+ stem, ext = posixpath.splitext(parts[-1])
2494
+ ext = ext.lower()
2495
+ if ext == ".py":
2496
+ package = "__init__"
2497
+ elif ext in TSJS_EXTENSIONS:
2498
+ package = "index"
2499
+ else:
2500
+ return None
2501
+ if stem != package:
2502
+ return stem
2503
+ return parts[-2] if len(parts) > 1 else None
2504
+
2505
+
2506
+ def _exports_itself(parts: tuple[str, ...], text: str) -> bool:
2507
+ """True for a TS/JS file that is itself the export, with a column-0
2508
+ `export default` or `module.exports =`: its module so named declares
2509
+ the name whatever kind the inventory gives it."""
2510
+ return _rule_family(parts[-1]) == "tsjs" and _SELF_EXPORT_RE.search(text) is not None
2511
+
2512
+
2513
+ def declared_line(
2514
+ source_file: str | tuple[str, ...],
2515
+ name: str,
2516
+ source_root: Path | None = None,
2517
+ *,
2518
+ lines: list[str] | None = None,
2519
+ cache: _SourceCache | None = None,
2520
+ module_fallback: bool = True,
2521
+ ) -> int | None:
2522
+ """The line at which one file declares `name`, by classify-stale's
2523
+ `defined_at` rule (see the module docstring): its first line at column
2524
+ 0 that the definition-line rules match with imports left out (a `def`,
2525
+ `class`, assignment, annotation, type alias, TS/JS declaration, or an
2526
+ alias that renames, such as `from X import Y as NAME`; a plain or star
2527
+ import, an `as` that keeps the imported name, a re-export and an
2528
+ import-valued binding never count), else, with `module_fallback`, its
2529
+ line 1 when it is the module or package so named (`_module_name`).
2530
+
2531
+ `source_file` is the file's path below the source root, as recorded
2532
+ (one that is absolute or climbs out of the root gives None) or as its
2533
+ segments. `lines` are its lines when the caller read them already;
2534
+ otherwise the file is read below `source_root` under the lookup's
2535
+ guards (`_readable_below`: no symlink on its path, its real path
2536
+ inside the root, not a file the lookup skips). Pass one `cache` for
2537
+ the names of one source root, and each file is read and parsed once.
2538
+ None when no line declares the name, the name is empty or dotted, the
2539
+ file is not a Python or TS/JS one, or it cannot be read.
2540
+ """
2541
+ parts = _cited_parts(source_file) if isinstance(source_file, str) else tuple(source_file)
2542
+ if not name or "." in name or not parts or _rule_family(parts[-1]) is None:
2543
+ return None
2544
+ cache = cache if cache is not None else _SourceCache()
2545
+ if lines is None:
2546
+ if source_root is None:
2547
+ raise ValueError("declared_line reads the file below source_root when no lines are given")
2548
+ if not _readable_below(source_root, parts):
2549
+ return None
2550
+ lines = cache.lines(source_root.joinpath(*parts))
2551
+ if lines is None:
2552
+ return None
2553
+ path = Path(*parts)
2554
+ top, _ = _line_matches(path.suffix.lower(), path, lines, name, cache, True)
2555
+ if top:
2556
+ return top[0]
2557
+ return 1 if module_fallback and _module_name(parts) == name else None
2558
+
2559
+
2560
+ # declared_line's `cache`, public for a script that loads this one.
2561
+ SourceCache = _SourceCache
2562
+
2563
+
2564
+ def _baseline_path(path: str) -> str:
2565
+ """A cited path as an extraction writes it: forward slashes, no
2566
+ leading `./`."""
2567
+ rel = path.strip().replace("\\", "/")
2568
+ while rel.startswith("./"):
2569
+ rel = rel[2:]
2570
+ return rel
2571
+
2572
+
2573
+ def _declaration_refused(source_root: Path, file: str) -> bool:
2574
+ """True when `declared_line` would not read a file that is there: a
2575
+ path outside the source root, a file the lookup skips (a test file, a
2576
+ minified bundle) or one with a symlink on its path. A file that is gone
2577
+ is not refused: its entries are deleted."""
2578
+ parts = _cited_parts(file)
2579
+ if parts is None:
2580
+ return True
2581
+ path = source_root.joinpath(*parts)
2582
+ return os.path.lexists(path) and not _readable_below(source_root, parts)
2583
+
2584
+
2585
+ def baseline_gaps(
2586
+ provenance: dict,
2587
+ source_root: Path,
2588
+ held: set[tuple[str, str]],
2589
+ files: set[str],
2590
+ listed: set[tuple] | frozenset = frozenset(),
2591
+ *,
2592
+ cache: _SourceCache | None = None,
2593
+ ) -> tuple[list[dict], list[dict]]:
2594
+ """The provenance entries an extraction left out whose cited file still
2595
+ declares them: the recipe runner never reports a module, an alias that
2596
+ renames, a dunder such as `__version__` or another Python or Rust
2597
+ underscore name, so a diff would read each as removed.
2598
+
2599
+ `held` holds the (name, file) of every export the extraction found,
2600
+ `files` the files whose entries are checked (the ones read), and
2601
+ `listed` the (name, file) of the gaps already listed elsewhere (the
2602
+ runner's own extraction gaps), each path with forward slashes. An
2603
+ entry is left out when its name is dotted (a member) and it is no
2604
+ `module` or `package` entry, when its file is not in `files`, when the
2605
+ extraction holds the name or its re-export target (`reexport_map`,
2606
+ else the entry's `reexported_as`) at that file, or when it is listed
2607
+ already. Each name and file is taken once.
2608
+
2609
+ Returns (gaps, unchecked). A gap is {"name", "file", "line", "entry":
2610
+ null}, plus the entry's `export_type` when it has one, at the line
2611
+ `declared_line` gives (only a `module` or `package` entry takes the
2612
+ module rule); an entry whose file declares it no more is no gap.
2613
+ `unchecked` holds {"name", "file", "export_type"} for each entry no
2614
+ rule can check, to read by eye: one whose file is neither Python nor
2615
+ TS/JS, one whose file the lookup will not read (outside the source
2616
+ root, a test file or a minified bundle, a symlink on its path), and a
2617
+ dotted `module` or `package` entry (`api.v1`). The entries are read
2618
+ file by file over one cache, so each file is read and parsed once."""
2619
+ checkable = PYTHON_EXTENSIONS | TSJS_EXTENSIONS
2620
+ renamed = extract_reexport_map(provenance)
2621
+ seen = set(listed)
2622
+ by_file: dict[str, list[tuple[str, object]]] = {}
2623
+ unchecked: list[dict] = []
2624
+ entries = provenance.get("entries")
2625
+ for entry in entries if isinstance(entries, list) else []:
2626
+ if not isinstance(entry, dict):
2627
+ continue
2628
+ name, cited = entry.get("export_name"), entry.get("source_file")
2629
+ if not isinstance(name, str) or not isinstance(cited, str):
2630
+ continue
2631
+ name, file, kind = name.strip(), _baseline_path(cited), entry.get("export_type")
2632
+ module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
2633
+ if not name or ("." in name and not module) or (name, file) in seen:
2634
+ continue
2635
+ if cited.replace("\\", "/") not in files and file not in files:
2636
+ continue
2637
+ if (name, file) in held or (renamed.get(name), file) in held:
2638
+ continue
2639
+ seen.add((name, file))
2640
+ if "." in name or Path(file).suffix.lower() not in checkable or _declaration_refused(source_root, file):
2641
+ unchecked.append({"name": name, "file": file, "export_type": kind})
2642
+ continue
2643
+ by_file.setdefault(file, []).append((name, kind))
2644
+ cache = cache if cache is not None else _SourceCache()
2645
+ gaps = []
2646
+ for file, named in by_file.items():
2647
+ for name, kind in named:
2648
+ module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
2649
+ line = declared_line(file, name, source_root, cache=cache, module_fallback=module)
2650
+ if line is None:
2651
+ continue
2652
+ gap = {"name": name, "file": file, "line": line, "entry": None}
2653
+ if kind is not None:
2654
+ gap["export_type"] = kind
2655
+ gaps.append(gap)
2656
+ return gaps, unchecked
2657
+
2658
+
2659
+ class _DeclarationLookup:
2660
+ """Where the source declares the stale names (classify-stale's
2661
+ `defined_at`): a column-0 declaration, an alias that renames included, a
2662
+ plain or star import and a re-export never counting.
2663
+
2664
+ A module or package named after the name (`_module_name`) declares it,
2665
+ at its line 1 when no line of it does, unless `find` takes the name as
2666
+ `strict` and the file is not a TS/JS one that is itself the export
2667
+ (`_exports_itself`). A file the name's entries cite that declares it
2668
+ gives its first such line. Otherwise the walk lists every file that
2669
+ declares it (`declared_in`), and exactly one must: the walk reads,
2670
+ sorted, the files below the walk root (`_walk_root`), leaving out the
2671
+ folders `_lookup_skips_folder` names, the `LOOKUP_WALK_FOLDERS`, the
2672
+ `LOOKUP_TOP_FOLDERS` right below the walk root and a package root
2673
+ (`_is_package_root`) that hold no cited file, the files
2674
+ `_lookup_skips_file` names and every symlink. Only the rule families
2675
+ (Python, TS/JS) the entries cite are read (both when no entry cites a
2676
+ file). A cited file is read inside the source root only, with no symlink
2677
+ on its path and the same skips but the walk-only ones, judged below the
2678
+ walk root when it lies there. The walk runs once for all the names no
2679
+ cited file settles: each file is read once, and only a file whose text
2680
+ holds a name goes through the rules. Any error of the filesystem raises
2681
+ (`_LOOKUP_ERRORS`)."""
2682
+
2683
+ def __init__(self, entries: list[dict], source_root: Path) -> None:
2684
+ self.root = source_root
2685
+ self.real_root = source_root.resolve()
2686
+ recorded = [f for f in map(_entry_file, entries) if f is not None]
2687
+ # a map that cites no file names no language: both families
2688
+ families = {_rule_family(f) for f in recorded}
2689
+ self.families = families - {None} if recorded else {"python", "tsjs"}
2690
+ self.folder = self._walk_root(recorded)
2691
+ # the folders that hold a cited file: a LOOKUP_TOP_FOLDERS one is walked
2692
+ self._cited_folders = {
2693
+ parts[:i]
2694
+ for parts in filter(None, map(_cited_parts, recorded))
2695
+ for i in range(1, len(parts))
2696
+ }
2697
+ # the cited files read, kept for the walk: a few files at most
2698
+ self._cited: dict[tuple[str, ...], list[str] | None] = {}
2699
+ self._cited_cache = _SourceCache()
2700
+
2701
+ # -- the walk root ------------------------------------------------------
2702
+
2703
+ def _symlinked(self, parts: tuple[str, ...]) -> bool:
2704
+ """True when a path below the source root goes through a symlink,
2705
+ itself included."""
2706
+ return _symlinked(self.root, parts)
2707
+
2708
+ def _real_folder(self, parts: tuple[str, ...]) -> tuple[str, ...]:
2709
+ """The deepest folder of a path below the source root that is one,
2710
+ with no symlink on its path."""
2711
+ while parts and (
2712
+ self._symlinked(parts) or not self.root.joinpath(*parts).is_dir()
2713
+ ):
2714
+ parts = parts[:-1]
2715
+ return parts
2716
+
2717
+ def _holds(self, folder: tuple[str, ...], filename: str) -> bool:
2718
+ return (self.root.joinpath(*folder) / filename).is_file()
2719
+
2720
+ def _package_root(
2721
+ self, folder: tuple[str, ...], family: str
2722
+ ) -> tuple[str, ...] | None:
2723
+ """The root of the package a folder below the source root lies in,
2724
+ None when it lies in none. Python: the nearest folder at or above it
2725
+ that holds `__init__.py`, then up while the parent holds one, never
2726
+ the source root itself (a stray `__init__.py` there would make the
2727
+ whole tree one package). TS/JS: the nearest folder at or above it
2728
+ that holds `package.json`."""
2729
+ marker = _PACKAGE_MARKERS[family]
2730
+ python = family == "python"
2731
+ here = folder
2732
+ while not (self._holds(here, marker) and (here or not python)):
2733
+ if not here:
2734
+ return None
2735
+ here = here[:-1]
2736
+ if python:
2737
+ while len(here) > 1 and self._holds(here[:-1], marker):
2738
+ here = here[:-1]
2739
+ return here
2740
+
2741
+ def _walk_root(self, recorded: list[str]) -> tuple[str, ...]:
2742
+ """The folder the walk reads: the deepest folder the entries' files
2743
+ share, widened to its package root, leaving out an entry outside
2744
+ every package (the source root when no entry lies in one)."""
2745
+ folders: list[tuple[str, ...]] = []
2746
+ widen: set[str] = set()
2747
+ for source_file in recorded:
2748
+ parts = _cited_parts(source_file)
2749
+ family = _rule_family(source_file) if parts else None
2750
+ if family is None:
2751
+ continue
2752
+ if self._package_root(self._real_folder(parts[:-1]), family) is None:
2753
+ continue # outside every package: a script, a docs page
2754
+ folders.append(parts[:-1])
2755
+ widen.add(family)
2756
+ if not folders:
2757
+ return ()
2758
+ shared = self._real_folder(tuple(os.path.commonprefix(folders)))
2759
+ roots = [self._package_root(shared, family) for family in sorted(widen)]
2760
+ return min((r for r in roots if r is not None), key=len, default=shared)
2761
+
2762
+ # -- reading ------------------------------------------------------------
2763
+
2764
+ def _below_walk_root(self, parts: tuple[str, ...]) -> tuple[str, ...]:
2765
+ depth = len(self.folder)
2766
+ return parts[depth:] if parts[:depth] == self.folder else parts
2767
+
2768
+ def _reads_cited(self, parts: tuple[str, ...]) -> bool:
2769
+ """True when the lookup reads a cited file: one of a family the
2770
+ entries cite, in no skipped place, with no symlink on its path and
2771
+ inside the source root."""
2772
+ if _rule_family(parts[-1]) not in self.families:
2773
+ return False
2774
+ below = self._below_walk_root(parts)
2775
+ if any(_lookup_skips_folder(s) for s in below[:-1]):
2776
+ return False
2777
+ return _readable_below(self.root, parts, self.real_root)
2778
+
2779
+ def _cited_lines(self, parts: tuple[str, ...]) -> list[str] | None:
2780
+ if parts not in self._cited:
2781
+ readable = self._reads_cited(parts)
2782
+ path = self.root.joinpath(*parts)
2783
+ self._cited[parts] = _text_lines(_read_source(path)) if readable else None
2784
+ return self._cited[parts]
2785
+
2786
+ def _is_package_root(self, folder: tuple[str, ...], filenames: list[str]) -> bool:
2787
+ """True when a walked folder (its `filenames`) is a package root of a
2788
+ family the walk reads, as `_package_root` finds one: TS/JS, a folder
2789
+ that holds `package.json`; Python, the top of an `__init__.py`
2790
+ chain, never the source root."""
2791
+ if "tsjs" in self.families and "package.json" in filenames:
2792
+ return True
2793
+ return (
2794
+ "python" in self.families and bool(folder) and "__init__.py" in filenames
2795
+ and (len(folder) == 1 or not self._holds(folder[:-1], "__init__.py"))
2796
+ )
2797
+
2798
+ def _tree(self) -> list[tuple[str, ...]]:
2799
+ """The files the walk reads, as segments below the source root,
2800
+ sorted."""
2801
+ top = self.root.joinpath(*self.folder)
2802
+ found: list[tuple[str, ...]] = []
2803
+
2804
+ def fail(exc: OSError) -> None:
2805
+ raise exc
2806
+
2807
+ for dirpath, dirnames, filenames in os.walk(top, onerror=fail):
2808
+ here = Path(dirpath)
2809
+ inner = here.relative_to(top).parts
2810
+ below = self.folder + inner
2811
+ # a LOOKUP_TOP_FOLDERS name is skipped right below the walk root
2812
+ # and a package root, unless it holds a cited file
2813
+ root = not inner or self._is_package_root(below, filenames)
2814
+ dirnames[:] = [
2815
+ d for d in dirnames
2816
+ if not _lookup_skips_folder(d)
2817
+ and d.lower() not in LOOKUP_WALK_FOLDERS
2818
+ and not (
2819
+ root and d.lower() in LOOKUP_TOP_FOLDERS
2820
+ and (*below, d) not in self._cited_folders
2821
+ )
2822
+ and not (here / d).is_symlink()
2823
+ ]
2824
+ for filename in filenames:
2825
+ path = here / filename
2826
+ if (
2827
+ _rule_family(filename) in self.families
2828
+ and not _lookup_skips_file(filename)
2829
+ and not path.is_symlink()
2830
+ and path.is_file()
2831
+ ):
2832
+ found.append((*below, filename))
2833
+ return sorted(found)
2834
+
2835
+ @staticmethod
2836
+ def _first(
2837
+ parts: tuple[str, ...],
2838
+ lines: list[str],
2839
+ name: str,
2840
+ cache: _SourceCache,
2841
+ module_fallback: bool = True,
2842
+ ) -> str | None:
2843
+ """`file:line` of the first line of a file that declares `name`, else,
2844
+ with `module_fallback`, its line 1 when it is the module or package
2845
+ so named (`declared_line`)."""
2846
+ line = declared_line(
2847
+ parts, name, lines=lines, cache=cache, module_fallback=module_fallback
2848
+ )
2849
+ return f"{'/'.join(parts)}:{line}" if line is not None else None
2850
+
2851
+ # -- the answer ---------------------------------------------------------
2852
+
2853
+ def find(
2854
+ self, cited: dict[str, list[str]], strict: frozenset[str] = frozenset()
2855
+ ) -> tuple[dict[str, str], dict[str, list[str]]]:
2856
+ """For the names of `cited` (name -> the source_file values its
2857
+ entries record): name -> `file:line` for each one the source
2858
+ declares, and name -> the sorted `file:line` of every walked file
2859
+ that declares it, for each name the walk looked for (the names no
2860
+ cited file settles). A dotted name is never looked up, and a name of
2861
+ `strict` takes no line 1 from a module or package so named that is
2862
+ not itself the export (`_exports_itself`), in a cited file or in
2863
+ the walk."""
2864
+ found: dict[str, str] = {}
2865
+ if not self.families:
2866
+ return found, {}
2867
+ pending: list[str] = []
2868
+ for name, files in cited.items():
2869
+ if not name or "." in name:
2870
+ continue
2871
+ for parts in dict.fromkeys(filter(None, map(_cited_parts, files))):
2872
+ lines = self._cited_lines(parts)
2873
+ hit = None if lines is None else self._first(
2874
+ parts, lines, name, self._cited_cache,
2875
+ name not in strict or _exports_itself(parts, "\n".join(lines)),
2876
+ )
2877
+ if hit is not None:
2878
+ found[name] = hit
2879
+ break
2880
+ else:
2881
+ pending.append(name)
2882
+ declared, unsure = self._walk(pending, strict) if pending else ({}, set())
2883
+ # exactly one walked file must declare a name, and none it holds
2884
+ # in an untokenizable Python file may
2885
+ found.update({
2886
+ name: h[0] for name, h in declared.items()
2887
+ if len(h) == 1 and name not in unsure
2888
+ })
2889
+ return found, declared
2890
+
2891
+ def _walk(
2892
+ self, names: list[str], strict: frozenset[str] = frozenset()
2893
+ ) -> tuple[dict[str, list[str]], set[str]]:
2894
+ """name -> the sorted `file:line` of each walked file that declares
2895
+ it, at the file's first declaration (a stub beside its
2896
+ implementation left out: `_without_stubs`; a module so named
2897
+ declaring no name of `strict` unless it is the export), and the
2898
+ names a Python file that
2899
+ does not tokenize holds, which it may declare."""
2900
+ hits: dict[str, list[str]] = {name: [] for name in names}
2901
+ unsure: set[str] = set()
2902
+ for parts in self._tree():
2903
+ module = _module_name(parts)
2904
+ lines = self._cited.get(parts)
2905
+ text = (
2906
+ _read_source(self.root.joinpath(*parts))
2907
+ if lines is None
2908
+ else "\n".join(lines) # a cited file, read already
2909
+ )
2910
+ # a strict name's module so named counts when it is the export
2911
+ fallback = module not in strict or _exports_itself(parts, text)
2912
+ # the substring test spares the rules most files and names
2913
+ here = sorted(
2914
+ n for n in names if n in text or (n == module and fallback)
2915
+ )
2916
+ if not here:
2917
+ continue
2918
+ if lines is None:
2919
+ lines = _text_lines(text)
2920
+ cache = _SourceCache() # this file's rules only: nothing kept
2921
+ path = Path(*parts)
2922
+ if path.suffix.lower() in PYTHON_EXTENSIONS and (
2923
+ cache.py_statement_starts(path, lines) is None
2924
+ ):
2925
+ unsure.update(n for n in here if n in text)
2926
+ for name in here:
2927
+ hit = self._first(parts, lines, name, cache, fallback)
2928
+ if hit is not None:
2929
+ hits[name].append(hit)
2930
+ return {name: sorted(_without_stubs(h)) for name, h in hits.items()}, unsure
2931
+
2932
+
2933
+ def inventory_kinds(data: object, path: str) -> dict[str, frozenset[str]]:
2934
+ """name -> the kinds a documented inventory gives it (`--inventory`):
2935
+ validate-inventory.py's `--output` file, `{"exports": [...]}`, or its
2936
+ whole result, the inventory under `inventory`. An item without a string
2937
+ `name` and `kind` is left out, and a kind is kept in lower case. Raises
2938
+ ValueError when no `exports` list is there."""
2939
+ if isinstance(data, dict) and isinstance(data.get("inventory"), dict):
2940
+ data = data["inventory"] # validate-inventory.py's whole result
2941
+ if not isinstance(data, dict) or not isinstance(data.get("exports"), list):
2942
+ raise ValueError(f"--inventory {path} holds no `exports` list")
2943
+ kinds: dict[str, set[str]] = {}
2944
+ for item in data["exports"]:
2945
+ if not isinstance(item, dict):
2946
+ continue
2947
+ name, kind = item.get("name"), item.get("kind")
2948
+ if isinstance(name, str) and isinstance(kind, str):
2949
+ kinds.setdefault(name, set()).add(kind.strip().lower())
2950
+ return {name: frozenset(k) for name, k in kinds.items()}
2951
+
2952
+
2953
+ def _module_entry(entry: dict) -> bool:
2954
+ """True for a `module` or `package` entry."""
2955
+ kind = entry.get("export_type")
2956
+ return isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
2957
+
2958
+
2959
+ def classify_stale(
2960
+ names: list[str],
2961
+ prov: dict,
2962
+ source_root: Path,
2963
+ kinds: dict[str, frozenset[str]] | None = None,
2964
+ ) -> list[dict]:
1651
2965
  """The fabricated-signature test for each stale name: the definition-lines
1652
- rules at every provenance entry whose raw `export_name` is the name. The
1653
- classify-stale section of the module docstring gives each reason, what a
1654
- `no-file` entry means and which entry gives `source`.
2966
+ rules at every provenance entry whose raw `export_name` is the name, and
2967
+ for a name that is not fabricated, where the source declares it
2968
+ (`defined_at`, and `declared_in` when the walk looked for it). The
2969
+ classify-stale section of the module docstring gives each reason, what
2970
+ a `no-file` entry means, which entry gives `source` and how
2971
+ `defined_at` and `declared_in` are found. `kinds` (`inventory_kinds`,
2972
+ None without `--inventory`) keeps a name of a `NO_MODULE_KINDS` kind
2973
+ from a module or package so named, unless an entry of the name is a
2974
+ `module` or `package` one.
1655
2975
  """
1656
2976
  entries = [e for e in prov.get("entries") or [] if isinstance(e, dict)]
1657
2977
  results: list[dict] = []
2978
+ # the names to look up (never a fabricated one): name -> its cited files
2979
+ cited: dict[str, list[str]] = {}
2980
+ # the names a module so named does not declare
2981
+ strict: set[str] = set()
1658
2982
  for name in names:
1659
2983
  mine = [e for e in entries if e.get("export_name") == name]
1660
2984
  checks: list[dict] = []
@@ -1683,15 +3007,33 @@ def classify_stale(names: list[str], prov: dict, source_root: Path) -> list[dict
1683
3007
  else:
1684
3008
  reason = STALE_UNCHECKED
1685
3009
  source = next(filter(None, map(_entry_citation, mine)), None)
3010
+ fabricated = reason == STALE_NOT_DEFINED
3011
+ if not fabricated:
3012
+ cited[name] = [f for f in map(_entry_file, mine) if f is not None]
3013
+ documented = (kinds or {}).get(name, frozenset())
3014
+ if documented & NO_MODULE_KINDS and not any(map(_module_entry, mine)):
3015
+ strict.add(name)
1686
3016
  results.append(
1687
3017
  {
1688
3018
  "name": name,
1689
- "fabricated": reason == STALE_NOT_DEFINED,
3019
+ "fabricated": fabricated,
1690
3020
  "source": source,
3021
+ "defined_at": None,
3022
+ "declared_in": None,
1691
3023
  "reason": reason,
1692
3024
  "entries": checks,
1693
3025
  }
1694
3026
  )
3027
+ try:
3028
+ found, declared = _DeclarationLookup(entries, source_root).find(
3029
+ cited, frozenset(strict)
3030
+ )
3031
+ except _LOOKUP_ERRORS:
3032
+ # a lookup that fails proves nothing: every name stays null
3033
+ found, declared = {}, {}
3034
+ for item in results:
3035
+ item["defined_at"] = found.get(item["name"])
3036
+ item["declared_in"] = declared.get(item["name"])
1695
3037
  return results
1696
3038
 
1697
3039
 
@@ -2381,13 +3723,13 @@ def verify(
2381
3723
  continue
2382
3724
  defs, every = matches
2383
3725
  if line not in defs:
2384
- if line in every:
2385
- # The recorded line is an indented match the
2386
- # column-0 rule dropped: keep it, so the finding
2387
- # lists several lines and nobody auto-moves it.
2388
- defs = sorted(set(defs) | {line})
3726
+ # An indented match the column-0 rule dropped joins
3727
+ # the lines, so nobody auto-moves it; an @overload
3728
+ # group gives its first line alone, the move target.
2389
3729
  item["reason"] = STALE_LINE_NOT_DEFINITION
2390
- item["definition_lines"] = defs
3730
+ item["definition_lines"] = _missed_definition_lines(
3731
+ sf, name, source_root, cache, line, defs, every
3732
+ )
2391
3733
  stale.append(item)
2392
3734
  stale.sort(key=lambda s: (s["export_name"], s.get("source_file") or ""))
2393
3735
 
@@ -2898,8 +4240,12 @@ def _cmd_definition_lines(args: argparse.Namespace) -> int:
2898
4240
 
2899
4241
  def _cmd_classify_stale(args: argparse.Namespace) -> int:
2900
4242
  names_path, prov_path = Path(args.names), Path(args.provenance)
4243
+ inv_path = Path(args.inventory) if args.inventory is not None else None
2901
4244
  root = Path(args.source_root)
2902
- for label, path in (("names", names_path), ("provenance map", prov_path)):
4245
+ inputs = [("names", names_path), ("provenance map", prov_path)]
4246
+ if inv_path is not None:
4247
+ inputs.append(("inventory", inv_path))
4248
+ for label, path in inputs:
2903
4249
  if not path.is_file():
2904
4250
  print(f"error: {label} not found: {path}", file=sys.stderr)
2905
4251
  return 2
@@ -2919,11 +4265,14 @@ def _cmd_classify_stale(args: argparse.Namespace) -> int:
2919
4265
  try:
2920
4266
  names = stale_names(data)
2921
4267
  prov = load_json_object(prov_path, "provenance map")
4268
+ kinds = None if inv_path is None else inventory_kinds(
4269
+ load_json_object(inv_path, "inventory"), args.inventory
4270
+ )
2922
4271
  except ValueError as exc:
2923
4272
  print(f"error: {exc}", file=sys.stderr)
2924
4273
  return 2
2925
4274
  try:
2926
- result = classify_stale(names, prov, root)
4275
+ result = classify_stale(names, prov, root, kinds)
2927
4276
  except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
2928
4277
  return _unexpected("stale-name classification", exc)
2929
4278
  return 0 if _emit(result, args.output) else 2
@@ -3225,8 +4574,8 @@ def _build_parser() -> argparse.ArgumentParser:
3225
4574
  p = sub.add_parser(
3226
4575
  "classify-stale",
3227
4576
  help=(
3228
- "decide which stale names are fabricated signatures: no cited "
3229
- "file defines them"
4577
+ "decide which stale names are fabricated signatures (no cited "
4578
+ "file defines them) and where the source declares the others"
3230
4579
  ),
3231
4580
  )
3232
4581
  p.add_argument(
@@ -3245,6 +4594,17 @@ def _build_parser() -> argparse.ArgumentParser:
3245
4594
  required=True,
3246
4595
  help="source tree root the entries' source_file paths are under",
3247
4596
  )
4597
+ p.add_argument(
4598
+ "--inventory",
4599
+ default=None,
4600
+ help=(
4601
+ "validate-inventory.py's documented inventory: a name it gives a "
4602
+ "function, hook, class, interface, enum or type kind takes no "
4603
+ "line 1 from a module so named, unless a provenance map entry of "
4604
+ "the name is a module or package one or the module is a TS/JS "
4605
+ "file that is itself the export"
4606
+ ),
4607
+ )
3248
4608
  p.add_argument(
3249
4609
  "-o",
3250
4610
  "--output",