bmad-module-skill-forge 3.0.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/docs/_data/pinned.yaml +1 -1
  3. package/docs/examples.md +1 -1
  4. package/docs/troubleshooting.md +8 -0
  5. package/docs/verifying-a-skill.md +7 -3
  6. package/docs/workflows.md +3 -3
  7. package/package.json +1 -1
  8. package/src/knowledge/qmd-registry.md +1 -1
  9. package/src/knowledge/version-paths.md +1 -1
  10. package/src/shared/health-check.md +2 -50
  11. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
  12. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +2 -2
  13. package/src/shared/scripts/skf-atomic-write.py +212 -59
  14. package/src/shared/scripts/skf-build-change-manifest.py +593 -47
  15. package/src/shared/scripts/skf-classify-changed-files.py +78 -0
  16. package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
  17. package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
  18. package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
  19. package/src/shared/scripts/skf-extract-public-api.py +1309 -157
  20. package/src/shared/scripts/skf-extraction-inventory.py +580 -46
  21. package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
  22. package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
  23. package/src/shared/scripts/skf-github-probe.py +47 -8
  24. package/src/shared/scripts/skf-hash-content.py +14 -8
  25. package/src/shared/scripts/skf-load-provenance.py +3 -1
  26. package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
  27. package/src/shared/scripts/skf-preflight.py +15 -1
  28. package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
  29. package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
  30. package/src/shared/scripts/skf-structural-diff.py +133 -18
  31. package/src/shared/scripts/skf-update-active-symlink.py +115 -62
  32. package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
  33. package/src/shared/scripts/skf-verify-provenance-completeness.py +977 -49
  34. package/src/shared/scripts/skf-write-skill-brief.py +41 -8
  35. package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
  36. package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
  37. package/src/skf-audit-skill/references/headless-contract.md +1 -1
  38. package/src/skf-audit-skill/references/health-check.md +2 -0
  39. package/src/skf-audit-skill/references/init.md +2 -1
  40. package/src/skf-audit-skill/references/re-index.md +14 -9
  41. package/src/skf-audit-skill/references/report.md +2 -2
  42. package/src/skf-audit-skill/references/semantic-diff.md +15 -15
  43. package/src/skf-audit-skill/references/structural-diff.md +10 -5
  44. package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
  45. package/src/skf-brief-skill/SKILL.md +1 -1
  46. package/src/skf-brief-skill/assets/scope-templates.md +1 -1
  47. package/src/skf-brief-skill/assets/skill-brief-schema.md +2 -9
  48. package/src/skf-brief-skill/references/analyze-target.md +6 -6
  49. package/src/skf-brief-skill/references/confirm-brief.md +1 -1
  50. package/src/skf-brief-skill/references/scope-definition.md +4 -4
  51. package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
  52. package/src/skf-brief-skill/references/version-resolution.md +5 -9
  53. package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
  54. package/src/skf-create-skill/assets/skill-sections.md +6 -6
  55. package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
  56. package/src/skf-create-skill/references/compile.md +31 -60
  57. package/src/skf-create-skill/references/component-extraction.md +3 -3
  58. package/src/skf-create-skill/references/enrich.md +2 -2
  59. package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
  60. package/src/skf-create-skill/references/extract.md +22 -19
  61. package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
  62. package/src/skf-create-skill/references/extraction-patterns.md +8 -9
  63. package/src/skf-create-skill/references/generate-artifacts.md +1 -1
  64. package/src/skf-create-skill/references/report.md +16 -5
  65. package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
  66. package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
  67. package/src/skf-drop-skill/references/execute.md +5 -1
  68. package/src/skf-drop-skill/references/health-check.md +2 -0
  69. package/src/skf-drop-skill/references/invocation-contract.md +1 -1
  70. package/src/skf-drop-skill/references/report.md +2 -2
  71. package/src/skf-drop-skill/references/select.md +7 -3
  72. package/src/skf-test-skill/references/coherence-check.md +1 -1
  73. package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
  74. package/src/skf-test-skill/references/coverage-check.md +39 -57
  75. package/src/skf-test-skill/references/init.md +3 -5
  76. package/src/skf-test-skill/references/report.md +10 -8
  77. package/src/skf-test-skill/references/scoring-rules.md +6 -5
  78. package/src/skf-test-skill/references/source-access-protocol.md +14 -17
  79. package/src/skf-test-skill/scripts/build-result-context.py +114 -21
  80. package/src/skf-test-skill/scripts/gap-ledger.py +315 -30
  81. package/src/skf-test-skill/scripts/load-coverage-inputs.py +534 -96
  82. package/src/skf-test-skill/scripts/reconcile-coverage.py +276 -2
  83. package/src/skf-test-skill/scripts/score-signatures.py +36 -16
  84. package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
  85. package/src/skf-update-skill/references/detect-changes.md +32 -13
  86. package/src/skf-update-skill/references/gap-driven.md +3 -3
  87. package/src/skf-update-skill/references/init.md +6 -2
  88. package/src/skf-update-skill/references/invocation-contract.md +1 -1
  89. package/src/skf-update-skill/references/merge.md +2 -2
  90. package/src/skf-update-skill/references/re-extract.md +2 -2
  91. package/src/skf-update-skill/references/report.md +3 -1
  92. package/src/skf-update-skill/references/write.md +43 -23
  93. package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
@@ -33,8 +33,9 @@ entries are per-citation, so the two sets measure different surfaces:
33
33
  Four more subcommands act on the same rules: `fix` applies the citation
34
34
  prefix and source-line fixes a `verify` result has one answer for,
35
35
  `definition-lines` gives the lines that define one export,
36
- `classify-stale` decides which stale names are fabricated signatures, and
37
- `kind-at` gives the node kind of the recipe that matches a source line.
36
+ `classify-stale` decides which stale names are fabricated signatures and
37
+ where the source declares the others, and `kind-at` gives the node kind
38
+ of the recipe that matches a source line.
38
39
 
39
40
  An LLM set-diff can silently pass a dropped or orphaned entry, and eyeballing
40
41
  whether a cited line still exists is not something the model can do reliably.
@@ -389,20 +390,26 @@ Subcommands:
389
390
  rule dropped joins `definition_lines`, as in `verify`.
390
391
 
391
392
  classify-stale --names <coverage.json> --provenance <provenance-map.json>
392
- --source-root <path> [-o <out.json>]
393
+ --source-root <path> [--inventory <inventory.json>]
394
+ [-o <out.json>]
393
395
 
394
396
  test-skill's fabricated-signature test, for each stale name (a
395
397
  documented name the enumerated source surface lacks): the
396
398
  definition-lines rules above, run for every provenance entry whose raw
397
399
  `export_name` is the name, at its `source_file` and `source_line`, with
398
400
  its `export_type`. `--names` holds reconcile-coverage.py's result (its
399
- `stale` names) or a JSON array of names.
401
+ `stale` names) or a JSON array of names. `--inventory` holds the
402
+ documented inventory, validate-inventory.py's `--output` file or its
403
+ whole result: the `kind` it gives each name decides the module rule
404
+ below.
400
405
 
401
406
  Emit JSON, one item per name, in the order given (a repeated name once):
402
407
  [
403
408
  {"name": "<as given>",
404
409
  "fabricated": <bool>,
405
410
  "source": "<source_file>:<source_line>" | "<source_file>" | null,
411
+ "defined_at": "<path>:<line>" | null,
412
+ "declared_in": ["<path>:<line>", ...] | null,
406
413
  "reason": "not-defined" | "defined" | "unchecked" | "no-entry",
407
414
  "entries": [{"source_file": "<as recorded>",
408
415
  "source_line": <as recorded>,
@@ -428,6 +435,80 @@ Subcommands:
428
435
  entry with a `source_file` (the file alone when its line is not a
429
436
  number), null when no entry has one.
430
437
 
438
+ `defined_at` is where the source declares a name that is not
439
+ fabricated, so test-skill can tell a documented extra (a name outside
440
+ the surface that the source still declares) from stale documentation;
441
+ a fabricated name is not looked up (null). A declaration is a line at
442
+ column 0 that the definition-line rules match with imports left out:
443
+ a `def`, `class`, assignment, annotation, type alias, TS/JS
444
+ declaration, `exports.NAME =`, or an alias that renames
445
+ (`import X as NAME`, `from X import Y as NAME`,
446
+ `export { Y as NAME }`, `export * as NAME from`), or a
447
+ `module.exports` key with a value or a method. A plain or star import,
448
+ an `as` that keeps the imported name (`from m import NAME as NAME`,
449
+ `export { NAME as NAME }`), `export import NAME =`,
450
+ `export default NAME`, an export-list item without `as`, a
451
+ `module.exports` shorthand key, a binding whose value is an import
452
+ (`require(...)` or `(await) import(...)`, on its line or the next:
453
+ `const NAME = require(...)`, `exports.NAME = require(...)`,
454
+ `{ NAME: require(...) }`), an indented line and any line of a Python
455
+ file that does not tokenize never count. A module or package named
456
+ NAME (`NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
457
+ `NAME/index.<ext>`) declares NAME too: at its line 1 when no line of it
458
+ declares NAME, unless `--inventory` gives NAME a `function`, `hook`,
459
+ `class`, `interface`, `enum` or `type` kind (any of its kinds, when it
460
+ gives several; a `method` kind is not read), no entry of NAME is a
461
+ `module` or `package` one, and the file is not a TS/JS one that is
462
+ itself the export (a column-0 `export default` or `module.exports =`),
463
+ so that a removed function named after its file is no declaration
464
+ (the inventory has no `module` kind). `declared_line` gives this
465
+ answer for one file, the module rule optional, to a script that loads
466
+ this one, and
467
+ `baseline_gaps` applies it to the map's entries an extraction left
468
+ out (skf-extraction-snapshot.py's baseline gaps and
469
+ skf-build-change-manifest.py's `baseline-gaps`, which apply the module
470
+ rule only to a `module` or `package` entry).
471
+
472
+ The files: when a file the name's entries cite declares it,
473
+ `defined_at` is that file's first declaration (the files in entry
474
+ order), and `declared_in` is null. Otherwise `declared_in` lists, as
475
+ sorted `<path>:<line>`, every file of the walk that declares the name,
476
+ at its first declaration (a type stub, `NAME.pyi` or `NAME.d.ts`,
477
+ beside its implementation is one file with it: the implementation),
478
+ and `defined_at` is the one item when the list has exactly one and no
479
+ walked Python file that does not tokenize holds the name (a homonym,
480
+ two files or more, gives null: test-skill decides it by the files the
481
+ skill cites). The walk reads, sorted by path, every file below the
482
+ walk root: the deepest folder that the files of the map's entries
483
+ share, widened to its package root (Python: the nearest folder at or
484
+ above it that holds `__init__.py`, then up while the parent holds
485
+ one, never the source root itself; TS/JS: the nearest folder at or
486
+ above it that holds `package.json`), leaving out the entries whose
487
+ file lies outside every package; the source root when none lies in
488
+ one. Only the rule families (Python, TS/JS) the map's entries cite are
489
+ read (both when no entry cites a file). The walk skips symlinks, hidden folders, test
490
+ files and test configuration files (`conftest.py`, `jest.config.*`),
491
+ `*.min.js`, and the folders generated, built, installed, vendored or
492
+ test output lives in (`node_modules`, `dist`, `build`, `vendor`,
493
+ `_vendor`, `tests`, `testing`, `fixtures`, `__mocks__`,
494
+ `site-packages`, `env`, `bower_components`, `esm`, `cjs`, `umd` and
495
+ the like), `docs`, `examples` and `benchmarks`, each judged on the
496
+ path below the walk root, the end-to-end test folders `e2e` and
497
+ `__fixtures__` wherever they lie, and, as the first folder below the
498
+ walk root or below a package root inside it (a folder that holds
499
+ `package.json`, the top of an `__init__.py` chain), `example`, `demo`,
500
+ `demos`, `sample`, `samples`, `scripts`, `playground`, `bench`,
501
+ `benchmark` and `migrations` unless it holds a file an entry cites
502
+ (deeper in a package such a folder can be package code). A cited file
503
+ is read only inside the source root (a path that is absolute or climbs
504
+ out of it is not read), with no symlink on its path and under the same
505
+ skips, those walk-only ones aside. The path is relative to the source
506
+ root. A dotted name, or a map whose entries cite files
507
+ but no Python or TS/JS one, is not looked up (`defined_at` and
508
+ `declared_in` null, as for a fabricated name), and a filesystem error
509
+ during the lookup gives every name null in both (the exit code stays
510
+ 0).
511
+
431
512
  kind-at --file <path> --line <n> --recipes <recipes>
432
513
  [--language <lang>] [--name <export_name>]
433
514
  [--source-root <path>] [-o <out.json>]
@@ -483,7 +564,8 @@ CLI examples:
483
564
  uv run skf-verify-provenance-completeness.py classify-stale \\
484
565
  --names {run_dir}/coverage.json \\
485
566
  --provenance {forge_version}/provenance-map.json \\
486
- --source-root {source_root} -o {run_dir}/stale.json
567
+ --source-root {source_root} --inventory {run_dir}/inventory.json \\
568
+ -o {run_dir}/stale.json
487
569
  uv run skf-verify-provenance-completeness.py kind-at \\
488
570
  --source-root {source_root} --file src/api.ts --line 12 --name Store \\
489
571
  --recipes {extractionPatternsData}
@@ -507,11 +589,11 @@ Exit codes:
507
589
  2 error, for every subcommand (an input file or folder not found,
508
590
  --skill-dir without a SKILL.md, malformed JSON or YAML, a verify
509
591
  result without stale[] and citations[], --names without a list of
510
- names, no recipe in --recipes, no PyYAML for kind-at, an atomic
511
- write that failed, an -o file that cannot be written, or any
512
- unexpected failure, such as an unreadable
513
- file; once fix has started writing, stderr names the files already
514
- written); one line on stderr and no JSON
592
+ names, --inventory without an `exports` list, no recipe in
593
+ --recipes, no PyYAML for kind-at, an atomic write that failed, an -o
594
+ file that cannot be written, or any unexpected failure, such as an
595
+ unreadable file; once fix has started writing, stderr names the files
596
+ already written); one line on stderr and no JSON
515
597
  """
516
598
 
517
599
  from __future__ import annotations
@@ -828,6 +910,9 @@ _PY_TARGET_LIST_RE = re.compile(
828
910
  )
829
911
  # `def \` or `class \`: the name is on the next line.
830
912
  _PY_SPLIT_DECL_RE = re.compile(r"^\s*(?:async\s+def|def|class)\s*\\$")
913
+ # A line that can matter to a name it does not hold: an import (which may go
914
+ # on over the next lines) or a `def \` / `class \` whose name is on the next.
915
+ _PY_NAMELESS_MATCH_RE = re.compile(r"^\s*(?:from|import|async|def|class)\b")
831
916
  # Tokens that never begin a statement.
832
917
  _PY_LAYOUT_TOKENS = frozenset(
833
918
  {
@@ -861,6 +946,15 @@ _JS_EXPORT_ITEM_RE = re.compile(
861
946
  rf"^(?:type\s+)?({_JS_NAME_OR_STRING})(?:\s+as\s+({_JS_NAME_OR_STRING}))?$"
862
947
  )
863
948
  _JS_MODULE_EXPORTS_OBJECT_RE = re.compile(r"^\s*module\.exports\s*=\s*\{(.*)$")
949
+ # A bound value that is an import: `require(...)`, `import(...)`, awaited, in
950
+ # parentheses (`(await import("./m")).X`) or in one helper call
951
+ # (`__importDefault(require("./m"))`).
952
+ _JS_IMPORT_VALUE_RE = re.compile(
953
+ r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?(?:await\s+)?(?:require|import)\s*\("
954
+ )
955
+ # The start of such a value that goes on on the next line: nothing yet, a
956
+ # parenthesis, or a helper call's name and parenthesis.
957
+ _JS_IMPORT_VALUE_OPEN_RE = re.compile(r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?$")
864
958
  # A `const`, `let` or `var` statement: its declarator list.
865
959
  _JS_VAR_DECL_RE = re.compile(
866
960
  rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(?!enum(?![\w$]))(.*)$"
@@ -870,6 +964,18 @@ _JS_SPLIT_DECL_RE = re.compile(
870
964
  rf"^\s*{_JS_MODIFIERS}(?P<kind>function(?:\s*\*)?|class|interface|type"
871
965
  r"|enum|namespace|let|var|const(?:\s+enum)?)$"
872
966
  )
967
+ # The keywords every line that opens a statement going on below holds (a
968
+ # `const` / `let` / `var` list, a split declaration, `export {`,
969
+ # `module.exports = {`): a line with none of them, and not the name, can
970
+ # match no rule while no statement is open.
971
+ _JS_OPENING_WORDS = ("const", "let", "var", "function", "class", "interface", "type", "enum", "namespace",
972
+ "export", "module")
973
+
974
+
975
+ def _js_quiet_lines(code_lines: list[str]) -> list[bool]:
976
+ """For each comment-free line, True when it holds none of
977
+ `_JS_OPENING_WORDS`."""
978
+ return [not any(word in code for word in _JS_OPENING_WORDS) for code in code_lines]
873
979
  _JS_VAR_KEYWORDS = frozenset({"const", "let", "var"})
874
980
  # How a `const` / `let` / `var` list that did not end on its line goes on
875
981
  # (`_js_declarators`): the next line begins a declarator (this one ended
@@ -909,6 +1015,21 @@ def _read_lines(path: Path) -> list[str] | None:
909
1015
  return None
910
1016
 
911
1017
 
1018
+ def _read_source(path: Path) -> str:
1019
+ """A text file's content, decoded as `_read_lines` decodes it. Raises
1020
+ OSError when the file cannot be read."""
1021
+ with path.open("r", encoding="utf-8-sig", errors="replace") as fh:
1022
+ return fh.read()
1023
+
1024
+
1025
+ def _text_lines(text: str) -> list[str]:
1026
+ """`_read_source`'s text as the lines `_read_lines` gives for the file."""
1027
+ lines = text.split("\n")
1028
+ if lines[-1] == "":
1029
+ lines.pop()
1030
+ return lines
1031
+
1032
+
912
1033
  def _is_column_0(text: str) -> bool:
913
1034
  """True when a line starts at column 0 (no leading whitespace)."""
914
1035
  return not text[:1].isspace()
@@ -992,6 +1113,7 @@ class _SourceCache:
992
1113
  self._lines: dict[Path, list[str] | None] = {}
993
1114
  self._starts: dict[Path, set[int] | None] = {}
994
1115
  self._js_code: dict[Path, list[str]] = {}
1116
+ self._js_quiet: dict[Path, list[bool]] = {}
995
1117
  self._js_decl: dict[tuple[str, str, bool], _DeclSplit] = {}
996
1118
 
997
1119
  def lines(self, path: Path) -> list[str] | None:
@@ -1009,6 +1131,11 @@ class _SourceCache:
1009
1131
  self._js_code[path] = _js_code_lines(lines)
1010
1132
  return self._js_code[path]
1011
1133
 
1134
+ def js_quiet_lines(self, path: Path, code_lines: list[str]) -> list[bool]:
1135
+ if path not in self._js_quiet:
1136
+ self._js_quiet[path] = _js_quiet_lines(code_lines)
1137
+ return self._js_quiet[path]
1138
+
1012
1139
  def js_declarators(
1013
1140
  self, text: str, opened: str = "", fresh: bool = True
1014
1141
  ) -> _DeclSplit:
@@ -1018,8 +1145,12 @@ class _SourceCache:
1018
1145
  return self._js_decl[key]
1019
1146
 
1020
1147
 
1021
- def _py_import_binds(items: str, name: str, from_import: bool) -> bool:
1022
- """True when an import item list binds `name`."""
1148
+ def _py_import_binds(
1149
+ items: str, name: str, from_import: bool, alias_only: bool = False
1150
+ ) -> bool:
1151
+ """True when an import item list binds `name` (with `alias_only`, only
1152
+ through an explicit `as name` that renames: `from m import name as
1153
+ name` and `import a.name as name` keep the imported name)."""
1023
1154
  items = items.split("#", 1)[0]
1024
1155
  for raw in items.replace("(", " ").replace(")", " ").replace("\\", " ").split(","):
1025
1156
  m = _PY_IMPORT_ITEM_RE.match(raw.strip())
@@ -1027,7 +1158,11 @@ def _py_import_binds(items: str, name: str, from_import: bool) -> bool:
1027
1158
  continue
1028
1159
  target, alias = m.group(1), m.group(2)
1029
1160
  if alias is not None:
1161
+ if alias_only and alias == target.rsplit(".", 1)[-1]:
1162
+ continue
1030
1163
  bound = alias
1164
+ elif alias_only:
1165
+ continue
1031
1166
  elif from_import:
1032
1167
  bound = target
1033
1168
  else:
@@ -1050,7 +1185,10 @@ def _py_assigned_names(code: str) -> set[str]:
1050
1185
 
1051
1186
 
1052
1187
  def _python_definition_lines(
1053
- lines: list[str], name: str, starts: set[int] | None
1188
+ lines: list[str],
1189
+ name: str,
1190
+ starts: set[int] | None,
1191
+ declarations_only: bool = False,
1054
1192
  ) -> list[tuple[int, bool]]:
1055
1193
  """(line, at column 0) for every line that defines `name`.
1056
1194
 
@@ -1059,7 +1197,9 @@ def _python_definition_lines(
1059
1197
  line of a `def \\` or `class \\` whose name is on the next line. A
1060
1198
  `from X import *` line counts only when no other line defines `name` (a
1061
1199
  re-export the extraction could not trace further is cited at its star
1062
- import).
1200
+ import). With `declarations_only`, an import counts only when it binds
1201
+ `name` through an explicit `as name` that renames (`_py_import_binds`),
1202
+ and a star import never does.
1063
1203
  """
1064
1204
  esc = re.escape(name)
1065
1205
  decl = re.compile(rf"^\s*(?:async\s+def|def|class)\s+{esc}(?!\w)")
@@ -1085,11 +1225,13 @@ def _python_definition_lines(
1085
1225
  code = stripped[:-1]
1086
1226
  else:
1087
1227
  cont = None
1088
- if _py_import_binds(code, name, from_import):
1228
+ if _py_import_binds(code, name, from_import, declarations_only):
1089
1229
  found.append((number, top))
1090
1230
  continue
1091
1231
  if starts is not None and number not in starts:
1092
1232
  continue
1233
+ if name not in code and not _PY_NAMELESS_MATCH_RE.match(code):
1234
+ continue # no rule below can match it: spares a scan per name
1093
1235
  top = _is_column_0(text)
1094
1236
  if (
1095
1237
  decl.match(code)
@@ -1114,27 +1256,33 @@ def _python_definition_lines(
1114
1256
  continue
1115
1257
  rest = m.group(1)
1116
1258
  if from_import and rest.strip() == "*":
1117
- star_imports.append((number, top))
1259
+ if not declarations_only:
1260
+ star_imports.append((number, top))
1118
1261
  continue
1119
1262
  if from_import and rest.lstrip().startswith("(") and ")" not in rest:
1120
1263
  cont = ("paren", top, True)
1121
1264
  elif rest.rstrip().endswith("\\"):
1122
1265
  cont = ("backslash", top, from_import)
1123
1266
  rest = rest.rstrip()[:-1]
1124
- if _py_import_binds(rest, name, from_import):
1267
+ if _py_import_binds(rest, name, from_import, declarations_only):
1125
1268
  found.append((number, top))
1126
1269
  return found or star_imports
1127
1270
 
1128
1271
 
1129
- def _js_export_item_binds(item: str, name: str) -> bool:
1272
+ def _js_export_item_binds(item: str, name: str, alias_only: bool = False) -> bool:
1130
1273
  """True when one `export { ... }` item exposes `name`: the name after
1131
- `as`, else the item's own name, an identifier or a string."""
1274
+ `as`, else (unless `alias_only`) the item's own name, an identifier or
1275
+ a string. With `alias_only`, an `as` that keeps the item's own name
1276
+ (`NAME as NAME`) does not count either."""
1132
1277
  m = _JS_EXPORT_ITEM_RE.match(item.strip())
1133
- if not m:
1278
+ if not m or (alias_only and m.group(2) is None):
1279
+ return False
1280
+ own, exposed = (
1281
+ s[1:-1] if s[:1] in ("'", '"') else s
1282
+ for s in (m.group(1), m.group(2) or m.group(1))
1283
+ )
1284
+ if alias_only and exposed == own:
1134
1285
  return False
1135
- exposed = m.group(2) or m.group(1)
1136
- if exposed[:1] in ("'", '"'):
1137
- exposed = exposed[1:-1]
1138
1286
  return exposed == name
1139
1287
 
1140
1288
 
@@ -1327,10 +1475,14 @@ def _tsjs_definition_lines(
1327
1475
  name: str,
1328
1476
  dotted: bool,
1329
1477
  declarators: Callable[..., _DeclSplit] = _js_declarators,
1478
+ declarations_only: bool = False,
1479
+ quiet: list[bool] | None = None,
1330
1480
  ) -> list[tuple[int, bool]]:
1331
1481
  """(line, at column 0) for every line that defines `name`.
1332
1482
 
1333
- `code_lines` are `lines` with comments removed (`_js_code_lines`), so a
1483
+ `quiet` is `_js_quiet_lines(code_lines)` when the caller has it (it does
1484
+ not depend on the name): a quiet line without the name is passed over
1485
+ while no statement is open. `code_lines` are `lines` with comments removed (`_js_code_lines`), so a
1334
1486
  JSDoc line or a commented-out declaration never matches. A line inside
1335
1487
  a multi-line `export { ... }` list or `module.exports = { ... }` object
1336
1488
  takes the column of the line the statement starts on, and so do the
@@ -1338,29 +1490,90 @@ def _tsjs_definition_lines(
1338
1490
  later declarator on a line of its own. Class members count only for a
1339
1491
  dotted export name. `declarators` splits a declarator list
1340
1492
  (`_js_declarators`, or `_SourceCache.js_declarators`, which splits each
1341
- line once for every export looked up).
1493
+ line once for every export looked up). With `declarations_only`,
1494
+ `export import NAME =`, `export default NAME`, an export-list item
1495
+ without an `as` that renames, a `module.exports` shorthand key and an
1496
+ import-valued binding (a declarator, `exports.NAME =` or a
1497
+ `module.exports` key whose value starts with `require(` or an
1498
+ `(await) import(`, on its line or the next one) never count, nor does
1499
+ a class member.
1342
1500
  """
1343
1501
  esc = re.escape(name)
1344
1502
  end = r"(?![\w$])"
1345
- statements = (
1503
+ statements = [
1346
1504
  re.compile(rf"^\s*{_JS_MODIFIERS}{_JS_KINDS}{esc}{end}"),
1347
- re.compile(rf"^\s*export\s+import\s+(?:type\s+)?{esc}\s*=(?![=>])"),
1348
1505
  re.compile(
1349
1506
  rf"^\s*export\s+(?:type\s+)?\*\s+as\s+"
1350
1507
  rf"(?:{esc}\s+|([\"']){esc}\1\s*)from{end}"
1351
1508
  ),
1352
- re.compile(rf"^\s*export\s+default\s+{esc}\s*;?\s*$"),
1353
1509
  re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])"),
1354
- )
1510
+ ]
1511
+ if not declarations_only:
1512
+ statements += [
1513
+ re.compile(rf"^\s*export\s+import\s+(?:type\s+)?{esc}\s*=(?![=>])"),
1514
+ re.compile(rf"^\s*export\s+default\s+{esc}\s*;?\s*$"),
1515
+ ]
1355
1516
  # One declarator of a `const` / `let` / `var` list: `NAME = v`,
1356
1517
  # `NAME: T = v`, `NAME!: T`, or a bare `NAME`. A destructuring pattern
1357
1518
  # starts with `{` or `[` and never matches.
1358
1519
  declarator = re.compile(rf"^\s*{esc}{end}\s*(?:!?\s*:|=|$)")
1520
+ # With declarations_only, the value bound to NAME (the rest of a
1521
+ # declarator, of `exports.NAME =` or of a `module.exports` key), to
1522
+ # test for an import.
1523
+ # (A type annotation holds no `=` but in `=>`: `NAME: () => T = v`.)
1524
+ declarator_value = re.compile(
1525
+ rf"^\s*{esc}{end}\s*(?:!?\s*:(?:[^=]|=>)*)?=(?![=>])(.*)$"
1526
+ )
1527
+ var_statement = re.compile(rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(.*)$")
1528
+ exports_value = re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])(.*)$")
1529
+ key_value = re.compile(rf"^\s*(?:(['\"]){esc}\1|{esc}{end})\s*:(.*)$")
1530
+
1531
+ def imported(value: str, number: int) -> bool:
1532
+ """True when a bound value is an import: `require(...)`, or an
1533
+ `import(...)` awaited or not, alone or in one helper call. A value
1534
+ that goes on on the next line (`NAME =`, `NAME = (` or
1535
+ `NAME = helper(` ends line `number`) is read there too."""
1536
+ if _JS_IMPORT_VALUE_OPEN_RE.match(value):
1537
+ value += next((c for c in code_lines[number:] if c.strip()), "")
1538
+ return _JS_IMPORT_VALUE_RE.match(value) is not None
1539
+
1540
+ def declares(piece: str, number: int) -> bool:
1541
+ if declarator.match(piece) is None:
1542
+ return False
1543
+ if not declarations_only:
1544
+ return True
1545
+ m = declarator_value.match(piece)
1546
+ return m is None or not imported(m.group(1), number)
1547
+
1548
+ def is_statement(code: str, number: int) -> bool:
1549
+ if not any(p.match(code) for p in statements):
1550
+ return False
1551
+ if not declarations_only:
1552
+ return True
1553
+ m = var_statement.match(code)
1554
+ if m is not None:
1555
+ m = declarator_value.match(m.group(1))
1556
+ else:
1557
+ m = exports_value.match(code)
1558
+ return m is None or not imported(m.group(1), number)
1559
+
1359
1560
  leading_name = re.compile(rf"^\s*{esc}{end}")
1561
+ # A key with a value, a method, or (unless declarations_only) a shorthand.
1562
+ key_end = r"[:(]" if declarations_only else r"[:(]|$"
1360
1563
  object_key = re.compile(
1361
1564
  rf"^\s*(?:(?:async|get|set)\s+|\*\s*)*(?:(['\"]){esc}\1|{esc}{end})"
1362
- rf"\s*(?:[:(]|$)"
1565
+ rf"\s*(?:{key_end})"
1363
1566
  )
1567
+
1568
+ def binds_key(piece: str, number: int) -> bool:
1569
+ if object_key.match(piece) is None:
1570
+ return False
1571
+ if not declarations_only:
1572
+ return True
1573
+ m = key_value.match(piece)
1574
+ return m is None or not imported(m.group(2), number)
1575
+
1576
+ dotted = dotted and not declarations_only
1364
1577
  member = re.compile(
1365
1578
  rf"^\s+(?P<mods>{_JS_MEMBER_MODIFIERS})(?:\*\s*)?{esc}{end}\s*[?!]?\s*"
1366
1579
  rf"(?:<[^>]*>\s*)?(?P<tail>[(:=;]|$)"
@@ -1375,7 +1588,11 @@ def _tsjs_definition_lines(
1375
1588
  # nested in another's initializer included: (column 0, brackets left
1376
1589
  # open, how, as `_js_declarators` gives them).
1377
1590
  decls: list[tuple[bool, str, str]] = []
1591
+ if quiet is None:
1592
+ quiet = _js_quiet_lines(code_lines)
1378
1593
  for number, (text, code) in enumerate(zip(lines, code_lines), start=1):
1594
+ if quiet[number - 1] and not (decls or split or cont) and name not in code:
1595
+ continue # no rule below can match it: spares a scan per name
1379
1596
  # A line of a list carried from above still goes through the rules
1380
1597
  # below: it may hold a class member or a local of an arrow function.
1381
1598
  if decls and code.strip():
@@ -1387,7 +1604,7 @@ def _tsjs_definition_lines(
1387
1604
  carry, pieces = declarators(code, opened, how == _DECL_NEXT)
1388
1605
  if carry is not None:
1389
1606
  carried.append((decl_top, *carry))
1390
- if any(declarator.match(p) for p in pieces):
1607
+ if any(declares(p, number) for p in pieces):
1391
1608
  # a later declarator on a line of its own, as Prettier
1392
1609
  # writes it: `export const a = 1,` then ` NAME = 2;`
1393
1610
  found.append((number, decl_top))
@@ -1399,7 +1616,7 @@ def _tsjs_definition_lines(
1399
1616
  carry, pieces = declarators(code)
1400
1617
  if carry is not None:
1401
1618
  decls.append((top, *carry))
1402
- hit = any(declarator.match(p) for p in pieces)
1619
+ hit = any(declares(p, number) for p in pieces)
1403
1620
  else:
1404
1621
  hit = leading_name.match(code) is not None
1405
1622
  if hit:
@@ -1411,14 +1628,18 @@ def _tsjs_definition_lines(
1411
1628
  depth, pieces = _js_depth1_pieces(code, depth)
1412
1629
  cont = (kind, top, depth) if depth > 0 else None
1413
1630
  if kind == "list":
1414
- hit = any(_js_export_item_binds(p, name) for p in pieces)
1631
+ hit = any(
1632
+ _js_export_item_binds(p, name, declarations_only) for p in pieces
1633
+ )
1415
1634
  else:
1416
- hit = any(object_key.match(p) for p in pieces)
1635
+ hit = any(binds_key(p, number) for p in pieces)
1417
1636
  if hit:
1418
1637
  found.append((number, top))
1419
1638
  continue
1420
1639
  top = _is_column_0(text)
1421
- if any(p.match(code) for p in statements):
1640
+ # every rule that matches a line needs the name in it
1641
+ named = name in code
1642
+ if named and is_statement(code, number):
1422
1643
  found.append((number, top))
1423
1644
  continue
1424
1645
  m = _JS_VAR_DECL_RE.match(code)
@@ -1426,7 +1647,7 @@ def _tsjs_definition_lines(
1426
1647
  carry, pieces = declarators(m.group(1))
1427
1648
  if carry is not None:
1428
1649
  decls.append((top, *carry))
1429
- if any(declarator.match(p) for p in pieces):
1650
+ if named and any(declares(p, number) for p in pieces):
1430
1651
  # a later declarator: `export const a = 1, NAME = 2;`
1431
1652
  found.append((number, top))
1432
1653
  continue
@@ -1439,7 +1660,7 @@ def _tsjs_definition_lines(
1439
1660
  depth, pieces = _js_depth1_pieces(m.group(1), 1)
1440
1661
  if depth > 0:
1441
1662
  cont = ("list", top, depth)
1442
- if any(_js_export_item_binds(p, name) for p in pieces):
1663
+ if any(_js_export_item_binds(p, name, declarations_only) for p in pieces):
1443
1664
  found.append((number, top))
1444
1665
  continue
1445
1666
  m = _JS_MODULE_EXPORTS_OBJECT_RE.match(code)
@@ -1447,7 +1668,7 @@ def _tsjs_definition_lines(
1447
1668
  depth, pieces = _js_depth1_pieces(m.group(1), 1)
1448
1669
  if depth > 0:
1449
1670
  cont = ("object", top, depth)
1450
- if any(object_key.match(p) for p in pieces):
1671
+ if any(binds_key(p, number) for p in pieces):
1451
1672
  found.append((number, top))
1452
1673
  continue
1453
1674
  if not dotted:
@@ -1494,12 +1715,19 @@ def _definition_matches(
1494
1715
  export_name: str,
1495
1716
  source_root: Path,
1496
1717
  cache: _SourceCache | None = None,
1718
+ declarations_only: bool = False,
1497
1719
  ) -> tuple[list[int], set[int]] | None:
1498
1720
  """(definition lines to report, every matching line) for an export.
1499
1721
 
1500
1722
  The first item applies the column-0 rule; the second keeps the indented
1501
1723
  matches that rule dropped. None when the file's extension has no
1502
1724
  definition rule or the file cannot be read.
1725
+
1726
+ `declarations_only` (classify-stale's `defined_at` lookup) keeps the
1727
+ lines that declare the name at column 0, both items alike: no plain
1728
+ import, star import or re-export (`_python_definition_lines` and
1729
+ `_tsjs_definition_lines` say which), and nothing in a Python file that
1730
+ does not tokenize.
1503
1731
  """
1504
1732
  rel = source_file.strip().replace("\\", "/")
1505
1733
  suffix = Path(rel).suffix.lower()
@@ -1510,18 +1738,39 @@ def _definition_matches(
1510
1738
  lines = cache.lines(path)
1511
1739
  if lines is None:
1512
1740
  return None
1741
+ return _line_matches(suffix, path, lines, export_name, cache, declarations_only)
1742
+
1743
+
1744
+ def _line_matches(
1745
+ suffix: str,
1746
+ path: Path,
1747
+ lines: list[str],
1748
+ export_name: str,
1749
+ cache: _SourceCache,
1750
+ declarations_only: bool = False,
1751
+ ) -> tuple[list[int], set[int]]:
1752
+ """`_definition_matches` over the lines of a Python or TS/JS file already
1753
+ read: `suffix`, the cited file's extension, picks the rules, and `path`
1754
+ keys `cache`."""
1513
1755
  name = definition_name(export_name)
1514
1756
  if not name:
1515
1757
  return [], set()
1516
1758
  dotted = "." in export_name
1517
1759
  if suffix in PYTHON_EXTENSIONS:
1518
1760
  starts = cache.py_statement_starts(path, lines)
1519
- found = _python_definition_lines(lines, name, starts)
1761
+ if declarations_only and starts is None:
1762
+ # every line would be a candidate, a docstring line included
1763
+ return [], set()
1764
+ found = _python_definition_lines(lines, name, starts, declarations_only)
1520
1765
  else:
1521
1766
  code_lines = cache.js_code_lines(path, lines)
1522
1767
  found = _tsjs_definition_lines(
1523
- lines, code_lines, name, dotted, cache.js_declarators
1768
+ lines, code_lines, name, dotted, cache.js_declarators, declarations_only,
1769
+ cache.js_quiet_lines(path, code_lines),
1524
1770
  )
1771
+ if declarations_only:
1772
+ top = {n for n, at_column_0 in found if at_column_0}
1773
+ return sorted(top), top
1525
1774
  return _prefer_module_level(found, dotted), {n for n, _ in found}
1526
1775
 
1527
1776
 
@@ -1647,14 +1896,657 @@ def _stale_line_check(
1647
1896
  return report["line_check"], report["definition_lines"]
1648
1897
 
1649
1898
 
1650
- def classify_stale(names: list[str], prov: dict, source_root: Path) -> list[dict]:
1899
+ # The folders and files the `defined_at` walk leaves out, judged
1900
+ # case-insensitively on the path below its walk root: generated, built and
1901
+ # installed output, a vendored copy of another package, tests and their
1902
+ # configuration, docs, examples and benchmarks.
1903
+ # Keep identical to EXCLUDED_DIR_NAMES in skf-resolve-authoritative-files.py.
1904
+ EXCLUDED_DIR_NAMES = {
1905
+ "node_modules", "__pycache__", "dist", "build", ".webpack",
1906
+ "target", ".next", ".nuxt", "out", "coverage", ".git",
1907
+ ".venv", "venv", ".tox", ".mypy_cache", ".pytest_cache",
1908
+ ".ruff_cache", ".gradle", ".idea", ".vscode",
1909
+ }
1910
+ # Keep identical to _VENDORED_SEGMENTS in skf-detect-language.py.
1911
+ _VENDORED_SEGMENTS = frozenset(
1912
+ {"vendor", "vendored", "third_party", "third-party", "thirdparty"}
1913
+ )
1914
+ # Keep identical to TEST_FOLDERS in skf-disqualify-candidates.py.
1915
+ TEST_FOLDERS = frozenset({"test", "tests", "__tests__", "spec", "specs"})
1916
+ # Keep identical to TEST_CONFIG_NAMES in skf-disqualify-candidates.py.
1917
+ TEST_CONFIG_NAMES = frozenset({
1918
+ "conftest.py", "pytest.ini", "phpunit.xml", "phpunit.xml.dist", ".rspec",
1919
+ })
1920
+ # Keep identical to TEST_CONFIG_PREFIXES in skf-disqualify-candidates.py.
1921
+ TEST_CONFIG_PREFIXES = (
1922
+ "jest.config.", "vitest.config.", "karma.conf.", "playwright.config.",
1923
+ "cypress.config.",
1924
+ )
1925
+ # Keep identical to TEST_FILE_RE in skf-disqualify-candidates.py.
1926
+ TEST_FILE_RE = re.compile(
1927
+ r"(\.(test|spec)\.[^./]+$)|(_test\.[^./]+$)|(^test_.+\.py$)|(_spec\.rb$)"
1928
+ )
1929
+ # Keep identical to PY_NON_CORE_TOPS in skf-extract-public-api.py.
1930
+ PY_NON_CORE_TOPS = frozenset({"docs", "examples", "benchmarks"})
1931
+ # The walk's own additions: pip's and setuptools' `_vendor`, test helpers
1932
+ # (`testing`, `fixtures`, `__mocks__`), installed packages (`site-packages`,
1933
+ # a virtualenv named `env`, `bower_components`) and the ESM, CommonJS and
1934
+ # UMD builds a JS package commits.
1935
+ LOOKUP_EXTRA_FOLDERS = frozenset({
1936
+ "_vendor", "testing", "fixtures", "__mocks__", "site-packages", "env",
1937
+ "bower_components", "esm", "cjs", "umd",
1938
+ })
1939
+ LOOKUP_SKIPPED_FOLDERS = frozenset(
1940
+ EXCLUDED_DIR_NAMES | _VENDORED_SEGMENTS | TEST_FOLDERS | PY_NON_CORE_TOPS
1941
+ | LOOKUP_EXTRA_FOLDERS
1942
+ )
1943
+ # The test folders the walk alone skips wherever they lie (end-to-end tests,
1944
+ # Jest's `__fixtures__` beside the code): a file the map cites in one is
1945
+ # still read.
1946
+ LOOKUP_WALK_FOLDERS = frozenset({"e2e", "__fixtures__"})
1947
+ # The folders the walk alone skips as the first folder below its walk root
1948
+ # or below a package root it passes through (`_package_root`'s roots):
1949
+ # examples and demos, samples, scripts, playgrounds, benchmarks and
1950
+ # migrations, which hold code that uses or changes the package rather than
1951
+ # its API. Deeper in a package such a folder can be live package code
1952
+ # (cognee's `cognee/modules/migrations/`), so it is walked there; one that
1953
+ # holds a file the map cites is walked too, and that file is still read.
1954
+ LOOKUP_TOP_FOLDERS = frozenset({
1955
+ "example", "demo", "demos", "sample", "samples", "scripts", "playground",
1956
+ "bench", "benchmark", "migrations",
1957
+ })
1958
+ # The kinds of a documented inventory (validate-inventory.py's `kind`) that a
1959
+ # module cannot be: such a name takes no line 1 from a module so named. A
1960
+ # `method` is left out: reconcile-coverage.py never lists a method as stale,
1961
+ # and a module may share a method's name.
1962
+ NO_MODULE_KINDS = frozenset({"function", "hook", "class", "interface", "enum", "type"})
1963
+ # A TS/JS file that is itself the export: its module so named declares the
1964
+ # name whatever its kind.
1965
+ _SELF_EXPORT_RE = re.compile(r"^(?:export\s+default\b|module\.exports\s*=(?!=))", re.M)
1966
+ # A minified bundle: built output, whatever folder holds it.
1967
+ LOOKUP_SKIPPED_SUFFIXES = (".min.js",)
1968
+ # The errors a lookup gives up on: every `defined_at` is then null.
1969
+ _LOOKUP_ERRORS = (OSError, ValueError, RuntimeError)
1970
+
1971
+
1972
+ def _rule_family(path: str) -> str | None:
1973
+ """The definition rules a file's extension picks: `python`, `tsjs` or
1974
+ None."""
1975
+ suffix = posixpath.splitext(path)[1].lower()
1976
+ if suffix in PYTHON_EXTENSIONS:
1977
+ return "python"
1978
+ return "tsjs" if suffix in TSJS_EXTENSIONS else None
1979
+
1980
+
1981
+ def _lookup_skips_folder(segment: str) -> bool:
1982
+ """True when the `defined_at` walk never goes into this folder."""
1983
+ segment = segment.lower()
1984
+ return segment.startswith(".") or segment in LOOKUP_SKIPPED_FOLDERS
1985
+
1986
+
1987
+ def _lookup_skips_file(filename: str) -> bool:
1988
+ """True when the `defined_at` lookup never reads this file: a test file,
1989
+ as skf-disqualify-candidates.py's `_is_test_file` judges its name (a
1990
+ test configuration file included), or a minified bundle."""
1991
+ name = filename.lower()
1992
+ return (
1993
+ name in TEST_CONFIG_NAMES
1994
+ or name.startswith(TEST_CONFIG_PREFIXES)
1995
+ or TEST_FILE_RE.search(name) is not None
1996
+ or name.endswith(LOOKUP_SKIPPED_SUFFIXES)
1997
+ )
1998
+
1999
+
2000
+ def _symlinked(root: Path, parts: tuple[str, ...]) -> bool:
2001
+ """True when a path below `root` goes through a symlink, itself
2002
+ included."""
2003
+ return any(root.joinpath(*parts[:i]).is_symlink() for i in range(1, len(parts) + 1))
2004
+
2005
+
2006
+ def _readable_below(root: Path, parts: tuple[str, ...], real_root: Path | None = None) -> bool:
2007
+ """True when the `defined_at` lookup may read the file at `parts` below
2008
+ `root`: not a file it skips (`_lookup_skips_file`), no symlink on its
2009
+ path, and its real path a file inside the root (`real_root`, the root
2010
+ resolved)."""
2011
+ if _lookup_skips_file(parts[-1]) or _symlinked(root, parts):
2012
+ return False
2013
+ real = root.joinpath(*parts).resolve()
2014
+ return real.is_relative_to(real_root or root.resolve()) and real.is_file()
2015
+
2016
+
2017
+ def _cited_parts(source_file: str) -> tuple[str, ...] | None:
2018
+ """A recorded `source_file` as path segments below the source root, None
2019
+ when it is absolute or climbs out of the root."""
2020
+ path = normalize_path(source_file)
2021
+ if not path or path == "." or posixpath.isabs(path):
2022
+ return None
2023
+ parts = tuple(path.split("/"))
2024
+ return None if parts[0] == ".." or ":" in parts[0] else parts
2025
+
2026
+
2027
+ # The file whose folder is a package, by rule family.
2028
+ _PACKAGE_MARKERS = {"python": "__init__.py", "tsjs": "package.json"}
2029
+ # Type stubs: a declaration file beside the module it types.
2030
+ _STUB_SUFFIXES = {".pyi": "python", ".d.ts": "tsjs", ".d.mts": "tsjs", ".d.cts": "tsjs"}
2031
+
2032
+
2033
+ def _stub_of(path: str) -> tuple[str, str] | None:
2034
+ """(the path without its suffix, family) of a type stub (`NAME.pyi`,
2035
+ `NAME.d.ts`), None for any other file."""
2036
+ lower = path.lower()
2037
+ for suffix, family in _STUB_SUFFIXES.items():
2038
+ if lower.endswith(suffix):
2039
+ return path[: -len(suffix)], family
2040
+ return None
2041
+
2042
+
2043
+ def _without_stubs(hits: list[str]) -> list[str]:
2044
+ """`file:line` items less each stub whose implementation (`NAME.py`
2045
+ beside `NAME.pyi`, a TS/JS `NAME.<ext>` beside `NAME.d.ts`) is an item
2046
+ too: the two are one declaring file, the implementation."""
2047
+ files = [h.rsplit(":", 1)[0] for h in hits]
2048
+ implementations = {
2049
+ (posixpath.splitext(f)[0], _rule_family(f))
2050
+ for f in files
2051
+ if _stub_of(f) is None and _rule_family(f) is not None
2052
+ }
2053
+ return [h for h, f in zip(hits, files) if _stub_of(f) not in implementations]
2054
+
2055
+
2056
+ def _module_name(parts: tuple[str, ...]) -> str | None:
2057
+ """The name a file is the module or package of, which it declares at
2058
+ its line 1: `NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
2059
+ `NAME/index.<ext>`. None for any other file, and for a package file
2060
+ with no folder above it below the source root."""
2061
+ stem, ext = posixpath.splitext(parts[-1])
2062
+ ext = ext.lower()
2063
+ if ext == ".py":
2064
+ package = "__init__"
2065
+ elif ext in TSJS_EXTENSIONS:
2066
+ package = "index"
2067
+ else:
2068
+ return None
2069
+ if stem != package:
2070
+ return stem
2071
+ return parts[-2] if len(parts) > 1 else None
2072
+
2073
+
2074
+ def _exports_itself(parts: tuple[str, ...], text: str) -> bool:
2075
+ """True for a TS/JS file that is itself the export, with a column-0
2076
+ `export default` or `module.exports =`: its module so named declares
2077
+ the name whatever kind the inventory gives it."""
2078
+ return _rule_family(parts[-1]) == "tsjs" and _SELF_EXPORT_RE.search(text) is not None
2079
+
2080
+
2081
+ def declared_line(
2082
+ source_file: str | tuple[str, ...],
2083
+ name: str,
2084
+ source_root: Path | None = None,
2085
+ *,
2086
+ lines: list[str] | None = None,
2087
+ cache: _SourceCache | None = None,
2088
+ module_fallback: bool = True,
2089
+ ) -> int | None:
2090
+ """The line at which one file declares `name`, by classify-stale's
2091
+ `defined_at` rule (see the module docstring): its first line at column
2092
+ 0 that the definition-line rules match with imports left out (a `def`,
2093
+ `class`, assignment, annotation, type alias, TS/JS declaration, or an
2094
+ alias that renames, such as `from X import Y as NAME`; a plain or star
2095
+ import, an `as` that keeps the imported name, a re-export and an
2096
+ import-valued binding never count), else, with `module_fallback`, its
2097
+ line 1 when it is the module or package so named (`_module_name`).
2098
+
2099
+ `source_file` is the file's path below the source root, as recorded
2100
+ (one that is absolute or climbs out of the root gives None) or as its
2101
+ segments. `lines` are its lines when the caller read them already;
2102
+ otherwise the file is read below `source_root` under the lookup's
2103
+ guards (`_readable_below`: no symlink on its path, its real path
2104
+ inside the root, not a file the lookup skips). Pass one `cache` for
2105
+ the names of one source root, and each file is read and parsed once.
2106
+ None when no line declares the name, the name is empty or dotted, the
2107
+ file is not a Python or TS/JS one, or it cannot be read.
2108
+ """
2109
+ parts = _cited_parts(source_file) if isinstance(source_file, str) else tuple(source_file)
2110
+ if not name or "." in name or not parts or _rule_family(parts[-1]) is None:
2111
+ return None
2112
+ cache = cache if cache is not None else _SourceCache()
2113
+ if lines is None:
2114
+ if source_root is None:
2115
+ raise ValueError("declared_line reads the file below source_root when no lines are given")
2116
+ if not _readable_below(source_root, parts):
2117
+ return None
2118
+ lines = cache.lines(source_root.joinpath(*parts))
2119
+ if lines is None:
2120
+ return None
2121
+ path = Path(*parts)
2122
+ top, _ = _line_matches(path.suffix.lower(), path, lines, name, cache, True)
2123
+ if top:
2124
+ return top[0]
2125
+ return 1 if module_fallback and _module_name(parts) == name else None
2126
+
2127
+
2128
+ # declared_line's `cache`, public for a script that loads this one.
2129
+ SourceCache = _SourceCache
2130
+
2131
+
2132
+ def _baseline_path(path: str) -> str:
2133
+ """A cited path as an extraction writes it: forward slashes, no
2134
+ leading `./`."""
2135
+ rel = path.strip().replace("\\", "/")
2136
+ while rel.startswith("./"):
2137
+ rel = rel[2:]
2138
+ return rel
2139
+
2140
+
2141
+ def _declaration_refused(source_root: Path, file: str) -> bool:
2142
+ """True when `declared_line` would not read a file that is there: a
2143
+ path outside the source root, a file the lookup skips (a test file, a
2144
+ minified bundle) or one with a symlink on its path. A file that is gone
2145
+ is not refused: its entries are deleted."""
2146
+ parts = _cited_parts(file)
2147
+ if parts is None:
2148
+ return True
2149
+ path = source_root.joinpath(*parts)
2150
+ return os.path.lexists(path) and not _readable_below(source_root, parts)
2151
+
2152
+
2153
+ def baseline_gaps(
2154
+ provenance: dict,
2155
+ source_root: Path,
2156
+ held: set[tuple[str, str]],
2157
+ files: set[str],
2158
+ listed: set[tuple] | frozenset = frozenset(),
2159
+ *,
2160
+ cache: _SourceCache | None = None,
2161
+ ) -> tuple[list[dict], list[dict]]:
2162
+ """The provenance entries an extraction left out whose cited file still
2163
+ declares them: the recipe runner never reports a module, an alias that
2164
+ renames, a dunder such as `__version__` or another underscore name, so
2165
+ a diff would read each as removed.
2166
+
2167
+ `held` holds the (name, file) of every export the extraction found,
2168
+ `files` the files whose entries are checked (the ones read), and
2169
+ `listed` the (name, file) of the gaps already listed elsewhere (the
2170
+ runner's own extraction gaps), each path with forward slashes. An
2171
+ entry is left out when its name is dotted (a member) and it is no
2172
+ `module` or `package` entry, when its file is not in `files`, when the
2173
+ extraction holds the name or its re-export target (`reexport_map`,
2174
+ else the entry's `reexported_as`) at that file, or when it is listed
2175
+ already. Each name and file is taken once.
2176
+
2177
+ Returns (gaps, unchecked). A gap is {"name", "file", "line", "entry":
2178
+ null}, plus the entry's `export_type` when it has one, at the line
2179
+ `declared_line` gives (only a `module` or `package` entry takes the
2180
+ module rule); an entry whose file declares it no more is no gap.
2181
+ `unchecked` holds {"name", "file", "export_type"} for each entry no
2182
+ rule can check, to read by eye: one whose file is neither Python nor
2183
+ TS/JS, one whose file the lookup will not read (outside the source
2184
+ root, a test file or a minified bundle, a symlink on its path), and a
2185
+ dotted `module` or `package` entry (`api.v1`). The entries are read
2186
+ file by file over one cache, so each file is read and parsed once."""
2187
+ checkable = PYTHON_EXTENSIONS | TSJS_EXTENSIONS
2188
+ renamed = extract_reexport_map(provenance)
2189
+ seen = set(listed)
2190
+ by_file: dict[str, list[tuple[str, object]]] = {}
2191
+ unchecked: list[dict] = []
2192
+ entries = provenance.get("entries")
2193
+ for entry in entries if isinstance(entries, list) else []:
2194
+ if not isinstance(entry, dict):
2195
+ continue
2196
+ name, cited = entry.get("export_name"), entry.get("source_file")
2197
+ if not isinstance(name, str) or not isinstance(cited, str):
2198
+ continue
2199
+ name, file, kind = name.strip(), _baseline_path(cited), entry.get("export_type")
2200
+ module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
2201
+ if not name or ("." in name and not module) or (name, file) in seen:
2202
+ continue
2203
+ if cited.replace("\\", "/") not in files and file not in files:
2204
+ continue
2205
+ if (name, file) in held or (renamed.get(name), file) in held:
2206
+ continue
2207
+ seen.add((name, file))
2208
+ if "." in name or Path(file).suffix.lower() not in checkable or _declaration_refused(source_root, file):
2209
+ unchecked.append({"name": name, "file": file, "export_type": kind})
2210
+ continue
2211
+ by_file.setdefault(file, []).append((name, kind))
2212
+ cache = cache if cache is not None else _SourceCache()
2213
+ gaps = []
2214
+ for file, named in by_file.items():
2215
+ for name, kind in named:
2216
+ module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
2217
+ line = declared_line(file, name, source_root, cache=cache, module_fallback=module)
2218
+ if line is None:
2219
+ continue
2220
+ gap = {"name": name, "file": file, "line": line, "entry": None}
2221
+ if kind is not None:
2222
+ gap["export_type"] = kind
2223
+ gaps.append(gap)
2224
+ return gaps, unchecked
2225
+
2226
+
2227
+ class _DeclarationLookup:
2228
+ """Where the source declares the stale names (classify-stale's
2229
+ `defined_at`): a column-0 declaration, an alias that renames included, a
2230
+ plain or star import and a re-export never counting.
2231
+
2232
+ A module or package named after the name (`_module_name`) declares it,
2233
+ at its line 1 when no line of it does, unless `find` takes the name as
2234
+ `strict` and the file is not a TS/JS one that is itself the export
2235
+ (`_exports_itself`). A file the name's entries cite that declares it
2236
+ gives its first such line. Otherwise the walk lists every file that
2237
+ declares it (`declared_in`), and exactly one must: the walk reads,
2238
+ sorted, the files below the walk root (`_walk_root`), leaving out the
2239
+ folders `_lookup_skips_folder` names, the `LOOKUP_WALK_FOLDERS`, the
2240
+ `LOOKUP_TOP_FOLDERS` right below the walk root and a package root
2241
+ (`_is_package_root`) that hold no cited file, the files
2242
+ `_lookup_skips_file` names and every symlink. Only the rule families
2243
+ (Python, TS/JS) the entries cite are read (both when no entry cites a
2244
+ file). A cited file is read inside the source root only, with no symlink
2245
+ on its path and the same skips but the walk-only ones, judged below the
2246
+ walk root when it lies there. The walk runs once for all the names no
2247
+ cited file settles: each file is read once, and only a file whose text
2248
+ holds a name goes through the rules. Any error of the filesystem raises
2249
+ (`_LOOKUP_ERRORS`)."""
2250
+
2251
+ def __init__(self, entries: list[dict], source_root: Path) -> None:
2252
+ self.root = source_root
2253
+ self.real_root = source_root.resolve()
2254
+ recorded = [f for f in map(_entry_file, entries) if f is not None]
2255
+ # a map that cites no file names no language: both families
2256
+ families = {_rule_family(f) for f in recorded}
2257
+ self.families = families - {None} if recorded else {"python", "tsjs"}
2258
+ self.folder = self._walk_root(recorded)
2259
+ # the folders that hold a cited file: a LOOKUP_TOP_FOLDERS one is walked
2260
+ self._cited_folders = {
2261
+ parts[:i]
2262
+ for parts in filter(None, map(_cited_parts, recorded))
2263
+ for i in range(1, len(parts))
2264
+ }
2265
+ # the cited files read, kept for the walk: a few files at most
2266
+ self._cited: dict[tuple[str, ...], list[str] | None] = {}
2267
+ self._cited_cache = _SourceCache()
2268
+
2269
+ # -- the walk root ------------------------------------------------------
2270
+
2271
+ def _symlinked(self, parts: tuple[str, ...]) -> bool:
2272
+ """True when a path below the source root goes through a symlink,
2273
+ itself included."""
2274
+ return _symlinked(self.root, parts)
2275
+
2276
+ def _real_folder(self, parts: tuple[str, ...]) -> tuple[str, ...]:
2277
+ """The deepest folder of a path below the source root that is one,
2278
+ with no symlink on its path."""
2279
+ while parts and (
2280
+ self._symlinked(parts) or not self.root.joinpath(*parts).is_dir()
2281
+ ):
2282
+ parts = parts[:-1]
2283
+ return parts
2284
+
2285
+ def _holds(self, folder: tuple[str, ...], filename: str) -> bool:
2286
+ return (self.root.joinpath(*folder) / filename).is_file()
2287
+
2288
+ def _package_root(
2289
+ self, folder: tuple[str, ...], family: str
2290
+ ) -> tuple[str, ...] | None:
2291
+ """The root of the package a folder below the source root lies in,
2292
+ None when it lies in none. Python: the nearest folder at or above it
2293
+ that holds `__init__.py`, then up while the parent holds one, never
2294
+ the source root itself (a stray `__init__.py` there would make the
2295
+ whole tree one package). TS/JS: the nearest folder at or above it
2296
+ that holds `package.json`."""
2297
+ marker = _PACKAGE_MARKERS[family]
2298
+ python = family == "python"
2299
+ here = folder
2300
+ while not (self._holds(here, marker) and (here or not python)):
2301
+ if not here:
2302
+ return None
2303
+ here = here[:-1]
2304
+ if python:
2305
+ while len(here) > 1 and self._holds(here[:-1], marker):
2306
+ here = here[:-1]
2307
+ return here
2308
+
2309
+ def _walk_root(self, recorded: list[str]) -> tuple[str, ...]:
2310
+ """The folder the walk reads: the deepest folder the entries' files
2311
+ share, widened to its package root, leaving out an entry outside
2312
+ every package (the source root when no entry lies in one)."""
2313
+ folders: list[tuple[str, ...]] = []
2314
+ widen: set[str] = set()
2315
+ for source_file in recorded:
2316
+ parts = _cited_parts(source_file)
2317
+ family = _rule_family(source_file) if parts else None
2318
+ if family is None:
2319
+ continue
2320
+ if self._package_root(self._real_folder(parts[:-1]), family) is None:
2321
+ continue # outside every package: a script, a docs page
2322
+ folders.append(parts[:-1])
2323
+ widen.add(family)
2324
+ if not folders:
2325
+ return ()
2326
+ shared = self._real_folder(tuple(os.path.commonprefix(folders)))
2327
+ roots = [self._package_root(shared, family) for family in sorted(widen)]
2328
+ return min((r for r in roots if r is not None), key=len, default=shared)
2329
+
2330
+ # -- reading ------------------------------------------------------------
2331
+
2332
+ def _below_walk_root(self, parts: tuple[str, ...]) -> tuple[str, ...]:
2333
+ depth = len(self.folder)
2334
+ return parts[depth:] if parts[:depth] == self.folder else parts
2335
+
2336
+ def _reads_cited(self, parts: tuple[str, ...]) -> bool:
2337
+ """True when the lookup reads a cited file: one of a family the
2338
+ entries cite, in no skipped place, with no symlink on its path and
2339
+ inside the source root."""
2340
+ if _rule_family(parts[-1]) not in self.families:
2341
+ return False
2342
+ below = self._below_walk_root(parts)
2343
+ if any(_lookup_skips_folder(s) for s in below[:-1]):
2344
+ return False
2345
+ return _readable_below(self.root, parts, self.real_root)
2346
+
2347
+ def _cited_lines(self, parts: tuple[str, ...]) -> list[str] | None:
2348
+ if parts not in self._cited:
2349
+ readable = self._reads_cited(parts)
2350
+ path = self.root.joinpath(*parts)
2351
+ self._cited[parts] = _text_lines(_read_source(path)) if readable else None
2352
+ return self._cited[parts]
2353
+
2354
+ def _is_package_root(self, folder: tuple[str, ...], filenames: list[str]) -> bool:
2355
+ """True when a walked folder (its `filenames`) is a package root of a
2356
+ family the walk reads, as `_package_root` finds one: TS/JS, a folder
2357
+ that holds `package.json`; Python, the top of an `__init__.py`
2358
+ chain, never the source root."""
2359
+ if "tsjs" in self.families and "package.json" in filenames:
2360
+ return True
2361
+ return (
2362
+ "python" in self.families and bool(folder) and "__init__.py" in filenames
2363
+ and (len(folder) == 1 or not self._holds(folder[:-1], "__init__.py"))
2364
+ )
2365
+
2366
+ def _tree(self) -> list[tuple[str, ...]]:
2367
+ """The files the walk reads, as segments below the source root,
2368
+ sorted."""
2369
+ top = self.root.joinpath(*self.folder)
2370
+ found: list[tuple[str, ...]] = []
2371
+
2372
+ def fail(exc: OSError) -> None:
2373
+ raise exc
2374
+
2375
+ for dirpath, dirnames, filenames in os.walk(top, onerror=fail):
2376
+ here = Path(dirpath)
2377
+ inner = here.relative_to(top).parts
2378
+ below = self.folder + inner
2379
+ # a LOOKUP_TOP_FOLDERS name is skipped right below the walk root
2380
+ # and a package root, unless it holds a cited file
2381
+ root = not inner or self._is_package_root(below, filenames)
2382
+ dirnames[:] = [
2383
+ d for d in dirnames
2384
+ if not _lookup_skips_folder(d)
2385
+ and d.lower() not in LOOKUP_WALK_FOLDERS
2386
+ and not (
2387
+ root and d.lower() in LOOKUP_TOP_FOLDERS
2388
+ and (*below, d) not in self._cited_folders
2389
+ )
2390
+ and not (here / d).is_symlink()
2391
+ ]
2392
+ for filename in filenames:
2393
+ path = here / filename
2394
+ if (
2395
+ _rule_family(filename) in self.families
2396
+ and not _lookup_skips_file(filename)
2397
+ and not path.is_symlink()
2398
+ and path.is_file()
2399
+ ):
2400
+ found.append((*below, filename))
2401
+ return sorted(found)
2402
+
2403
+ @staticmethod
2404
+ def _first(
2405
+ parts: tuple[str, ...],
2406
+ lines: list[str],
2407
+ name: str,
2408
+ cache: _SourceCache,
2409
+ module_fallback: bool = True,
2410
+ ) -> str | None:
2411
+ """`file:line` of the first line of a file that declares `name`, else,
2412
+ with `module_fallback`, its line 1 when it is the module or package
2413
+ so named (`declared_line`)."""
2414
+ line = declared_line(
2415
+ parts, name, lines=lines, cache=cache, module_fallback=module_fallback
2416
+ )
2417
+ return f"{'/'.join(parts)}:{line}" if line is not None else None
2418
+
2419
+ # -- the answer ---------------------------------------------------------
2420
+
2421
+ def find(
2422
+ self, cited: dict[str, list[str]], strict: frozenset[str] = frozenset()
2423
+ ) -> tuple[dict[str, str], dict[str, list[str]]]:
2424
+ """For the names of `cited` (name -> the source_file values its
2425
+ entries record): name -> `file:line` for each one the source
2426
+ declares, and name -> the sorted `file:line` of every walked file
2427
+ that declares it, for each name the walk looked for (the names no
2428
+ cited file settles). A dotted name is never looked up, and a name of
2429
+ `strict` takes no line 1 from a module or package so named that is
2430
+ not itself the export (`_exports_itself`), in a cited file or in
2431
+ the walk."""
2432
+ found: dict[str, str] = {}
2433
+ if not self.families:
2434
+ return found, {}
2435
+ pending: list[str] = []
2436
+ for name, files in cited.items():
2437
+ if not name or "." in name:
2438
+ continue
2439
+ for parts in dict.fromkeys(filter(None, map(_cited_parts, files))):
2440
+ lines = self._cited_lines(parts)
2441
+ hit = None if lines is None else self._first(
2442
+ parts, lines, name, self._cited_cache,
2443
+ name not in strict or _exports_itself(parts, "\n".join(lines)),
2444
+ )
2445
+ if hit is not None:
2446
+ found[name] = hit
2447
+ break
2448
+ else:
2449
+ pending.append(name)
2450
+ declared, unsure = self._walk(pending, strict) if pending else ({}, set())
2451
+ # exactly one walked file must declare a name, and none it holds
2452
+ # in an untokenizable Python file may
2453
+ found.update({
2454
+ name: h[0] for name, h in declared.items()
2455
+ if len(h) == 1 and name not in unsure
2456
+ })
2457
+ return found, declared
2458
+
2459
+ def _walk(
2460
+ self, names: list[str], strict: frozenset[str] = frozenset()
2461
+ ) -> tuple[dict[str, list[str]], set[str]]:
2462
+ """name -> the sorted `file:line` of each walked file that declares
2463
+ it, at the file's first declaration (a stub beside its
2464
+ implementation left out: `_without_stubs`; a module so named
2465
+ declaring no name of `strict` unless it is the export), and the
2466
+ names a Python file that
2467
+ does not tokenize holds, which it may declare."""
2468
+ hits: dict[str, list[str]] = {name: [] for name in names}
2469
+ unsure: set[str] = set()
2470
+ for parts in self._tree():
2471
+ module = _module_name(parts)
2472
+ lines = self._cited.get(parts)
2473
+ text = (
2474
+ _read_source(self.root.joinpath(*parts))
2475
+ if lines is None
2476
+ else "\n".join(lines) # a cited file, read already
2477
+ )
2478
+ # a strict name's module so named counts when it is the export
2479
+ fallback = module not in strict or _exports_itself(parts, text)
2480
+ # the substring test spares the rules most files and names
2481
+ here = sorted(
2482
+ n for n in names if n in text or (n == module and fallback)
2483
+ )
2484
+ if not here:
2485
+ continue
2486
+ if lines is None:
2487
+ lines = _text_lines(text)
2488
+ cache = _SourceCache() # this file's rules only: nothing kept
2489
+ path = Path(*parts)
2490
+ if path.suffix.lower() in PYTHON_EXTENSIONS and (
2491
+ cache.py_statement_starts(path, lines) is None
2492
+ ):
2493
+ unsure.update(n for n in here if n in text)
2494
+ for name in here:
2495
+ hit = self._first(parts, lines, name, cache, fallback)
2496
+ if hit is not None:
2497
+ hits[name].append(hit)
2498
+ return {name: sorted(_without_stubs(h)) for name, h in hits.items()}, unsure
2499
+
2500
+
2501
+ def inventory_kinds(data: object, path: str) -> dict[str, frozenset[str]]:
2502
+ """name -> the kinds a documented inventory gives it (`--inventory`):
2503
+ validate-inventory.py's `--output` file, `{"exports": [...]}`, or its
2504
+ whole result, the inventory under `inventory`. An item without a string
2505
+ `name` and `kind` is left out, and a kind is kept in lower case. Raises
2506
+ ValueError when no `exports` list is there."""
2507
+ if isinstance(data, dict) and isinstance(data.get("inventory"), dict):
2508
+ data = data["inventory"] # validate-inventory.py's whole result
2509
+ if not isinstance(data, dict) or not isinstance(data.get("exports"), list):
2510
+ raise ValueError(f"--inventory {path} holds no `exports` list")
2511
+ kinds: dict[str, set[str]] = {}
2512
+ for item in data["exports"]:
2513
+ if not isinstance(item, dict):
2514
+ continue
2515
+ name, kind = item.get("name"), item.get("kind")
2516
+ if isinstance(name, str) and isinstance(kind, str):
2517
+ kinds.setdefault(name, set()).add(kind.strip().lower())
2518
+ return {name: frozenset(k) for name, k in kinds.items()}
2519
+
2520
+
2521
+ def _module_entry(entry: dict) -> bool:
2522
+ """True for a `module` or `package` entry."""
2523
+ kind = entry.get("export_type")
2524
+ return isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
2525
+
2526
+
2527
+ def classify_stale(
2528
+ names: list[str],
2529
+ prov: dict,
2530
+ source_root: Path,
2531
+ kinds: dict[str, frozenset[str]] | None = None,
2532
+ ) -> list[dict]:
1651
2533
  """The fabricated-signature test for each stale name: the definition-lines
1652
- rules at every provenance entry whose raw `export_name` is the name. The
1653
- classify-stale section of the module docstring gives each reason, what a
1654
- `no-file` entry means and which entry gives `source`.
2534
+ rules at every provenance entry whose raw `export_name` is the name, and
2535
+ for a name that is not fabricated, where the source declares it
2536
+ (`defined_at`, and `declared_in` when the walk looked for it). The
2537
+ classify-stale section of the module docstring gives each reason, what
2538
+ a `no-file` entry means, which entry gives `source` and how
2539
+ `defined_at` and `declared_in` are found. `kinds` (`inventory_kinds`,
2540
+ None without `--inventory`) keeps a name of a `NO_MODULE_KINDS` kind
2541
+ from a module or package so named, unless an entry of the name is a
2542
+ `module` or `package` one.
1655
2543
  """
1656
2544
  entries = [e for e in prov.get("entries") or [] if isinstance(e, dict)]
1657
2545
  results: list[dict] = []
2546
+ # the names to look up (never a fabricated one): name -> its cited files
2547
+ cited: dict[str, list[str]] = {}
2548
+ # the names a module so named does not declare
2549
+ strict: set[str] = set()
1658
2550
  for name in names:
1659
2551
  mine = [e for e in entries if e.get("export_name") == name]
1660
2552
  checks: list[dict] = []
@@ -1683,15 +2575,33 @@ def classify_stale(names: list[str], prov: dict, source_root: Path) -> list[dict
1683
2575
  else:
1684
2576
  reason = STALE_UNCHECKED
1685
2577
  source = next(filter(None, map(_entry_citation, mine)), None)
2578
+ fabricated = reason == STALE_NOT_DEFINED
2579
+ if not fabricated:
2580
+ cited[name] = [f for f in map(_entry_file, mine) if f is not None]
2581
+ documented = (kinds or {}).get(name, frozenset())
2582
+ if documented & NO_MODULE_KINDS and not any(map(_module_entry, mine)):
2583
+ strict.add(name)
1686
2584
  results.append(
1687
2585
  {
1688
2586
  "name": name,
1689
- "fabricated": reason == STALE_NOT_DEFINED,
2587
+ "fabricated": fabricated,
1690
2588
  "source": source,
2589
+ "defined_at": None,
2590
+ "declared_in": None,
1691
2591
  "reason": reason,
1692
2592
  "entries": checks,
1693
2593
  }
1694
2594
  )
2595
+ try:
2596
+ found, declared = _DeclarationLookup(entries, source_root).find(
2597
+ cited, frozenset(strict)
2598
+ )
2599
+ except _LOOKUP_ERRORS:
2600
+ # a lookup that fails proves nothing: every name stays null
2601
+ found, declared = {}, {}
2602
+ for item in results:
2603
+ item["defined_at"] = found.get(item["name"])
2604
+ item["declared_in"] = declared.get(item["name"])
1695
2605
  return results
1696
2606
 
1697
2607
 
@@ -2898,8 +3808,12 @@ def _cmd_definition_lines(args: argparse.Namespace) -> int:
2898
3808
 
2899
3809
  def _cmd_classify_stale(args: argparse.Namespace) -> int:
2900
3810
  names_path, prov_path = Path(args.names), Path(args.provenance)
3811
+ inv_path = Path(args.inventory) if args.inventory is not None else None
2901
3812
  root = Path(args.source_root)
2902
- for label, path in (("names", names_path), ("provenance map", prov_path)):
3813
+ inputs = [("names", names_path), ("provenance map", prov_path)]
3814
+ if inv_path is not None:
3815
+ inputs.append(("inventory", inv_path))
3816
+ for label, path in inputs:
2903
3817
  if not path.is_file():
2904
3818
  print(f"error: {label} not found: {path}", file=sys.stderr)
2905
3819
  return 2
@@ -2919,11 +3833,14 @@ def _cmd_classify_stale(args: argparse.Namespace) -> int:
2919
3833
  try:
2920
3834
  names = stale_names(data)
2921
3835
  prov = load_json_object(prov_path, "provenance map")
3836
+ kinds = None if inv_path is None else inventory_kinds(
3837
+ load_json_object(inv_path, "inventory"), args.inventory
3838
+ )
2922
3839
  except ValueError as exc:
2923
3840
  print(f"error: {exc}", file=sys.stderr)
2924
3841
  return 2
2925
3842
  try:
2926
- result = classify_stale(names, prov, root)
3843
+ result = classify_stale(names, prov, root, kinds)
2927
3844
  except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
2928
3845
  return _unexpected("stale-name classification", exc)
2929
3846
  return 0 if _emit(result, args.output) else 2
@@ -3225,8 +4142,8 @@ def _build_parser() -> argparse.ArgumentParser:
3225
4142
  p = sub.add_parser(
3226
4143
  "classify-stale",
3227
4144
  help=(
3228
- "decide which stale names are fabricated signatures: no cited "
3229
- "file defines them"
4145
+ "decide which stale names are fabricated signatures (no cited "
4146
+ "file defines them) and where the source declares the others"
3230
4147
  ),
3231
4148
  )
3232
4149
  p.add_argument(
@@ -3245,6 +4162,17 @@ def _build_parser() -> argparse.ArgumentParser:
3245
4162
  required=True,
3246
4163
  help="source tree root the entries' source_file paths are under",
3247
4164
  )
4165
+ p.add_argument(
4166
+ "--inventory",
4167
+ default=None,
4168
+ help=(
4169
+ "validate-inventory.py's documented inventory: a name it gives a "
4170
+ "function, hook, class, interface, enum or type kind takes no "
4171
+ "line 1 from a module so named, unless a provenance map entry of "
4172
+ "the name is a module or package one or the module is a TS/JS "
4173
+ "file that is itself the export"
4174
+ ),
4175
+ )
3248
4176
  p.add_argument(
3249
4177
  "-o",
3250
4178
  "--output",