bmad-module-skill-forge 3.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/docs/_data/pinned.yaml +1 -1
- package/docs/examples.md +1 -1
- package/docs/troubleshooting.md +8 -0
- package/docs/verifying-a-skill.md +7 -3
- package/docs/workflows.md +3 -3
- package/package.json +1 -1
- package/src/knowledge/qmd-registry.md +1 -1
- package/src/knowledge/version-paths.md +1 -1
- package/src/shared/health-check.md +2 -50
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +2 -2
- package/src/shared/scripts/skf-atomic-write.py +212 -59
- package/src/shared/scripts/skf-build-change-manifest.py +593 -47
- package/src/shared/scripts/skf-classify-changed-files.py +78 -0
- package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
- package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
- package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
- package/src/shared/scripts/skf-extract-public-api.py +1309 -157
- package/src/shared/scripts/skf-extraction-inventory.py +580 -46
- package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
- package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
- package/src/shared/scripts/skf-github-probe.py +47 -8
- package/src/shared/scripts/skf-hash-content.py +14 -8
- package/src/shared/scripts/skf-load-provenance.py +3 -1
- package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
- package/src/shared/scripts/skf-preflight.py +15 -1
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
- package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
- package/src/shared/scripts/skf-structural-diff.py +133 -18
- package/src/shared/scripts/skf-update-active-symlink.py +115 -62
- package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
- package/src/shared/scripts/skf-verify-provenance-completeness.py +977 -49
- package/src/shared/scripts/skf-write-skill-brief.py +41 -8
- package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
- package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
- package/src/skf-audit-skill/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +2 -1
- package/src/skf-audit-skill/references/re-index.md +14 -9
- package/src/skf-audit-skill/references/report.md +2 -2
- package/src/skf-audit-skill/references/semantic-diff.md +15 -15
- package/src/skf-audit-skill/references/structural-diff.md +10 -5
- package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
- package/src/skf-brief-skill/SKILL.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +1 -1
- package/src/skf-brief-skill/assets/skill-brief-schema.md +2 -9
- package/src/skf-brief-skill/references/analyze-target.md +6 -6
- package/src/skf-brief-skill/references/confirm-brief.md +1 -1
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
- package/src/skf-brief-skill/references/version-resolution.md +5 -9
- package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
- package/src/skf-create-skill/assets/skill-sections.md +6 -6
- package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
- package/src/skf-create-skill/references/compile.md +31 -60
- package/src/skf-create-skill/references/component-extraction.md +3 -3
- package/src/skf-create-skill/references/enrich.md +2 -2
- package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
- package/src/skf-create-skill/references/extract.md +22 -19
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
- package/src/skf-create-skill/references/extraction-patterns.md +8 -9
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +16 -5
- package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
- package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
- package/src/skf-drop-skill/references/execute.md +5 -1
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +2 -2
- package/src/skf-drop-skill/references/select.md +7 -3
- package/src/skf-test-skill/references/coherence-check.md +1 -1
- package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
- package/src/skf-test-skill/references/coverage-check.md +39 -57
- package/src/skf-test-skill/references/init.md +3 -5
- package/src/skf-test-skill/references/report.md +10 -8
- package/src/skf-test-skill/references/scoring-rules.md +6 -5
- package/src/skf-test-skill/references/source-access-protocol.md +14 -17
- package/src/skf-test-skill/scripts/build-result-context.py +114 -21
- package/src/skf-test-skill/scripts/gap-ledger.py +315 -30
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +534 -96
- package/src/skf-test-skill/scripts/reconcile-coverage.py +276 -2
- package/src/skf-test-skill/scripts/score-signatures.py +36 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
- package/src/skf-update-skill/references/detect-changes.md +32 -13
- package/src/skf-update-skill/references/gap-driven.md +3 -3
- package/src/skf-update-skill/references/init.md +6 -2
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +2 -2
- package/src/skf-update-skill/references/re-extract.md +2 -2
- package/src/skf-update-skill/references/report.md +3 -1
- package/src/skf-update-skill/references/write.md +43 -23
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
|
@@ -33,8 +33,9 @@ entries are per-citation, so the two sets measure different surfaces:
|
|
|
33
33
|
Four more subcommands act on the same rules: `fix` applies the citation
|
|
34
34
|
prefix and source-line fixes a `verify` result has one answer for,
|
|
35
35
|
`definition-lines` gives the lines that define one export,
|
|
36
|
-
`classify-stale` decides which stale names are fabricated signatures
|
|
37
|
-
`kind-at` gives the node kind
|
|
36
|
+
`classify-stale` decides which stale names are fabricated signatures and
|
|
37
|
+
where the source declares the others, and `kind-at` gives the node kind
|
|
38
|
+
of the recipe that matches a source line.
|
|
38
39
|
|
|
39
40
|
An LLM set-diff can silently pass a dropped or orphaned entry, and eyeballing
|
|
40
41
|
whether a cited line still exists is not something the model can do reliably.
|
|
@@ -389,20 +390,26 @@ Subcommands:
|
|
|
389
390
|
rule dropped joins `definition_lines`, as in `verify`.
|
|
390
391
|
|
|
391
392
|
classify-stale --names <coverage.json> --provenance <provenance-map.json>
|
|
392
|
-
--source-root <path> [
|
|
393
|
+
--source-root <path> [--inventory <inventory.json>]
|
|
394
|
+
[-o <out.json>]
|
|
393
395
|
|
|
394
396
|
test-skill's fabricated-signature test, for each stale name (a
|
|
395
397
|
documented name the enumerated source surface lacks): the
|
|
396
398
|
definition-lines rules above, run for every provenance entry whose raw
|
|
397
399
|
`export_name` is the name, at its `source_file` and `source_line`, with
|
|
398
400
|
its `export_type`. `--names` holds reconcile-coverage.py's result (its
|
|
399
|
-
`stale` names) or a JSON array of names.
|
|
401
|
+
`stale` names) or a JSON array of names. `--inventory` holds the
|
|
402
|
+
documented inventory, validate-inventory.py's `--output` file or its
|
|
403
|
+
whole result: the `kind` it gives each name decides the module rule
|
|
404
|
+
below.
|
|
400
405
|
|
|
401
406
|
Emit JSON, one item per name, in the order given (a repeated name once):
|
|
402
407
|
[
|
|
403
408
|
{"name": "<as given>",
|
|
404
409
|
"fabricated": <bool>,
|
|
405
410
|
"source": "<source_file>:<source_line>" | "<source_file>" | null,
|
|
411
|
+
"defined_at": "<path>:<line>" | null,
|
|
412
|
+
"declared_in": ["<path>:<line>", ...] | null,
|
|
406
413
|
"reason": "not-defined" | "defined" | "unchecked" | "no-entry",
|
|
407
414
|
"entries": [{"source_file": "<as recorded>",
|
|
408
415
|
"source_line": <as recorded>,
|
|
@@ -428,6 +435,80 @@ Subcommands:
|
|
|
428
435
|
entry with a `source_file` (the file alone when its line is not a
|
|
429
436
|
number), null when no entry has one.
|
|
430
437
|
|
|
438
|
+
`defined_at` is where the source declares a name that is not
|
|
439
|
+
fabricated, so test-skill can tell a documented extra (a name outside
|
|
440
|
+
the surface that the source still declares) from stale documentation;
|
|
441
|
+
a fabricated name is not looked up (null). A declaration is a line at
|
|
442
|
+
column 0 that the definition-line rules match with imports left out:
|
|
443
|
+
a `def`, `class`, assignment, annotation, type alias, TS/JS
|
|
444
|
+
declaration, `exports.NAME =`, or an alias that renames
|
|
445
|
+
(`import X as NAME`, `from X import Y as NAME`,
|
|
446
|
+
`export { Y as NAME }`, `export * as NAME from`), or a
|
|
447
|
+
`module.exports` key with a value or a method. A plain or star import,
|
|
448
|
+
an `as` that keeps the imported name (`from m import NAME as NAME`,
|
|
449
|
+
`export { NAME as NAME }`), `export import NAME =`,
|
|
450
|
+
`export default NAME`, an export-list item without `as`, a
|
|
451
|
+
`module.exports` shorthand key, a binding whose value is an import
|
|
452
|
+
(`require(...)` or `(await) import(...)`, on its line or the next:
|
|
453
|
+
`const NAME = require(...)`, `exports.NAME = require(...)`,
|
|
454
|
+
`{ NAME: require(...) }`), an indented line and any line of a Python
|
|
455
|
+
file that does not tokenize never count. A module or package named
|
|
456
|
+
NAME (`NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
|
|
457
|
+
`NAME/index.<ext>`) declares NAME too: at its line 1 when no line of it
|
|
458
|
+
declares NAME, unless `--inventory` gives NAME a `function`, `hook`,
|
|
459
|
+
`class`, `interface`, `enum` or `type` kind (any of its kinds, when it
|
|
460
|
+
gives several; a `method` kind is not read), no entry of NAME is a
|
|
461
|
+
`module` or `package` one, and the file is not a TS/JS one that is
|
|
462
|
+
itself the export (a column-0 `export default` or `module.exports =`),
|
|
463
|
+
so that a removed function named after its file is no declaration
|
|
464
|
+
(the inventory has no `module` kind). `declared_line` gives this
|
|
465
|
+
answer for one file, the module rule optional, to a script that loads
|
|
466
|
+
this one, and
|
|
467
|
+
`baseline_gaps` applies it to the map's entries an extraction left
|
|
468
|
+
out (skf-extraction-snapshot.py's baseline gaps and
|
|
469
|
+
skf-build-change-manifest.py's `baseline-gaps`, which apply the module
|
|
470
|
+
rule only to a `module` or `package` entry).
|
|
471
|
+
|
|
472
|
+
The files: when a file the name's entries cite declares it,
|
|
473
|
+
`defined_at` is that file's first declaration (the files in entry
|
|
474
|
+
order), and `declared_in` is null. Otherwise `declared_in` lists, as
|
|
475
|
+
sorted `<path>:<line>`, every file of the walk that declares the name,
|
|
476
|
+
at its first declaration (a type stub, `NAME.pyi` or `NAME.d.ts`,
|
|
477
|
+
beside its implementation is one file with it: the implementation),
|
|
478
|
+
and `defined_at` is the one item when the list has exactly one and no
|
|
479
|
+
walked Python file that does not tokenize holds the name (a homonym,
|
|
480
|
+
two files or more, gives null: test-skill decides it by the files the
|
|
481
|
+
skill cites). The walk reads, sorted by path, every file below the
|
|
482
|
+
walk root: the deepest folder that the files of the map's entries
|
|
483
|
+
share, widened to its package root (Python: the nearest folder at or
|
|
484
|
+
above it that holds `__init__.py`, then up while the parent holds
|
|
485
|
+
one, never the source root itself; TS/JS: the nearest folder at or
|
|
486
|
+
above it that holds `package.json`), leaving out the entries whose
|
|
487
|
+
file lies outside every package; the source root when none lies in
|
|
488
|
+
one. Only the rule families (Python, TS/JS) the map's entries cite are
|
|
489
|
+
read (both when no entry cites a file). The walk skips symlinks, hidden folders, test
|
|
490
|
+
files and test configuration files (`conftest.py`, `jest.config.*`),
|
|
491
|
+
`*.min.js`, and the folders generated, built, installed, vendored or
|
|
492
|
+
test output lives in (`node_modules`, `dist`, `build`, `vendor`,
|
|
493
|
+
`_vendor`, `tests`, `testing`, `fixtures`, `__mocks__`,
|
|
494
|
+
`site-packages`, `env`, `bower_components`, `esm`, `cjs`, `umd` and
|
|
495
|
+
the like), `docs`, `examples` and `benchmarks`, each judged on the
|
|
496
|
+
path below the walk root, the end-to-end test folders `e2e` and
|
|
497
|
+
`__fixtures__` wherever they lie, and, as the first folder below the
|
|
498
|
+
walk root or below a package root inside it (a folder that holds
|
|
499
|
+
`package.json`, the top of an `__init__.py` chain), `example`, `demo`,
|
|
500
|
+
`demos`, `sample`, `samples`, `scripts`, `playground`, `bench`,
|
|
501
|
+
`benchmark` and `migrations` unless it holds a file an entry cites
|
|
502
|
+
(deeper in a package such a folder can be package code). A cited file
|
|
503
|
+
is read only inside the source root (a path that is absolute or climbs
|
|
504
|
+
out of it is not read), with no symlink on its path and under the same
|
|
505
|
+
skips, those walk-only ones aside. The path is relative to the source
|
|
506
|
+
root. A dotted name, or a map whose entries cite files
|
|
507
|
+
but no Python or TS/JS one, is not looked up (`defined_at` and
|
|
508
|
+
`declared_in` null, as for a fabricated name), and a filesystem error
|
|
509
|
+
during the lookup gives every name null in both (the exit code stays
|
|
510
|
+
0).
|
|
511
|
+
|
|
431
512
|
kind-at --file <path> --line <n> --recipes <recipes>
|
|
432
513
|
[--language <lang>] [--name <export_name>]
|
|
433
514
|
[--source-root <path>] [-o <out.json>]
|
|
@@ -483,7 +564,8 @@ CLI examples:
|
|
|
483
564
|
uv run skf-verify-provenance-completeness.py classify-stale \\
|
|
484
565
|
--names {run_dir}/coverage.json \\
|
|
485
566
|
--provenance {forge_version}/provenance-map.json \\
|
|
486
|
-
--source-root {source_root}
|
|
567
|
+
--source-root {source_root} --inventory {run_dir}/inventory.json \\
|
|
568
|
+
-o {run_dir}/stale.json
|
|
487
569
|
uv run skf-verify-provenance-completeness.py kind-at \\
|
|
488
570
|
--source-root {source_root} --file src/api.ts --line 12 --name Store \\
|
|
489
571
|
--recipes {extractionPatternsData}
|
|
@@ -507,11 +589,11 @@ Exit codes:
|
|
|
507
589
|
2 error, for every subcommand (an input file or folder not found,
|
|
508
590
|
--skill-dir without a SKILL.md, malformed JSON or YAML, a verify
|
|
509
591
|
result without stale[] and citations[], --names without a list of
|
|
510
|
-
names,
|
|
511
|
-
|
|
512
|
-
unexpected failure, such as an
|
|
513
|
-
file; once fix has started writing, stderr names the files
|
|
514
|
-
written); one line on stderr and no JSON
|
|
592
|
+
names, --inventory without an `exports` list, no recipe in
|
|
593
|
+
--recipes, no PyYAML for kind-at, an atomic write that failed, an -o
|
|
594
|
+
file that cannot be written, or any unexpected failure, such as an
|
|
595
|
+
unreadable file; once fix has started writing, stderr names the files
|
|
596
|
+
already written); one line on stderr and no JSON
|
|
515
597
|
"""
|
|
516
598
|
|
|
517
599
|
from __future__ import annotations
|
|
@@ -828,6 +910,9 @@ _PY_TARGET_LIST_RE = re.compile(
|
|
|
828
910
|
)
|
|
829
911
|
# `def \` or `class \`: the name is on the next line.
|
|
830
912
|
_PY_SPLIT_DECL_RE = re.compile(r"^\s*(?:async\s+def|def|class)\s*\\$")
|
|
913
|
+
# A line that can matter to a name it does not hold: an import (which may go
|
|
914
|
+
# on over the next lines) or a `def \` / `class \` whose name is on the next.
|
|
915
|
+
_PY_NAMELESS_MATCH_RE = re.compile(r"^\s*(?:from|import|async|def|class)\b")
|
|
831
916
|
# Tokens that never begin a statement.
|
|
832
917
|
_PY_LAYOUT_TOKENS = frozenset(
|
|
833
918
|
{
|
|
@@ -861,6 +946,15 @@ _JS_EXPORT_ITEM_RE = re.compile(
|
|
|
861
946
|
rf"^(?:type\s+)?({_JS_NAME_OR_STRING})(?:\s+as\s+({_JS_NAME_OR_STRING}))?$"
|
|
862
947
|
)
|
|
863
948
|
_JS_MODULE_EXPORTS_OBJECT_RE = re.compile(r"^\s*module\.exports\s*=\s*\{(.*)$")
|
|
949
|
+
# A bound value that is an import: `require(...)`, `import(...)`, awaited, in
|
|
950
|
+
# parentheses (`(await import("./m")).X`) or in one helper call
|
|
951
|
+
# (`__importDefault(require("./m"))`).
|
|
952
|
+
_JS_IMPORT_VALUE_RE = re.compile(
|
|
953
|
+
r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?(?:await\s+)?(?:require|import)\s*\("
|
|
954
|
+
)
|
|
955
|
+
# The start of such a value that goes on on the next line: nothing yet, a
|
|
956
|
+
# parenthesis, or a helper call's name and parenthesis.
|
|
957
|
+
_JS_IMPORT_VALUE_OPEN_RE = re.compile(r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?$")
|
|
864
958
|
# A `const`, `let` or `var` statement: its declarator list.
|
|
865
959
|
_JS_VAR_DECL_RE = re.compile(
|
|
866
960
|
rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(?!enum(?![\w$]))(.*)$"
|
|
@@ -870,6 +964,18 @@ _JS_SPLIT_DECL_RE = re.compile(
|
|
|
870
964
|
rf"^\s*{_JS_MODIFIERS}(?P<kind>function(?:\s*\*)?|class|interface|type"
|
|
871
965
|
r"|enum|namespace|let|var|const(?:\s+enum)?)$"
|
|
872
966
|
)
|
|
967
|
+
# The keywords every line that opens a statement going on below holds (a
|
|
968
|
+
# `const` / `let` / `var` list, a split declaration, `export {`,
|
|
969
|
+
# `module.exports = {`): a line with none of them, and not the name, can
|
|
970
|
+
# match no rule while no statement is open.
|
|
971
|
+
_JS_OPENING_WORDS = ("const", "let", "var", "function", "class", "interface", "type", "enum", "namespace",
|
|
972
|
+
"export", "module")
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
def _js_quiet_lines(code_lines: list[str]) -> list[bool]:
|
|
976
|
+
"""For each comment-free line, True when it holds none of
|
|
977
|
+
`_JS_OPENING_WORDS`."""
|
|
978
|
+
return [not any(word in code for word in _JS_OPENING_WORDS) for code in code_lines]
|
|
873
979
|
_JS_VAR_KEYWORDS = frozenset({"const", "let", "var"})
|
|
874
980
|
# How a `const` / `let` / `var` list that did not end on its line goes on
|
|
875
981
|
# (`_js_declarators`): the next line begins a declarator (this one ended
|
|
@@ -909,6 +1015,21 @@ def _read_lines(path: Path) -> list[str] | None:
|
|
|
909
1015
|
return None
|
|
910
1016
|
|
|
911
1017
|
|
|
1018
|
+
def _read_source(path: Path) -> str:
|
|
1019
|
+
"""A text file's content, decoded as `_read_lines` decodes it. Raises
|
|
1020
|
+
OSError when the file cannot be read."""
|
|
1021
|
+
with path.open("r", encoding="utf-8-sig", errors="replace") as fh:
|
|
1022
|
+
return fh.read()
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
def _text_lines(text: str) -> list[str]:
|
|
1026
|
+
"""`_read_source`'s text as the lines `_read_lines` gives for the file."""
|
|
1027
|
+
lines = text.split("\n")
|
|
1028
|
+
if lines[-1] == "":
|
|
1029
|
+
lines.pop()
|
|
1030
|
+
return lines
|
|
1031
|
+
|
|
1032
|
+
|
|
912
1033
|
def _is_column_0(text: str) -> bool:
|
|
913
1034
|
"""True when a line starts at column 0 (no leading whitespace)."""
|
|
914
1035
|
return not text[:1].isspace()
|
|
@@ -992,6 +1113,7 @@ class _SourceCache:
|
|
|
992
1113
|
self._lines: dict[Path, list[str] | None] = {}
|
|
993
1114
|
self._starts: dict[Path, set[int] | None] = {}
|
|
994
1115
|
self._js_code: dict[Path, list[str]] = {}
|
|
1116
|
+
self._js_quiet: dict[Path, list[bool]] = {}
|
|
995
1117
|
self._js_decl: dict[tuple[str, str, bool], _DeclSplit] = {}
|
|
996
1118
|
|
|
997
1119
|
def lines(self, path: Path) -> list[str] | None:
|
|
@@ -1009,6 +1131,11 @@ class _SourceCache:
|
|
|
1009
1131
|
self._js_code[path] = _js_code_lines(lines)
|
|
1010
1132
|
return self._js_code[path]
|
|
1011
1133
|
|
|
1134
|
+
def js_quiet_lines(self, path: Path, code_lines: list[str]) -> list[bool]:
|
|
1135
|
+
if path not in self._js_quiet:
|
|
1136
|
+
self._js_quiet[path] = _js_quiet_lines(code_lines)
|
|
1137
|
+
return self._js_quiet[path]
|
|
1138
|
+
|
|
1012
1139
|
def js_declarators(
|
|
1013
1140
|
self, text: str, opened: str = "", fresh: bool = True
|
|
1014
1141
|
) -> _DeclSplit:
|
|
@@ -1018,8 +1145,12 @@ class _SourceCache:
|
|
|
1018
1145
|
return self._js_decl[key]
|
|
1019
1146
|
|
|
1020
1147
|
|
|
1021
|
-
def _py_import_binds(
|
|
1022
|
-
|
|
1148
|
+
def _py_import_binds(
|
|
1149
|
+
items: str, name: str, from_import: bool, alias_only: bool = False
|
|
1150
|
+
) -> bool:
|
|
1151
|
+
"""True when an import item list binds `name` (with `alias_only`, only
|
|
1152
|
+
through an explicit `as name` that renames: `from m import name as
|
|
1153
|
+
name` and `import a.name as name` keep the imported name)."""
|
|
1023
1154
|
items = items.split("#", 1)[0]
|
|
1024
1155
|
for raw in items.replace("(", " ").replace(")", " ").replace("\\", " ").split(","):
|
|
1025
1156
|
m = _PY_IMPORT_ITEM_RE.match(raw.strip())
|
|
@@ -1027,7 +1158,11 @@ def _py_import_binds(items: str, name: str, from_import: bool) -> bool:
|
|
|
1027
1158
|
continue
|
|
1028
1159
|
target, alias = m.group(1), m.group(2)
|
|
1029
1160
|
if alias is not None:
|
|
1161
|
+
if alias_only and alias == target.rsplit(".", 1)[-1]:
|
|
1162
|
+
continue
|
|
1030
1163
|
bound = alias
|
|
1164
|
+
elif alias_only:
|
|
1165
|
+
continue
|
|
1031
1166
|
elif from_import:
|
|
1032
1167
|
bound = target
|
|
1033
1168
|
else:
|
|
@@ -1050,7 +1185,10 @@ def _py_assigned_names(code: str) -> set[str]:
|
|
|
1050
1185
|
|
|
1051
1186
|
|
|
1052
1187
|
def _python_definition_lines(
|
|
1053
|
-
lines: list[str],
|
|
1188
|
+
lines: list[str],
|
|
1189
|
+
name: str,
|
|
1190
|
+
starts: set[int] | None,
|
|
1191
|
+
declarations_only: bool = False,
|
|
1054
1192
|
) -> list[tuple[int, bool]]:
|
|
1055
1193
|
"""(line, at column 0) for every line that defines `name`.
|
|
1056
1194
|
|
|
@@ -1059,7 +1197,9 @@ def _python_definition_lines(
|
|
|
1059
1197
|
line of a `def \\` or `class \\` whose name is on the next line. A
|
|
1060
1198
|
`from X import *` line counts only when no other line defines `name` (a
|
|
1061
1199
|
re-export the extraction could not trace further is cited at its star
|
|
1062
|
-
import).
|
|
1200
|
+
import). With `declarations_only`, an import counts only when it binds
|
|
1201
|
+
`name` through an explicit `as name` that renames (`_py_import_binds`),
|
|
1202
|
+
and a star import never does.
|
|
1063
1203
|
"""
|
|
1064
1204
|
esc = re.escape(name)
|
|
1065
1205
|
decl = re.compile(rf"^\s*(?:async\s+def|def|class)\s+{esc}(?!\w)")
|
|
@@ -1085,11 +1225,13 @@ def _python_definition_lines(
|
|
|
1085
1225
|
code = stripped[:-1]
|
|
1086
1226
|
else:
|
|
1087
1227
|
cont = None
|
|
1088
|
-
if _py_import_binds(code, name, from_import):
|
|
1228
|
+
if _py_import_binds(code, name, from_import, declarations_only):
|
|
1089
1229
|
found.append((number, top))
|
|
1090
1230
|
continue
|
|
1091
1231
|
if starts is not None and number not in starts:
|
|
1092
1232
|
continue
|
|
1233
|
+
if name not in code and not _PY_NAMELESS_MATCH_RE.match(code):
|
|
1234
|
+
continue # no rule below can match it: spares a scan per name
|
|
1093
1235
|
top = _is_column_0(text)
|
|
1094
1236
|
if (
|
|
1095
1237
|
decl.match(code)
|
|
@@ -1114,27 +1256,33 @@ def _python_definition_lines(
|
|
|
1114
1256
|
continue
|
|
1115
1257
|
rest = m.group(1)
|
|
1116
1258
|
if from_import and rest.strip() == "*":
|
|
1117
|
-
|
|
1259
|
+
if not declarations_only:
|
|
1260
|
+
star_imports.append((number, top))
|
|
1118
1261
|
continue
|
|
1119
1262
|
if from_import and rest.lstrip().startswith("(") and ")" not in rest:
|
|
1120
1263
|
cont = ("paren", top, True)
|
|
1121
1264
|
elif rest.rstrip().endswith("\\"):
|
|
1122
1265
|
cont = ("backslash", top, from_import)
|
|
1123
1266
|
rest = rest.rstrip()[:-1]
|
|
1124
|
-
if _py_import_binds(rest, name, from_import):
|
|
1267
|
+
if _py_import_binds(rest, name, from_import, declarations_only):
|
|
1125
1268
|
found.append((number, top))
|
|
1126
1269
|
return found or star_imports
|
|
1127
1270
|
|
|
1128
1271
|
|
|
1129
|
-
def _js_export_item_binds(item: str, name: str) -> bool:
|
|
1272
|
+
def _js_export_item_binds(item: str, name: str, alias_only: bool = False) -> bool:
|
|
1130
1273
|
"""True when one `export { ... }` item exposes `name`: the name after
|
|
1131
|
-
`as`, else the item's own name, an identifier or
|
|
1274
|
+
`as`, else (unless `alias_only`) the item's own name, an identifier or
|
|
1275
|
+
a string. With `alias_only`, an `as` that keeps the item's own name
|
|
1276
|
+
(`NAME as NAME`) does not count either."""
|
|
1132
1277
|
m = _JS_EXPORT_ITEM_RE.match(item.strip())
|
|
1133
|
-
if not m:
|
|
1278
|
+
if not m or (alias_only and m.group(2) is None):
|
|
1279
|
+
return False
|
|
1280
|
+
own, exposed = (
|
|
1281
|
+
s[1:-1] if s[:1] in ("'", '"') else s
|
|
1282
|
+
for s in (m.group(1), m.group(2) or m.group(1))
|
|
1283
|
+
)
|
|
1284
|
+
if alias_only and exposed == own:
|
|
1134
1285
|
return False
|
|
1135
|
-
exposed = m.group(2) or m.group(1)
|
|
1136
|
-
if exposed[:1] in ("'", '"'):
|
|
1137
|
-
exposed = exposed[1:-1]
|
|
1138
1286
|
return exposed == name
|
|
1139
1287
|
|
|
1140
1288
|
|
|
@@ -1327,10 +1475,14 @@ def _tsjs_definition_lines(
|
|
|
1327
1475
|
name: str,
|
|
1328
1476
|
dotted: bool,
|
|
1329
1477
|
declarators: Callable[..., _DeclSplit] = _js_declarators,
|
|
1478
|
+
declarations_only: bool = False,
|
|
1479
|
+
quiet: list[bool] | None = None,
|
|
1330
1480
|
) -> list[tuple[int, bool]]:
|
|
1331
1481
|
"""(line, at column 0) for every line that defines `name`.
|
|
1332
1482
|
|
|
1333
|
-
`
|
|
1483
|
+
`quiet` is `_js_quiet_lines(code_lines)` when the caller has it (it does
|
|
1484
|
+
not depend on the name): a quiet line without the name is passed over
|
|
1485
|
+
while no statement is open. `code_lines` are `lines` with comments removed (`_js_code_lines`), so a
|
|
1334
1486
|
JSDoc line or a commented-out declaration never matches. A line inside
|
|
1335
1487
|
a multi-line `export { ... }` list or `module.exports = { ... }` object
|
|
1336
1488
|
takes the column of the line the statement starts on, and so do the
|
|
@@ -1338,29 +1490,90 @@ def _tsjs_definition_lines(
|
|
|
1338
1490
|
later declarator on a line of its own. Class members count only for a
|
|
1339
1491
|
dotted export name. `declarators` splits a declarator list
|
|
1340
1492
|
(`_js_declarators`, or `_SourceCache.js_declarators`, which splits each
|
|
1341
|
-
line once for every export looked up).
|
|
1493
|
+
line once for every export looked up). With `declarations_only`,
|
|
1494
|
+
`export import NAME =`, `export default NAME`, an export-list item
|
|
1495
|
+
without an `as` that renames, a `module.exports` shorthand key and an
|
|
1496
|
+
import-valued binding (a declarator, `exports.NAME =` or a
|
|
1497
|
+
`module.exports` key whose value starts with `require(` or an
|
|
1498
|
+
`(await) import(`, on its line or the next one) never count, nor does
|
|
1499
|
+
a class member.
|
|
1342
1500
|
"""
|
|
1343
1501
|
esc = re.escape(name)
|
|
1344
1502
|
end = r"(?![\w$])"
|
|
1345
|
-
statements =
|
|
1503
|
+
statements = [
|
|
1346
1504
|
re.compile(rf"^\s*{_JS_MODIFIERS}{_JS_KINDS}{esc}{end}"),
|
|
1347
|
-
re.compile(rf"^\s*export\s+import\s+(?:type\s+)?{esc}\s*=(?![=>])"),
|
|
1348
1505
|
re.compile(
|
|
1349
1506
|
rf"^\s*export\s+(?:type\s+)?\*\s+as\s+"
|
|
1350
1507
|
rf"(?:{esc}\s+|([\"']){esc}\1\s*)from{end}"
|
|
1351
1508
|
),
|
|
1352
|
-
re.compile(rf"^\s*export\s+default\s+{esc}\s*;?\s*$"),
|
|
1353
1509
|
re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])"),
|
|
1354
|
-
|
|
1510
|
+
]
|
|
1511
|
+
if not declarations_only:
|
|
1512
|
+
statements += [
|
|
1513
|
+
re.compile(rf"^\s*export\s+import\s+(?:type\s+)?{esc}\s*=(?![=>])"),
|
|
1514
|
+
re.compile(rf"^\s*export\s+default\s+{esc}\s*;?\s*$"),
|
|
1515
|
+
]
|
|
1355
1516
|
# One declarator of a `const` / `let` / `var` list: `NAME = v`,
|
|
1356
1517
|
# `NAME: T = v`, `NAME!: T`, or a bare `NAME`. A destructuring pattern
|
|
1357
1518
|
# starts with `{` or `[` and never matches.
|
|
1358
1519
|
declarator = re.compile(rf"^\s*{esc}{end}\s*(?:!?\s*:|=|$)")
|
|
1520
|
+
# With declarations_only, the value bound to NAME (the rest of a
|
|
1521
|
+
# declarator, of `exports.NAME =` or of a `module.exports` key), to
|
|
1522
|
+
# test for an import.
|
|
1523
|
+
# (A type annotation holds no `=` but in `=>`: `NAME: () => T = v`.)
|
|
1524
|
+
declarator_value = re.compile(
|
|
1525
|
+
rf"^\s*{esc}{end}\s*(?:!?\s*:(?:[^=]|=>)*)?=(?![=>])(.*)$"
|
|
1526
|
+
)
|
|
1527
|
+
var_statement = re.compile(rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(.*)$")
|
|
1528
|
+
exports_value = re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])(.*)$")
|
|
1529
|
+
key_value = re.compile(rf"^\s*(?:(['\"]){esc}\1|{esc}{end})\s*:(.*)$")
|
|
1530
|
+
|
|
1531
|
+
def imported(value: str, number: int) -> bool:
|
|
1532
|
+
"""True when a bound value is an import: `require(...)`, or an
|
|
1533
|
+
`import(...)` awaited or not, alone or in one helper call. A value
|
|
1534
|
+
that goes on on the next line (`NAME =`, `NAME = (` or
|
|
1535
|
+
`NAME = helper(` ends line `number`) is read there too."""
|
|
1536
|
+
if _JS_IMPORT_VALUE_OPEN_RE.match(value):
|
|
1537
|
+
value += next((c for c in code_lines[number:] if c.strip()), "")
|
|
1538
|
+
return _JS_IMPORT_VALUE_RE.match(value) is not None
|
|
1539
|
+
|
|
1540
|
+
def declares(piece: str, number: int) -> bool:
|
|
1541
|
+
if declarator.match(piece) is None:
|
|
1542
|
+
return False
|
|
1543
|
+
if not declarations_only:
|
|
1544
|
+
return True
|
|
1545
|
+
m = declarator_value.match(piece)
|
|
1546
|
+
return m is None or not imported(m.group(1), number)
|
|
1547
|
+
|
|
1548
|
+
def is_statement(code: str, number: int) -> bool:
|
|
1549
|
+
if not any(p.match(code) for p in statements):
|
|
1550
|
+
return False
|
|
1551
|
+
if not declarations_only:
|
|
1552
|
+
return True
|
|
1553
|
+
m = var_statement.match(code)
|
|
1554
|
+
if m is not None:
|
|
1555
|
+
m = declarator_value.match(m.group(1))
|
|
1556
|
+
else:
|
|
1557
|
+
m = exports_value.match(code)
|
|
1558
|
+
return m is None or not imported(m.group(1), number)
|
|
1559
|
+
|
|
1359
1560
|
leading_name = re.compile(rf"^\s*{esc}{end}")
|
|
1561
|
+
# A key with a value, a method, or (unless declarations_only) a shorthand.
|
|
1562
|
+
key_end = r"[:(]" if declarations_only else r"[:(]|$"
|
|
1360
1563
|
object_key = re.compile(
|
|
1361
1564
|
rf"^\s*(?:(?:async|get|set)\s+|\*\s*)*(?:(['\"]){esc}\1|{esc}{end})"
|
|
1362
|
-
rf"\s*(?:
|
|
1565
|
+
rf"\s*(?:{key_end})"
|
|
1363
1566
|
)
|
|
1567
|
+
|
|
1568
|
+
def binds_key(piece: str, number: int) -> bool:
|
|
1569
|
+
if object_key.match(piece) is None:
|
|
1570
|
+
return False
|
|
1571
|
+
if not declarations_only:
|
|
1572
|
+
return True
|
|
1573
|
+
m = key_value.match(piece)
|
|
1574
|
+
return m is None or not imported(m.group(2), number)
|
|
1575
|
+
|
|
1576
|
+
dotted = dotted and not declarations_only
|
|
1364
1577
|
member = re.compile(
|
|
1365
1578
|
rf"^\s+(?P<mods>{_JS_MEMBER_MODIFIERS})(?:\*\s*)?{esc}{end}\s*[?!]?\s*"
|
|
1366
1579
|
rf"(?:<[^>]*>\s*)?(?P<tail>[(:=;]|$)"
|
|
@@ -1375,7 +1588,11 @@ def _tsjs_definition_lines(
|
|
|
1375
1588
|
# nested in another's initializer included: (column 0, brackets left
|
|
1376
1589
|
# open, how, as `_js_declarators` gives them).
|
|
1377
1590
|
decls: list[tuple[bool, str, str]] = []
|
|
1591
|
+
if quiet is None:
|
|
1592
|
+
quiet = _js_quiet_lines(code_lines)
|
|
1378
1593
|
for number, (text, code) in enumerate(zip(lines, code_lines), start=1):
|
|
1594
|
+
if quiet[number - 1] and not (decls or split or cont) and name not in code:
|
|
1595
|
+
continue # no rule below can match it: spares a scan per name
|
|
1379
1596
|
# A line of a list carried from above still goes through the rules
|
|
1380
1597
|
# below: it may hold a class member or a local of an arrow function.
|
|
1381
1598
|
if decls and code.strip():
|
|
@@ -1387,7 +1604,7 @@ def _tsjs_definition_lines(
|
|
|
1387
1604
|
carry, pieces = declarators(code, opened, how == _DECL_NEXT)
|
|
1388
1605
|
if carry is not None:
|
|
1389
1606
|
carried.append((decl_top, *carry))
|
|
1390
|
-
if any(
|
|
1607
|
+
if any(declares(p, number) for p in pieces):
|
|
1391
1608
|
# a later declarator on a line of its own, as Prettier
|
|
1392
1609
|
# writes it: `export const a = 1,` then ` NAME = 2;`
|
|
1393
1610
|
found.append((number, decl_top))
|
|
@@ -1399,7 +1616,7 @@ def _tsjs_definition_lines(
|
|
|
1399
1616
|
carry, pieces = declarators(code)
|
|
1400
1617
|
if carry is not None:
|
|
1401
1618
|
decls.append((top, *carry))
|
|
1402
|
-
hit = any(
|
|
1619
|
+
hit = any(declares(p, number) for p in pieces)
|
|
1403
1620
|
else:
|
|
1404
1621
|
hit = leading_name.match(code) is not None
|
|
1405
1622
|
if hit:
|
|
@@ -1411,14 +1628,18 @@ def _tsjs_definition_lines(
|
|
|
1411
1628
|
depth, pieces = _js_depth1_pieces(code, depth)
|
|
1412
1629
|
cont = (kind, top, depth) if depth > 0 else None
|
|
1413
1630
|
if kind == "list":
|
|
1414
|
-
hit = any(
|
|
1631
|
+
hit = any(
|
|
1632
|
+
_js_export_item_binds(p, name, declarations_only) for p in pieces
|
|
1633
|
+
)
|
|
1415
1634
|
else:
|
|
1416
|
-
hit = any(
|
|
1635
|
+
hit = any(binds_key(p, number) for p in pieces)
|
|
1417
1636
|
if hit:
|
|
1418
1637
|
found.append((number, top))
|
|
1419
1638
|
continue
|
|
1420
1639
|
top = _is_column_0(text)
|
|
1421
|
-
|
|
1640
|
+
# every rule that matches a line needs the name in it
|
|
1641
|
+
named = name in code
|
|
1642
|
+
if named and is_statement(code, number):
|
|
1422
1643
|
found.append((number, top))
|
|
1423
1644
|
continue
|
|
1424
1645
|
m = _JS_VAR_DECL_RE.match(code)
|
|
@@ -1426,7 +1647,7 @@ def _tsjs_definition_lines(
|
|
|
1426
1647
|
carry, pieces = declarators(m.group(1))
|
|
1427
1648
|
if carry is not None:
|
|
1428
1649
|
decls.append((top, *carry))
|
|
1429
|
-
if any(
|
|
1650
|
+
if named and any(declares(p, number) for p in pieces):
|
|
1430
1651
|
# a later declarator: `export const a = 1, NAME = 2;`
|
|
1431
1652
|
found.append((number, top))
|
|
1432
1653
|
continue
|
|
@@ -1439,7 +1660,7 @@ def _tsjs_definition_lines(
|
|
|
1439
1660
|
depth, pieces = _js_depth1_pieces(m.group(1), 1)
|
|
1440
1661
|
if depth > 0:
|
|
1441
1662
|
cont = ("list", top, depth)
|
|
1442
|
-
if any(_js_export_item_binds(p, name) for p in pieces):
|
|
1663
|
+
if any(_js_export_item_binds(p, name, declarations_only) for p in pieces):
|
|
1443
1664
|
found.append((number, top))
|
|
1444
1665
|
continue
|
|
1445
1666
|
m = _JS_MODULE_EXPORTS_OBJECT_RE.match(code)
|
|
@@ -1447,7 +1668,7 @@ def _tsjs_definition_lines(
|
|
|
1447
1668
|
depth, pieces = _js_depth1_pieces(m.group(1), 1)
|
|
1448
1669
|
if depth > 0:
|
|
1449
1670
|
cont = ("object", top, depth)
|
|
1450
|
-
if any(
|
|
1671
|
+
if any(binds_key(p, number) for p in pieces):
|
|
1451
1672
|
found.append((number, top))
|
|
1452
1673
|
continue
|
|
1453
1674
|
if not dotted:
|
|
@@ -1494,12 +1715,19 @@ def _definition_matches(
|
|
|
1494
1715
|
export_name: str,
|
|
1495
1716
|
source_root: Path,
|
|
1496
1717
|
cache: _SourceCache | None = None,
|
|
1718
|
+
declarations_only: bool = False,
|
|
1497
1719
|
) -> tuple[list[int], set[int]] | None:
|
|
1498
1720
|
"""(definition lines to report, every matching line) for an export.
|
|
1499
1721
|
|
|
1500
1722
|
The first item applies the column-0 rule; the second keeps the indented
|
|
1501
1723
|
matches that rule dropped. None when the file's extension has no
|
|
1502
1724
|
definition rule or the file cannot be read.
|
|
1725
|
+
|
|
1726
|
+
`declarations_only` (classify-stale's `defined_at` lookup) keeps the
|
|
1727
|
+
lines that declare the name at column 0, both items alike: no plain
|
|
1728
|
+
import, star import or re-export (`_python_definition_lines` and
|
|
1729
|
+
`_tsjs_definition_lines` say which), and nothing in a Python file that
|
|
1730
|
+
does not tokenize.
|
|
1503
1731
|
"""
|
|
1504
1732
|
rel = source_file.strip().replace("\\", "/")
|
|
1505
1733
|
suffix = Path(rel).suffix.lower()
|
|
@@ -1510,18 +1738,39 @@ def _definition_matches(
|
|
|
1510
1738
|
lines = cache.lines(path)
|
|
1511
1739
|
if lines is None:
|
|
1512
1740
|
return None
|
|
1741
|
+
return _line_matches(suffix, path, lines, export_name, cache, declarations_only)
|
|
1742
|
+
|
|
1743
|
+
|
|
1744
|
+
def _line_matches(
|
|
1745
|
+
suffix: str,
|
|
1746
|
+
path: Path,
|
|
1747
|
+
lines: list[str],
|
|
1748
|
+
export_name: str,
|
|
1749
|
+
cache: _SourceCache,
|
|
1750
|
+
declarations_only: bool = False,
|
|
1751
|
+
) -> tuple[list[int], set[int]]:
|
|
1752
|
+
"""`_definition_matches` over the lines of a Python or TS/JS file already
|
|
1753
|
+
read: `suffix`, the cited file's extension, picks the rules, and `path`
|
|
1754
|
+
keys `cache`."""
|
|
1513
1755
|
name = definition_name(export_name)
|
|
1514
1756
|
if not name:
|
|
1515
1757
|
return [], set()
|
|
1516
1758
|
dotted = "." in export_name
|
|
1517
1759
|
if suffix in PYTHON_EXTENSIONS:
|
|
1518
1760
|
starts = cache.py_statement_starts(path, lines)
|
|
1519
|
-
|
|
1761
|
+
if declarations_only and starts is None:
|
|
1762
|
+
# every line would be a candidate, a docstring line included
|
|
1763
|
+
return [], set()
|
|
1764
|
+
found = _python_definition_lines(lines, name, starts, declarations_only)
|
|
1520
1765
|
else:
|
|
1521
1766
|
code_lines = cache.js_code_lines(path, lines)
|
|
1522
1767
|
found = _tsjs_definition_lines(
|
|
1523
|
-
lines, code_lines, name, dotted, cache.js_declarators
|
|
1768
|
+
lines, code_lines, name, dotted, cache.js_declarators, declarations_only,
|
|
1769
|
+
cache.js_quiet_lines(path, code_lines),
|
|
1524
1770
|
)
|
|
1771
|
+
if declarations_only:
|
|
1772
|
+
top = {n for n, at_column_0 in found if at_column_0}
|
|
1773
|
+
return sorted(top), top
|
|
1525
1774
|
return _prefer_module_level(found, dotted), {n for n, _ in found}
|
|
1526
1775
|
|
|
1527
1776
|
|
|
@@ -1647,14 +1896,657 @@ def _stale_line_check(
|
|
|
1647
1896
|
return report["line_check"], report["definition_lines"]
|
|
1648
1897
|
|
|
1649
1898
|
|
|
1650
|
-
|
|
1899
|
+
# The folders and files the `defined_at` walk leaves out, judged
|
|
1900
|
+
# case-insensitively on the path below its walk root: generated, built and
|
|
1901
|
+
# installed output, a vendored copy of another package, tests and their
|
|
1902
|
+
# configuration, docs, examples and benchmarks.
|
|
1903
|
+
# Keep identical to EXCLUDED_DIR_NAMES in skf-resolve-authoritative-files.py.
|
|
1904
|
+
EXCLUDED_DIR_NAMES = {
|
|
1905
|
+
"node_modules", "__pycache__", "dist", "build", ".webpack",
|
|
1906
|
+
"target", ".next", ".nuxt", "out", "coverage", ".git",
|
|
1907
|
+
".venv", "venv", ".tox", ".mypy_cache", ".pytest_cache",
|
|
1908
|
+
".ruff_cache", ".gradle", ".idea", ".vscode",
|
|
1909
|
+
}
|
|
1910
|
+
# Keep identical to _VENDORED_SEGMENTS in skf-detect-language.py.
|
|
1911
|
+
_VENDORED_SEGMENTS = frozenset(
|
|
1912
|
+
{"vendor", "vendored", "third_party", "third-party", "thirdparty"}
|
|
1913
|
+
)
|
|
1914
|
+
# Keep identical to TEST_FOLDERS in skf-disqualify-candidates.py.
|
|
1915
|
+
TEST_FOLDERS = frozenset({"test", "tests", "__tests__", "spec", "specs"})
|
|
1916
|
+
# Keep identical to TEST_CONFIG_NAMES in skf-disqualify-candidates.py.
|
|
1917
|
+
TEST_CONFIG_NAMES = frozenset({
|
|
1918
|
+
"conftest.py", "pytest.ini", "phpunit.xml", "phpunit.xml.dist", ".rspec",
|
|
1919
|
+
})
|
|
1920
|
+
# Keep identical to TEST_CONFIG_PREFIXES in skf-disqualify-candidates.py.
|
|
1921
|
+
TEST_CONFIG_PREFIXES = (
|
|
1922
|
+
"jest.config.", "vitest.config.", "karma.conf.", "playwright.config.",
|
|
1923
|
+
"cypress.config.",
|
|
1924
|
+
)
|
|
1925
|
+
# Keep identical to TEST_FILE_RE in skf-disqualify-candidates.py.
|
|
1926
|
+
TEST_FILE_RE = re.compile(
|
|
1927
|
+
r"(\.(test|spec)\.[^./]+$)|(_test\.[^./]+$)|(^test_.+\.py$)|(_spec\.rb$)"
|
|
1928
|
+
)
|
|
1929
|
+
# Keep identical to PY_NON_CORE_TOPS in skf-extract-public-api.py.
|
|
1930
|
+
PY_NON_CORE_TOPS = frozenset({"docs", "examples", "benchmarks"})
|
|
1931
|
+
# The walk's own additions: pip's and setuptools' `_vendor`, test helpers
|
|
1932
|
+
# (`testing`, `fixtures`, `__mocks__`), installed packages (`site-packages`,
|
|
1933
|
+
# a virtualenv named `env`, `bower_components`) and the ESM, CommonJS and
|
|
1934
|
+
# UMD builds a JS package commits.
|
|
1935
|
+
LOOKUP_EXTRA_FOLDERS = frozenset({
|
|
1936
|
+
"_vendor", "testing", "fixtures", "__mocks__", "site-packages", "env",
|
|
1937
|
+
"bower_components", "esm", "cjs", "umd",
|
|
1938
|
+
})
|
|
1939
|
+
LOOKUP_SKIPPED_FOLDERS = frozenset(
|
|
1940
|
+
EXCLUDED_DIR_NAMES | _VENDORED_SEGMENTS | TEST_FOLDERS | PY_NON_CORE_TOPS
|
|
1941
|
+
| LOOKUP_EXTRA_FOLDERS
|
|
1942
|
+
)
|
|
1943
|
+
# The test folders the walk alone skips wherever they lie (end-to-end tests,
|
|
1944
|
+
# Jest's `__fixtures__` beside the code): a file the map cites in one is
|
|
1945
|
+
# still read.
|
|
1946
|
+
LOOKUP_WALK_FOLDERS = frozenset({"e2e", "__fixtures__"})
|
|
1947
|
+
# The folders the walk alone skips as the first folder below its walk root
|
|
1948
|
+
# or below a package root it passes through (`_package_root`'s roots):
|
|
1949
|
+
# examples and demos, samples, scripts, playgrounds, benchmarks and
|
|
1950
|
+
# migrations, which hold code that uses or changes the package rather than
|
|
1951
|
+
# its API. Deeper in a package such a folder can be live package code
|
|
1952
|
+
# (cognee's `cognee/modules/migrations/`), so it is walked there; one that
|
|
1953
|
+
# holds a file the map cites is walked too, and that file is still read.
|
|
1954
|
+
LOOKUP_TOP_FOLDERS = frozenset({
|
|
1955
|
+
"example", "demo", "demos", "sample", "samples", "scripts", "playground",
|
|
1956
|
+
"bench", "benchmark", "migrations",
|
|
1957
|
+
})
|
|
1958
|
+
# The kinds of a documented inventory (validate-inventory.py's `kind`) that a
|
|
1959
|
+
# module cannot be: such a name takes no line 1 from a module so named. A
|
|
1960
|
+
# `method` is left out: reconcile-coverage.py never lists a method as stale,
|
|
1961
|
+
# and a module may share a method's name.
|
|
1962
|
+
NO_MODULE_KINDS = frozenset({"function", "hook", "class", "interface", "enum", "type"})
|
|
1963
|
+
# A TS/JS file that is itself the export: its module so named declares the
|
|
1964
|
+
# name whatever its kind.
|
|
1965
|
+
_SELF_EXPORT_RE = re.compile(r"^(?:export\s+default\b|module\.exports\s*=(?!=))", re.M)
|
|
1966
|
+
# A minified bundle: built output, whatever folder holds it.
|
|
1967
|
+
LOOKUP_SKIPPED_SUFFIXES = (".min.js",)
|
|
1968
|
+
# The errors a lookup gives up on: every `defined_at` is then null.
|
|
1969
|
+
_LOOKUP_ERRORS = (OSError, ValueError, RuntimeError)
|
|
1970
|
+
|
|
1971
|
+
|
|
1972
|
+
def _rule_family(path: str) -> str | None:
|
|
1973
|
+
"""The definition rules a file's extension picks: `python`, `tsjs` or
|
|
1974
|
+
None."""
|
|
1975
|
+
suffix = posixpath.splitext(path)[1].lower()
|
|
1976
|
+
if suffix in PYTHON_EXTENSIONS:
|
|
1977
|
+
return "python"
|
|
1978
|
+
return "tsjs" if suffix in TSJS_EXTENSIONS else None
|
|
1979
|
+
|
|
1980
|
+
|
|
1981
|
+
def _lookup_skips_folder(segment: str) -> bool:
|
|
1982
|
+
"""True when the `defined_at` walk never goes into this folder."""
|
|
1983
|
+
segment = segment.lower()
|
|
1984
|
+
return segment.startswith(".") or segment in LOOKUP_SKIPPED_FOLDERS
|
|
1985
|
+
|
|
1986
|
+
|
|
1987
|
+
def _lookup_skips_file(filename: str) -> bool:
|
|
1988
|
+
"""True when the `defined_at` lookup never reads this file: a test file,
|
|
1989
|
+
as skf-disqualify-candidates.py's `_is_test_file` judges its name (a
|
|
1990
|
+
test configuration file included), or a minified bundle."""
|
|
1991
|
+
name = filename.lower()
|
|
1992
|
+
return (
|
|
1993
|
+
name in TEST_CONFIG_NAMES
|
|
1994
|
+
or name.startswith(TEST_CONFIG_PREFIXES)
|
|
1995
|
+
or TEST_FILE_RE.search(name) is not None
|
|
1996
|
+
or name.endswith(LOOKUP_SKIPPED_SUFFIXES)
|
|
1997
|
+
)
|
|
1998
|
+
|
|
1999
|
+
|
|
2000
|
+
def _symlinked(root: Path, parts: tuple[str, ...]) -> bool:
|
|
2001
|
+
"""True when a path below `root` goes through a symlink, itself
|
|
2002
|
+
included."""
|
|
2003
|
+
return any(root.joinpath(*parts[:i]).is_symlink() for i in range(1, len(parts) + 1))
|
|
2004
|
+
|
|
2005
|
+
|
|
2006
|
+
def _readable_below(root: Path, parts: tuple[str, ...], real_root: Path | None = None) -> bool:
|
|
2007
|
+
"""True when the `defined_at` lookup may read the file at `parts` below
|
|
2008
|
+
`root`: not a file it skips (`_lookup_skips_file`), no symlink on its
|
|
2009
|
+
path, and its real path a file inside the root (`real_root`, the root
|
|
2010
|
+
resolved)."""
|
|
2011
|
+
if _lookup_skips_file(parts[-1]) or _symlinked(root, parts):
|
|
2012
|
+
return False
|
|
2013
|
+
real = root.joinpath(*parts).resolve()
|
|
2014
|
+
return real.is_relative_to(real_root or root.resolve()) and real.is_file()
|
|
2015
|
+
|
|
2016
|
+
|
|
2017
|
+
def _cited_parts(source_file: str) -> tuple[str, ...] | None:
|
|
2018
|
+
"""A recorded `source_file` as path segments below the source root, None
|
|
2019
|
+
when it is absolute or climbs out of the root."""
|
|
2020
|
+
path = normalize_path(source_file)
|
|
2021
|
+
if not path or path == "." or posixpath.isabs(path):
|
|
2022
|
+
return None
|
|
2023
|
+
parts = tuple(path.split("/"))
|
|
2024
|
+
return None if parts[0] == ".." or ":" in parts[0] else parts
|
|
2025
|
+
|
|
2026
|
+
|
|
2027
|
+
# The file whose folder is a package, by rule family.
|
|
2028
|
+
_PACKAGE_MARKERS = {"python": "__init__.py", "tsjs": "package.json"}
|
|
2029
|
+
# Type stubs: a declaration file beside the module it types.
|
|
2030
|
+
_STUB_SUFFIXES = {".pyi": "python", ".d.ts": "tsjs", ".d.mts": "tsjs", ".d.cts": "tsjs"}
|
|
2031
|
+
|
|
2032
|
+
|
|
2033
|
+
def _stub_of(path: str) -> tuple[str, str] | None:
|
|
2034
|
+
"""(the path without its suffix, family) of a type stub (`NAME.pyi`,
|
|
2035
|
+
`NAME.d.ts`), None for any other file."""
|
|
2036
|
+
lower = path.lower()
|
|
2037
|
+
for suffix, family in _STUB_SUFFIXES.items():
|
|
2038
|
+
if lower.endswith(suffix):
|
|
2039
|
+
return path[: -len(suffix)], family
|
|
2040
|
+
return None
|
|
2041
|
+
|
|
2042
|
+
|
|
2043
|
+
def _without_stubs(hits: list[str]) -> list[str]:
|
|
2044
|
+
"""`file:line` items less each stub whose implementation (`NAME.py`
|
|
2045
|
+
beside `NAME.pyi`, a TS/JS `NAME.<ext>` beside `NAME.d.ts`) is an item
|
|
2046
|
+
too: the two are one declaring file, the implementation."""
|
|
2047
|
+
files = [h.rsplit(":", 1)[0] for h in hits]
|
|
2048
|
+
implementations = {
|
|
2049
|
+
(posixpath.splitext(f)[0], _rule_family(f))
|
|
2050
|
+
for f in files
|
|
2051
|
+
if _stub_of(f) is None and _rule_family(f) is not None
|
|
2052
|
+
}
|
|
2053
|
+
return [h for h, f in zip(hits, files) if _stub_of(f) not in implementations]
|
|
2054
|
+
|
|
2055
|
+
|
|
2056
|
+
def _module_name(parts: tuple[str, ...]) -> str | None:
|
|
2057
|
+
"""The name a file is the module or package of, which it declares at
|
|
2058
|
+
its line 1: `NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
|
|
2059
|
+
`NAME/index.<ext>`. None for any other file, and for a package file
|
|
2060
|
+
with no folder above it below the source root."""
|
|
2061
|
+
stem, ext = posixpath.splitext(parts[-1])
|
|
2062
|
+
ext = ext.lower()
|
|
2063
|
+
if ext == ".py":
|
|
2064
|
+
package = "__init__"
|
|
2065
|
+
elif ext in TSJS_EXTENSIONS:
|
|
2066
|
+
package = "index"
|
|
2067
|
+
else:
|
|
2068
|
+
return None
|
|
2069
|
+
if stem != package:
|
|
2070
|
+
return stem
|
|
2071
|
+
return parts[-2] if len(parts) > 1 else None
|
|
2072
|
+
|
|
2073
|
+
|
|
2074
|
+
def _exports_itself(parts: tuple[str, ...], text: str) -> bool:
|
|
2075
|
+
"""True for a TS/JS file that is itself the export, with a column-0
|
|
2076
|
+
`export default` or `module.exports =`: its module so named declares
|
|
2077
|
+
the name whatever kind the inventory gives it."""
|
|
2078
|
+
return _rule_family(parts[-1]) == "tsjs" and _SELF_EXPORT_RE.search(text) is not None
|
|
2079
|
+
|
|
2080
|
+
|
|
2081
|
+
def declared_line(
|
|
2082
|
+
source_file: str | tuple[str, ...],
|
|
2083
|
+
name: str,
|
|
2084
|
+
source_root: Path | None = None,
|
|
2085
|
+
*,
|
|
2086
|
+
lines: list[str] | None = None,
|
|
2087
|
+
cache: _SourceCache | None = None,
|
|
2088
|
+
module_fallback: bool = True,
|
|
2089
|
+
) -> int | None:
|
|
2090
|
+
"""The line at which one file declares `name`, by classify-stale's
|
|
2091
|
+
`defined_at` rule (see the module docstring): its first line at column
|
|
2092
|
+
0 that the definition-line rules match with imports left out (a `def`,
|
|
2093
|
+
`class`, assignment, annotation, type alias, TS/JS declaration, or an
|
|
2094
|
+
alias that renames, such as `from X import Y as NAME`; a plain or star
|
|
2095
|
+
import, an `as` that keeps the imported name, a re-export and an
|
|
2096
|
+
import-valued binding never count), else, with `module_fallback`, its
|
|
2097
|
+
line 1 when it is the module or package so named (`_module_name`).
|
|
2098
|
+
|
|
2099
|
+
`source_file` is the file's path below the source root, as recorded
|
|
2100
|
+
(one that is absolute or climbs out of the root gives None) or as its
|
|
2101
|
+
segments. `lines` are its lines when the caller read them already;
|
|
2102
|
+
otherwise the file is read below `source_root` under the lookup's
|
|
2103
|
+
guards (`_readable_below`: no symlink on its path, its real path
|
|
2104
|
+
inside the root, not a file the lookup skips). Pass one `cache` for
|
|
2105
|
+
the names of one source root, and each file is read and parsed once.
|
|
2106
|
+
None when no line declares the name, the name is empty or dotted, the
|
|
2107
|
+
file is not a Python or TS/JS one, or it cannot be read.
|
|
2108
|
+
"""
|
|
2109
|
+
parts = _cited_parts(source_file) if isinstance(source_file, str) else tuple(source_file)
|
|
2110
|
+
if not name or "." in name or not parts or _rule_family(parts[-1]) is None:
|
|
2111
|
+
return None
|
|
2112
|
+
cache = cache if cache is not None else _SourceCache()
|
|
2113
|
+
if lines is None:
|
|
2114
|
+
if source_root is None:
|
|
2115
|
+
raise ValueError("declared_line reads the file below source_root when no lines are given")
|
|
2116
|
+
if not _readable_below(source_root, parts):
|
|
2117
|
+
return None
|
|
2118
|
+
lines = cache.lines(source_root.joinpath(*parts))
|
|
2119
|
+
if lines is None:
|
|
2120
|
+
return None
|
|
2121
|
+
path = Path(*parts)
|
|
2122
|
+
top, _ = _line_matches(path.suffix.lower(), path, lines, name, cache, True)
|
|
2123
|
+
if top:
|
|
2124
|
+
return top[0]
|
|
2125
|
+
return 1 if module_fallback and _module_name(parts) == name else None
|
|
2126
|
+
|
|
2127
|
+
|
|
2128
|
+
# declared_line's `cache`, public for a script that loads this one.
|
|
2129
|
+
SourceCache = _SourceCache
|
|
2130
|
+
|
|
2131
|
+
|
|
2132
|
+
def _baseline_path(path: str) -> str:
|
|
2133
|
+
"""A cited path as an extraction writes it: forward slashes, no
|
|
2134
|
+
leading `./`."""
|
|
2135
|
+
rel = path.strip().replace("\\", "/")
|
|
2136
|
+
while rel.startswith("./"):
|
|
2137
|
+
rel = rel[2:]
|
|
2138
|
+
return rel
|
|
2139
|
+
|
|
2140
|
+
|
|
2141
|
+
def _declaration_refused(source_root: Path, file: str) -> bool:
|
|
2142
|
+
"""True when `declared_line` would not read a file that is there: a
|
|
2143
|
+
path outside the source root, a file the lookup skips (a test file, a
|
|
2144
|
+
minified bundle) or one with a symlink on its path. A file that is gone
|
|
2145
|
+
is not refused: its entries are deleted."""
|
|
2146
|
+
parts = _cited_parts(file)
|
|
2147
|
+
if parts is None:
|
|
2148
|
+
return True
|
|
2149
|
+
path = source_root.joinpath(*parts)
|
|
2150
|
+
return os.path.lexists(path) and not _readable_below(source_root, parts)
|
|
2151
|
+
|
|
2152
|
+
|
|
2153
|
+
def baseline_gaps(
|
|
2154
|
+
provenance: dict,
|
|
2155
|
+
source_root: Path,
|
|
2156
|
+
held: set[tuple[str, str]],
|
|
2157
|
+
files: set[str],
|
|
2158
|
+
listed: set[tuple] | frozenset = frozenset(),
|
|
2159
|
+
*,
|
|
2160
|
+
cache: _SourceCache | None = None,
|
|
2161
|
+
) -> tuple[list[dict], list[dict]]:
|
|
2162
|
+
"""The provenance entries an extraction left out whose cited file still
|
|
2163
|
+
declares them: the recipe runner never reports a module, an alias that
|
|
2164
|
+
renames, a dunder such as `__version__` or another underscore name, so
|
|
2165
|
+
a diff would read each as removed.
|
|
2166
|
+
|
|
2167
|
+
`held` holds the (name, file) of every export the extraction found,
|
|
2168
|
+
`files` the files whose entries are checked (the ones read), and
|
|
2169
|
+
`listed` the (name, file) of the gaps already listed elsewhere (the
|
|
2170
|
+
runner's own extraction gaps), each path with forward slashes. An
|
|
2171
|
+
entry is left out when its name is dotted (a member) and it is no
|
|
2172
|
+
`module` or `package` entry, when its file is not in `files`, when the
|
|
2173
|
+
extraction holds the name or its re-export target (`reexport_map`,
|
|
2174
|
+
else the entry's `reexported_as`) at that file, or when it is listed
|
|
2175
|
+
already. Each name and file is taken once.
|
|
2176
|
+
|
|
2177
|
+
Returns (gaps, unchecked). A gap is {"name", "file", "line", "entry":
|
|
2178
|
+
null}, plus the entry's `export_type` when it has one, at the line
|
|
2179
|
+
`declared_line` gives (only a `module` or `package` entry takes the
|
|
2180
|
+
module rule); an entry whose file declares it no more is no gap.
|
|
2181
|
+
`unchecked` holds {"name", "file", "export_type"} for each entry no
|
|
2182
|
+
rule can check, to read by eye: one whose file is neither Python nor
|
|
2183
|
+
TS/JS, one whose file the lookup will not read (outside the source
|
|
2184
|
+
root, a test file or a minified bundle, a symlink on its path), and a
|
|
2185
|
+
dotted `module` or `package` entry (`api.v1`). The entries are read
|
|
2186
|
+
file by file over one cache, so each file is read and parsed once."""
|
|
2187
|
+
checkable = PYTHON_EXTENSIONS | TSJS_EXTENSIONS
|
|
2188
|
+
renamed = extract_reexport_map(provenance)
|
|
2189
|
+
seen = set(listed)
|
|
2190
|
+
by_file: dict[str, list[tuple[str, object]]] = {}
|
|
2191
|
+
unchecked: list[dict] = []
|
|
2192
|
+
entries = provenance.get("entries")
|
|
2193
|
+
for entry in entries if isinstance(entries, list) else []:
|
|
2194
|
+
if not isinstance(entry, dict):
|
|
2195
|
+
continue
|
|
2196
|
+
name, cited = entry.get("export_name"), entry.get("source_file")
|
|
2197
|
+
if not isinstance(name, str) or not isinstance(cited, str):
|
|
2198
|
+
continue
|
|
2199
|
+
name, file, kind = name.strip(), _baseline_path(cited), entry.get("export_type")
|
|
2200
|
+
module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
|
|
2201
|
+
if not name or ("." in name and not module) or (name, file) in seen:
|
|
2202
|
+
continue
|
|
2203
|
+
if cited.replace("\\", "/") not in files and file not in files:
|
|
2204
|
+
continue
|
|
2205
|
+
if (name, file) in held or (renamed.get(name), file) in held:
|
|
2206
|
+
continue
|
|
2207
|
+
seen.add((name, file))
|
|
2208
|
+
if "." in name or Path(file).suffix.lower() not in checkable or _declaration_refused(source_root, file):
|
|
2209
|
+
unchecked.append({"name": name, "file": file, "export_type": kind})
|
|
2210
|
+
continue
|
|
2211
|
+
by_file.setdefault(file, []).append((name, kind))
|
|
2212
|
+
cache = cache if cache is not None else _SourceCache()
|
|
2213
|
+
gaps = []
|
|
2214
|
+
for file, named in by_file.items():
|
|
2215
|
+
for name, kind in named:
|
|
2216
|
+
module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
|
|
2217
|
+
line = declared_line(file, name, source_root, cache=cache, module_fallback=module)
|
|
2218
|
+
if line is None:
|
|
2219
|
+
continue
|
|
2220
|
+
gap = {"name": name, "file": file, "line": line, "entry": None}
|
|
2221
|
+
if kind is not None:
|
|
2222
|
+
gap["export_type"] = kind
|
|
2223
|
+
gaps.append(gap)
|
|
2224
|
+
return gaps, unchecked
|
|
2225
|
+
|
|
2226
|
+
|
|
2227
|
+
class _DeclarationLookup:
|
|
2228
|
+
"""Where the source declares the stale names (classify-stale's
|
|
2229
|
+
`defined_at`): a column-0 declaration, an alias that renames included, a
|
|
2230
|
+
plain or star import and a re-export never counting.
|
|
2231
|
+
|
|
2232
|
+
A module or package named after the name (`_module_name`) declares it,
|
|
2233
|
+
at its line 1 when no line of it does, unless `find` takes the name as
|
|
2234
|
+
`strict` and the file is not a TS/JS one that is itself the export
|
|
2235
|
+
(`_exports_itself`). A file the name's entries cite that declares it
|
|
2236
|
+
gives its first such line. Otherwise the walk lists every file that
|
|
2237
|
+
declares it (`declared_in`), and exactly one must: the walk reads,
|
|
2238
|
+
sorted, the files below the walk root (`_walk_root`), leaving out the
|
|
2239
|
+
folders `_lookup_skips_folder` names, the `LOOKUP_WALK_FOLDERS`, the
|
|
2240
|
+
`LOOKUP_TOP_FOLDERS` right below the walk root and a package root
|
|
2241
|
+
(`_is_package_root`) that hold no cited file, the files
|
|
2242
|
+
`_lookup_skips_file` names and every symlink. Only the rule families
|
|
2243
|
+
(Python, TS/JS) the entries cite are read (both when no entry cites a
|
|
2244
|
+
file). A cited file is read inside the source root only, with no symlink
|
|
2245
|
+
on its path and the same skips but the walk-only ones, judged below the
|
|
2246
|
+
walk root when it lies there. The walk runs once for all the names no
|
|
2247
|
+
cited file settles: each file is read once, and only a file whose text
|
|
2248
|
+
holds a name goes through the rules. Any error of the filesystem raises
|
|
2249
|
+
(`_LOOKUP_ERRORS`)."""
|
|
2250
|
+
|
|
2251
|
+
def __init__(self, entries: list[dict], source_root: Path) -> None:
|
|
2252
|
+
self.root = source_root
|
|
2253
|
+
self.real_root = source_root.resolve()
|
|
2254
|
+
recorded = [f for f in map(_entry_file, entries) if f is not None]
|
|
2255
|
+
# a map that cites no file names no language: both families
|
|
2256
|
+
families = {_rule_family(f) for f in recorded}
|
|
2257
|
+
self.families = families - {None} if recorded else {"python", "tsjs"}
|
|
2258
|
+
self.folder = self._walk_root(recorded)
|
|
2259
|
+
# the folders that hold a cited file: a LOOKUP_TOP_FOLDERS one is walked
|
|
2260
|
+
self._cited_folders = {
|
|
2261
|
+
parts[:i]
|
|
2262
|
+
for parts in filter(None, map(_cited_parts, recorded))
|
|
2263
|
+
for i in range(1, len(parts))
|
|
2264
|
+
}
|
|
2265
|
+
# the cited files read, kept for the walk: a few files at most
|
|
2266
|
+
self._cited: dict[tuple[str, ...], list[str] | None] = {}
|
|
2267
|
+
self._cited_cache = _SourceCache()
|
|
2268
|
+
|
|
2269
|
+
# -- the walk root ------------------------------------------------------
|
|
2270
|
+
|
|
2271
|
+
def _symlinked(self, parts: tuple[str, ...]) -> bool:
|
|
2272
|
+
"""True when a path below the source root goes through a symlink,
|
|
2273
|
+
itself included."""
|
|
2274
|
+
return _symlinked(self.root, parts)
|
|
2275
|
+
|
|
2276
|
+
def _real_folder(self, parts: tuple[str, ...]) -> tuple[str, ...]:
|
|
2277
|
+
"""The deepest folder of a path below the source root that is one,
|
|
2278
|
+
with no symlink on its path."""
|
|
2279
|
+
while parts and (
|
|
2280
|
+
self._symlinked(parts) or not self.root.joinpath(*parts).is_dir()
|
|
2281
|
+
):
|
|
2282
|
+
parts = parts[:-1]
|
|
2283
|
+
return parts
|
|
2284
|
+
|
|
2285
|
+
def _holds(self, folder: tuple[str, ...], filename: str) -> bool:
|
|
2286
|
+
return (self.root.joinpath(*folder) / filename).is_file()
|
|
2287
|
+
|
|
2288
|
+
def _package_root(
|
|
2289
|
+
self, folder: tuple[str, ...], family: str
|
|
2290
|
+
) -> tuple[str, ...] | None:
|
|
2291
|
+
"""The root of the package a folder below the source root lies in,
|
|
2292
|
+
None when it lies in none. Python: the nearest folder at or above it
|
|
2293
|
+
that holds `__init__.py`, then up while the parent holds one, never
|
|
2294
|
+
the source root itself (a stray `__init__.py` there would make the
|
|
2295
|
+
whole tree one package). TS/JS: the nearest folder at or above it
|
|
2296
|
+
that holds `package.json`."""
|
|
2297
|
+
marker = _PACKAGE_MARKERS[family]
|
|
2298
|
+
python = family == "python"
|
|
2299
|
+
here = folder
|
|
2300
|
+
while not (self._holds(here, marker) and (here or not python)):
|
|
2301
|
+
if not here:
|
|
2302
|
+
return None
|
|
2303
|
+
here = here[:-1]
|
|
2304
|
+
if python:
|
|
2305
|
+
while len(here) > 1 and self._holds(here[:-1], marker):
|
|
2306
|
+
here = here[:-1]
|
|
2307
|
+
return here
|
|
2308
|
+
|
|
2309
|
+
def _walk_root(self, recorded: list[str]) -> tuple[str, ...]:
|
|
2310
|
+
"""The folder the walk reads: the deepest folder the entries' files
|
|
2311
|
+
share, widened to its package root, leaving out an entry outside
|
|
2312
|
+
every package (the source root when no entry lies in one)."""
|
|
2313
|
+
folders: list[tuple[str, ...]] = []
|
|
2314
|
+
widen: set[str] = set()
|
|
2315
|
+
for source_file in recorded:
|
|
2316
|
+
parts = _cited_parts(source_file)
|
|
2317
|
+
family = _rule_family(source_file) if parts else None
|
|
2318
|
+
if family is None:
|
|
2319
|
+
continue
|
|
2320
|
+
if self._package_root(self._real_folder(parts[:-1]), family) is None:
|
|
2321
|
+
continue # outside every package: a script, a docs page
|
|
2322
|
+
folders.append(parts[:-1])
|
|
2323
|
+
widen.add(family)
|
|
2324
|
+
if not folders:
|
|
2325
|
+
return ()
|
|
2326
|
+
shared = self._real_folder(tuple(os.path.commonprefix(folders)))
|
|
2327
|
+
roots = [self._package_root(shared, family) for family in sorted(widen)]
|
|
2328
|
+
return min((r for r in roots if r is not None), key=len, default=shared)
|
|
2329
|
+
|
|
2330
|
+
# -- reading ------------------------------------------------------------
|
|
2331
|
+
|
|
2332
|
+
def _below_walk_root(self, parts: tuple[str, ...]) -> tuple[str, ...]:
|
|
2333
|
+
depth = len(self.folder)
|
|
2334
|
+
return parts[depth:] if parts[:depth] == self.folder else parts
|
|
2335
|
+
|
|
2336
|
+
def _reads_cited(self, parts: tuple[str, ...]) -> bool:
|
|
2337
|
+
"""True when the lookup reads a cited file: one of a family the
|
|
2338
|
+
entries cite, in no skipped place, with no symlink on its path and
|
|
2339
|
+
inside the source root."""
|
|
2340
|
+
if _rule_family(parts[-1]) not in self.families:
|
|
2341
|
+
return False
|
|
2342
|
+
below = self._below_walk_root(parts)
|
|
2343
|
+
if any(_lookup_skips_folder(s) for s in below[:-1]):
|
|
2344
|
+
return False
|
|
2345
|
+
return _readable_below(self.root, parts, self.real_root)
|
|
2346
|
+
|
|
2347
|
+
def _cited_lines(self, parts: tuple[str, ...]) -> list[str] | None:
|
|
2348
|
+
if parts not in self._cited:
|
|
2349
|
+
readable = self._reads_cited(parts)
|
|
2350
|
+
path = self.root.joinpath(*parts)
|
|
2351
|
+
self._cited[parts] = _text_lines(_read_source(path)) if readable else None
|
|
2352
|
+
return self._cited[parts]
|
|
2353
|
+
|
|
2354
|
+
def _is_package_root(self, folder: tuple[str, ...], filenames: list[str]) -> bool:
|
|
2355
|
+
"""True when a walked folder (its `filenames`) is a package root of a
|
|
2356
|
+
family the walk reads, as `_package_root` finds one: TS/JS, a folder
|
|
2357
|
+
that holds `package.json`; Python, the top of an `__init__.py`
|
|
2358
|
+
chain, never the source root."""
|
|
2359
|
+
if "tsjs" in self.families and "package.json" in filenames:
|
|
2360
|
+
return True
|
|
2361
|
+
return (
|
|
2362
|
+
"python" in self.families and bool(folder) and "__init__.py" in filenames
|
|
2363
|
+
and (len(folder) == 1 or not self._holds(folder[:-1], "__init__.py"))
|
|
2364
|
+
)
|
|
2365
|
+
|
|
2366
|
+
def _tree(self) -> list[tuple[str, ...]]:
|
|
2367
|
+
"""The files the walk reads, as segments below the source root,
|
|
2368
|
+
sorted."""
|
|
2369
|
+
top = self.root.joinpath(*self.folder)
|
|
2370
|
+
found: list[tuple[str, ...]] = []
|
|
2371
|
+
|
|
2372
|
+
def fail(exc: OSError) -> None:
|
|
2373
|
+
raise exc
|
|
2374
|
+
|
|
2375
|
+
for dirpath, dirnames, filenames in os.walk(top, onerror=fail):
|
|
2376
|
+
here = Path(dirpath)
|
|
2377
|
+
inner = here.relative_to(top).parts
|
|
2378
|
+
below = self.folder + inner
|
|
2379
|
+
# a LOOKUP_TOP_FOLDERS name is skipped right below the walk root
|
|
2380
|
+
# and a package root, unless it holds a cited file
|
|
2381
|
+
root = not inner or self._is_package_root(below, filenames)
|
|
2382
|
+
dirnames[:] = [
|
|
2383
|
+
d for d in dirnames
|
|
2384
|
+
if not _lookup_skips_folder(d)
|
|
2385
|
+
and d.lower() not in LOOKUP_WALK_FOLDERS
|
|
2386
|
+
and not (
|
|
2387
|
+
root and d.lower() in LOOKUP_TOP_FOLDERS
|
|
2388
|
+
and (*below, d) not in self._cited_folders
|
|
2389
|
+
)
|
|
2390
|
+
and not (here / d).is_symlink()
|
|
2391
|
+
]
|
|
2392
|
+
for filename in filenames:
|
|
2393
|
+
path = here / filename
|
|
2394
|
+
if (
|
|
2395
|
+
_rule_family(filename) in self.families
|
|
2396
|
+
and not _lookup_skips_file(filename)
|
|
2397
|
+
and not path.is_symlink()
|
|
2398
|
+
and path.is_file()
|
|
2399
|
+
):
|
|
2400
|
+
found.append((*below, filename))
|
|
2401
|
+
return sorted(found)
|
|
2402
|
+
|
|
2403
|
+
@staticmethod
|
|
2404
|
+
def _first(
|
|
2405
|
+
parts: tuple[str, ...],
|
|
2406
|
+
lines: list[str],
|
|
2407
|
+
name: str,
|
|
2408
|
+
cache: _SourceCache,
|
|
2409
|
+
module_fallback: bool = True,
|
|
2410
|
+
) -> str | None:
|
|
2411
|
+
"""`file:line` of the first line of a file that declares `name`, else,
|
|
2412
|
+
with `module_fallback`, its line 1 when it is the module or package
|
|
2413
|
+
so named (`declared_line`)."""
|
|
2414
|
+
line = declared_line(
|
|
2415
|
+
parts, name, lines=lines, cache=cache, module_fallback=module_fallback
|
|
2416
|
+
)
|
|
2417
|
+
return f"{'/'.join(parts)}:{line}" if line is not None else None
|
|
2418
|
+
|
|
2419
|
+
# -- the answer ---------------------------------------------------------
|
|
2420
|
+
|
|
2421
|
+
def find(
|
|
2422
|
+
self, cited: dict[str, list[str]], strict: frozenset[str] = frozenset()
|
|
2423
|
+
) -> tuple[dict[str, str], dict[str, list[str]]]:
|
|
2424
|
+
"""For the names of `cited` (name -> the source_file values its
|
|
2425
|
+
entries record): name -> `file:line` for each one the source
|
|
2426
|
+
declares, and name -> the sorted `file:line` of every walked file
|
|
2427
|
+
that declares it, for each name the walk looked for (the names no
|
|
2428
|
+
cited file settles). A dotted name is never looked up, and a name of
|
|
2429
|
+
`strict` takes no line 1 from a module or package so named that is
|
|
2430
|
+
not itself the export (`_exports_itself`), in a cited file or in
|
|
2431
|
+
the walk."""
|
|
2432
|
+
found: dict[str, str] = {}
|
|
2433
|
+
if not self.families:
|
|
2434
|
+
return found, {}
|
|
2435
|
+
pending: list[str] = []
|
|
2436
|
+
for name, files in cited.items():
|
|
2437
|
+
if not name or "." in name:
|
|
2438
|
+
continue
|
|
2439
|
+
for parts in dict.fromkeys(filter(None, map(_cited_parts, files))):
|
|
2440
|
+
lines = self._cited_lines(parts)
|
|
2441
|
+
hit = None if lines is None else self._first(
|
|
2442
|
+
parts, lines, name, self._cited_cache,
|
|
2443
|
+
name not in strict or _exports_itself(parts, "\n".join(lines)),
|
|
2444
|
+
)
|
|
2445
|
+
if hit is not None:
|
|
2446
|
+
found[name] = hit
|
|
2447
|
+
break
|
|
2448
|
+
else:
|
|
2449
|
+
pending.append(name)
|
|
2450
|
+
declared, unsure = self._walk(pending, strict) if pending else ({}, set())
|
|
2451
|
+
# exactly one walked file must declare a name, and none it holds
|
|
2452
|
+
# in an untokenizable Python file may
|
|
2453
|
+
found.update({
|
|
2454
|
+
name: h[0] for name, h in declared.items()
|
|
2455
|
+
if len(h) == 1 and name not in unsure
|
|
2456
|
+
})
|
|
2457
|
+
return found, declared
|
|
2458
|
+
|
|
2459
|
+
def _walk(
|
|
2460
|
+
self, names: list[str], strict: frozenset[str] = frozenset()
|
|
2461
|
+
) -> tuple[dict[str, list[str]], set[str]]:
|
|
2462
|
+
"""name -> the sorted `file:line` of each walked file that declares
|
|
2463
|
+
it, at the file's first declaration (a stub beside its
|
|
2464
|
+
implementation left out: `_without_stubs`; a module so named
|
|
2465
|
+
declaring no name of `strict` unless it is the export), and the
|
|
2466
|
+
names a Python file that
|
|
2467
|
+
does not tokenize holds, which it may declare."""
|
|
2468
|
+
hits: dict[str, list[str]] = {name: [] for name in names}
|
|
2469
|
+
unsure: set[str] = set()
|
|
2470
|
+
for parts in self._tree():
|
|
2471
|
+
module = _module_name(parts)
|
|
2472
|
+
lines = self._cited.get(parts)
|
|
2473
|
+
text = (
|
|
2474
|
+
_read_source(self.root.joinpath(*parts))
|
|
2475
|
+
if lines is None
|
|
2476
|
+
else "\n".join(lines) # a cited file, read already
|
|
2477
|
+
)
|
|
2478
|
+
# a strict name's module so named counts when it is the export
|
|
2479
|
+
fallback = module not in strict or _exports_itself(parts, text)
|
|
2480
|
+
# the substring test spares the rules most files and names
|
|
2481
|
+
here = sorted(
|
|
2482
|
+
n for n in names if n in text or (n == module and fallback)
|
|
2483
|
+
)
|
|
2484
|
+
if not here:
|
|
2485
|
+
continue
|
|
2486
|
+
if lines is None:
|
|
2487
|
+
lines = _text_lines(text)
|
|
2488
|
+
cache = _SourceCache() # this file's rules only: nothing kept
|
|
2489
|
+
path = Path(*parts)
|
|
2490
|
+
if path.suffix.lower() in PYTHON_EXTENSIONS and (
|
|
2491
|
+
cache.py_statement_starts(path, lines) is None
|
|
2492
|
+
):
|
|
2493
|
+
unsure.update(n for n in here if n in text)
|
|
2494
|
+
for name in here:
|
|
2495
|
+
hit = self._first(parts, lines, name, cache, fallback)
|
|
2496
|
+
if hit is not None:
|
|
2497
|
+
hits[name].append(hit)
|
|
2498
|
+
return {name: sorted(_without_stubs(h)) for name, h in hits.items()}, unsure
|
|
2499
|
+
|
|
2500
|
+
|
|
2501
|
+
def inventory_kinds(data: object, path: str) -> dict[str, frozenset[str]]:
|
|
2502
|
+
"""name -> the kinds a documented inventory gives it (`--inventory`):
|
|
2503
|
+
validate-inventory.py's `--output` file, `{"exports": [...]}`, or its
|
|
2504
|
+
whole result, the inventory under `inventory`. An item without a string
|
|
2505
|
+
`name` and `kind` is left out, and a kind is kept in lower case. Raises
|
|
2506
|
+
ValueError when no `exports` list is there."""
|
|
2507
|
+
if isinstance(data, dict) and isinstance(data.get("inventory"), dict):
|
|
2508
|
+
data = data["inventory"] # validate-inventory.py's whole result
|
|
2509
|
+
if not isinstance(data, dict) or not isinstance(data.get("exports"), list):
|
|
2510
|
+
raise ValueError(f"--inventory {path} holds no `exports` list")
|
|
2511
|
+
kinds: dict[str, set[str]] = {}
|
|
2512
|
+
for item in data["exports"]:
|
|
2513
|
+
if not isinstance(item, dict):
|
|
2514
|
+
continue
|
|
2515
|
+
name, kind = item.get("name"), item.get("kind")
|
|
2516
|
+
if isinstance(name, str) and isinstance(kind, str):
|
|
2517
|
+
kinds.setdefault(name, set()).add(kind.strip().lower())
|
|
2518
|
+
return {name: frozenset(k) for name, k in kinds.items()}
|
|
2519
|
+
|
|
2520
|
+
|
|
2521
|
+
def _module_entry(entry: dict) -> bool:
|
|
2522
|
+
"""True for a `module` or `package` entry."""
|
|
2523
|
+
kind = entry.get("export_type")
|
|
2524
|
+
return isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
|
|
2525
|
+
|
|
2526
|
+
|
|
2527
|
+
def classify_stale(
|
|
2528
|
+
names: list[str],
|
|
2529
|
+
prov: dict,
|
|
2530
|
+
source_root: Path,
|
|
2531
|
+
kinds: dict[str, frozenset[str]] | None = None,
|
|
2532
|
+
) -> list[dict]:
|
|
1651
2533
|
"""The fabricated-signature test for each stale name: the definition-lines
|
|
1652
|
-
rules at every provenance entry whose raw `export_name` is the name
|
|
1653
|
-
|
|
1654
|
-
`
|
|
2534
|
+
rules at every provenance entry whose raw `export_name` is the name, and
|
|
2535
|
+
for a name that is not fabricated, where the source declares it
|
|
2536
|
+
(`defined_at`, and `declared_in` when the walk looked for it). The
|
|
2537
|
+
classify-stale section of the module docstring gives each reason, what
|
|
2538
|
+
a `no-file` entry means, which entry gives `source` and how
|
|
2539
|
+
`defined_at` and `declared_in` are found. `kinds` (`inventory_kinds`,
|
|
2540
|
+
None without `--inventory`) keeps a name of a `NO_MODULE_KINDS` kind
|
|
2541
|
+
from a module or package so named, unless an entry of the name is a
|
|
2542
|
+
`module` or `package` one.
|
|
1655
2543
|
"""
|
|
1656
2544
|
entries = [e for e in prov.get("entries") or [] if isinstance(e, dict)]
|
|
1657
2545
|
results: list[dict] = []
|
|
2546
|
+
# the names to look up (never a fabricated one): name -> its cited files
|
|
2547
|
+
cited: dict[str, list[str]] = {}
|
|
2548
|
+
# the names a module so named does not declare
|
|
2549
|
+
strict: set[str] = set()
|
|
1658
2550
|
for name in names:
|
|
1659
2551
|
mine = [e for e in entries if e.get("export_name") == name]
|
|
1660
2552
|
checks: list[dict] = []
|
|
@@ -1683,15 +2575,33 @@ def classify_stale(names: list[str], prov: dict, source_root: Path) -> list[dict
|
|
|
1683
2575
|
else:
|
|
1684
2576
|
reason = STALE_UNCHECKED
|
|
1685
2577
|
source = next(filter(None, map(_entry_citation, mine)), None)
|
|
2578
|
+
fabricated = reason == STALE_NOT_DEFINED
|
|
2579
|
+
if not fabricated:
|
|
2580
|
+
cited[name] = [f for f in map(_entry_file, mine) if f is not None]
|
|
2581
|
+
documented = (kinds or {}).get(name, frozenset())
|
|
2582
|
+
if documented & NO_MODULE_KINDS and not any(map(_module_entry, mine)):
|
|
2583
|
+
strict.add(name)
|
|
1686
2584
|
results.append(
|
|
1687
2585
|
{
|
|
1688
2586
|
"name": name,
|
|
1689
|
-
"fabricated":
|
|
2587
|
+
"fabricated": fabricated,
|
|
1690
2588
|
"source": source,
|
|
2589
|
+
"defined_at": None,
|
|
2590
|
+
"declared_in": None,
|
|
1691
2591
|
"reason": reason,
|
|
1692
2592
|
"entries": checks,
|
|
1693
2593
|
}
|
|
1694
2594
|
)
|
|
2595
|
+
try:
|
|
2596
|
+
found, declared = _DeclarationLookup(entries, source_root).find(
|
|
2597
|
+
cited, frozenset(strict)
|
|
2598
|
+
)
|
|
2599
|
+
except _LOOKUP_ERRORS:
|
|
2600
|
+
# a lookup that fails proves nothing: every name stays null
|
|
2601
|
+
found, declared = {}, {}
|
|
2602
|
+
for item in results:
|
|
2603
|
+
item["defined_at"] = found.get(item["name"])
|
|
2604
|
+
item["declared_in"] = declared.get(item["name"])
|
|
1695
2605
|
return results
|
|
1696
2606
|
|
|
1697
2607
|
|
|
@@ -2898,8 +3808,12 @@ def _cmd_definition_lines(args: argparse.Namespace) -> int:
|
|
|
2898
3808
|
|
|
2899
3809
|
def _cmd_classify_stale(args: argparse.Namespace) -> int:
|
|
2900
3810
|
names_path, prov_path = Path(args.names), Path(args.provenance)
|
|
3811
|
+
inv_path = Path(args.inventory) if args.inventory is not None else None
|
|
2901
3812
|
root = Path(args.source_root)
|
|
2902
|
-
|
|
3813
|
+
inputs = [("names", names_path), ("provenance map", prov_path)]
|
|
3814
|
+
if inv_path is not None:
|
|
3815
|
+
inputs.append(("inventory", inv_path))
|
|
3816
|
+
for label, path in inputs:
|
|
2903
3817
|
if not path.is_file():
|
|
2904
3818
|
print(f"error: {label} not found: {path}", file=sys.stderr)
|
|
2905
3819
|
return 2
|
|
@@ -2919,11 +3833,14 @@ def _cmd_classify_stale(args: argparse.Namespace) -> int:
|
|
|
2919
3833
|
try:
|
|
2920
3834
|
names = stale_names(data)
|
|
2921
3835
|
prov = load_json_object(prov_path, "provenance map")
|
|
3836
|
+
kinds = None if inv_path is None else inventory_kinds(
|
|
3837
|
+
load_json_object(inv_path, "inventory"), args.inventory
|
|
3838
|
+
)
|
|
2922
3839
|
except ValueError as exc:
|
|
2923
3840
|
print(f"error: {exc}", file=sys.stderr)
|
|
2924
3841
|
return 2
|
|
2925
3842
|
try:
|
|
2926
|
-
result = classify_stale(names, prov, root)
|
|
3843
|
+
result = classify_stale(names, prov, root, kinds)
|
|
2927
3844
|
except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
|
|
2928
3845
|
return _unexpected("stale-name classification", exc)
|
|
2929
3846
|
return 0 if _emit(result, args.output) else 2
|
|
@@ -3225,8 +4142,8 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
3225
4142
|
p = sub.add_parser(
|
|
3226
4143
|
"classify-stale",
|
|
3227
4144
|
help=(
|
|
3228
|
-
"decide which stale names are fabricated signatures
|
|
3229
|
-
"file defines them"
|
|
4145
|
+
"decide which stale names are fabricated signatures (no cited "
|
|
4146
|
+
"file defines them) and where the source declares the others"
|
|
3230
4147
|
),
|
|
3231
4148
|
)
|
|
3232
4149
|
p.add_argument(
|
|
@@ -3245,6 +4162,17 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
3245
4162
|
required=True,
|
|
3246
4163
|
help="source tree root the entries' source_file paths are under",
|
|
3247
4164
|
)
|
|
4165
|
+
p.add_argument(
|
|
4166
|
+
"--inventory",
|
|
4167
|
+
default=None,
|
|
4168
|
+
help=(
|
|
4169
|
+
"validate-inventory.py's documented inventory: a name it gives a "
|
|
4170
|
+
"function, hook, class, interface, enum or type kind takes no "
|
|
4171
|
+
"line 1 from a module so named, unless a provenance map entry of "
|
|
4172
|
+
"the name is a module or package one or the module is a TS/JS "
|
|
4173
|
+
"file that is itself the export"
|
|
4174
|
+
),
|
|
4175
|
+
)
|
|
3248
4176
|
p.add_argument(
|
|
3249
4177
|
"-o",
|
|
3250
4178
|
"--output",
|