bmad-module-skill-forge 4.0.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +3 -3
- package/docs/_data/pinned.yaml +19 -13
- package/docs/agents.md +1 -1
- package/docs/architecture.md +5 -5
- package/docs/campaign.md +1 -1
- package/docs/concepts.md +4 -4
- package/docs/examples.md +32 -25
- package/docs/getting-started.md +3 -3
- package/docs/how-it-works.md +2 -2
- package/docs/index.md +1 -1
- package/docs/skill-model.md +41 -39
- package/docs/troubleshooting.md +9 -3
- package/docs/verifying-a-skill.md +42 -21
- package/docs/why-skf.md +1 -1
- package/docs/workflows.md +6 -6
- package/package.json +1 -1
- package/src/knowledge/tool-resolution.md +6 -6
- package/src/shared/references/headless-gate-convention.md +4 -1
- package/src/shared/references/output-contract-schema.md +3 -3
- package/src/shared/references/pipeline-contracts.md +2 -1
- package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +1 -1
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +11 -3
- package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
- package/src/shared/scripts/skf-build-change-manifest.py +1180 -189
- package/src/shared/scripts/skf-classify-changed-files.py +15 -10
- package/src/shared/scripts/skf-count-tokens.py +54 -10
- package/src/shared/scripts/skf-detect-workspaces.py +67 -0
- package/src/shared/scripts/skf-emit-result-envelope.py +64 -20
- package/src/shared/scripts/skf-extract-public-api.py +472 -120
- package/src/shared/scripts/skf-extraction-inventory.py +10 -6
- package/src/shared/scripts/skf-extraction-snapshot.py +6 -6
- package/src/shared/scripts/skf-parse-gaps.py +136 -26
- package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
- package/src/shared/scripts/skf-resolve-customization.py +197 -0
- package/src/shared/scripts/skf-shard-body.py +239 -41
- package/src/shared/scripts/skf-validate-output.py +12 -3
- package/src/shared/scripts/skf-verify-provenance-completeness.py +574 -45
- package/src/shared/tool-requirements.yaml +8 -2
- package/src/skf-analyze-source/SKILL.md +3 -2
- package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
- package/src/skf-analyze-source/references/generate-briefs.md +1 -1
- package/src/skf-analyze-source/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/SKILL.md +4 -3
- package/src/skf-audit-skill/references/re-index.md +2 -2
- package/src/skf-audit-skill/references/report.md +1 -1
- package/src/skf-brief-skill/SKILL.md +3 -2
- package/src/skf-brief-skill/assets/skill-brief-schema.md +2 -2
- package/src/skf-brief-skill/references/invocation-contract.md +2 -2
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-campaign/SKILL.md +3 -2
- package/src/skf-campaign/references/campaign-contracts.md +2 -0
- package/src/skf-campaign/references/step-05-skill-loop.md +8 -3
- package/src/skf-campaign/references/step-10-export.md +3 -2
- package/src/skf-create-skill/SKILL.md +4 -3
- package/src/skf-create-skill/assets/skill-sections.md +1 -1
- package/src/skf-create-skill/references/ccc-index-check.md +1 -1
- package/src/skf-create-skill/references/component-extraction.md +1 -1
- package/src/skf-create-skill/references/extract.md +3 -2
- package/src/skf-create-skill/references/extraction-patterns.md +3 -3
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +8 -3
- package/src/skf-create-skill/references/sub/ccc-discover.md +1 -1
- package/src/skf-create-skill/references/validate.md +1 -1
- package/src/skf-create-stack-skill/SKILL.md +3 -2
- package/src/skf-create-stack-skill/references/generate-output.md +1 -1
- package/src/skf-create-stack-skill/references/invocation-contract.md +1 -1
- package/src/skf-create-stack-skill/references/parallel-extract.md +2 -2
- package/src/skf-create-stack-skill/references/report.md +1 -1
- package/src/skf-create-stack-skill/references/validate.md +1 -1
- package/src/skf-drop-skill/SKILL.md +3 -2
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +1 -1
- package/src/skf-export-skill/SKILL.md +3 -2
- package/src/skf-export-skill/assets/snippet-format.md +5 -5
- package/src/skf-export-skill/references/generate-snippet.md +22 -15
- package/src/skf-export-skill/references/invocation-contract.md +1 -1
- package/src/skf-export-skill/references/load-skill.md +1 -1
- package/src/skf-export-skill/references/summary.md +1 -1
- package/src/skf-export-skill/references/token-report.md +29 -9
- package/src/skf-forger/SKILL.md +3 -2
- package/src/skf-forger/references/pipeline-mode.md +2 -2
- package/src/skf-quick-skill/SKILL.md +4 -3
- package/src/skf-quick-skill/references/finalize.md +1 -1
- package/src/skf-quick-skill/references/halt-contract.md +2 -2
- package/src/skf-quick-skill/references/write-and-validate.md +1 -1
- package/src/skf-refine-architecture/SKILL.md +8 -9
- package/src/skf-refine-architecture/customize.toml +1 -1
- package/src/skf-refine-architecture/references/report.md +1 -1
- package/src/skf-rename-skill/SKILL.md +3 -2
- package/src/skf-rename-skill/references/invocation-contract.md +2 -2
- package/src/skf-setup/SKILL.md +3 -2
- package/src/skf-setup/references/ccc-index.md +1 -1
- package/src/skf-setup/references/invocation-contract.md +1 -1
- package/src/skf-test-skill/SKILL.md +3 -2
- package/src/skf-test-skill/assets/output-section-formats.md +6 -4
- package/src/skf-test-skill/references/coherence-check.md +11 -8
- package/src/skf-test-skill/references/coverage-check-branches.md +81 -0
- package/src/skf-test-skill/references/coverage-check-tiers.md +3 -3
- package/src/skf-test-skill/references/coverage-check.md +60 -97
- package/src/skf-test-skill/references/init.md +2 -1
- package/src/skf-test-skill/references/inventory-contract.md +27 -0
- package/src/skf-test-skill/references/invocation-contract.md +3 -3
- package/src/skf-test-skill/references/migration-section-rules.md +26 -14
- package/src/skf-test-skill/references/report.md +5 -5
- package/src/skf-test-skill/references/scoring-rules.md +8 -7
- package/src/skf-test-skill/references/signature-comparison-rules.md +72 -0
- package/src/skf-test-skill/references/source-access-protocol.md +14 -10
- package/src/skf-test-skill/references/step-hard-gate.md +1 -0
- package/src/skf-test-skill/scripts/build-result-context.py +20 -3
- package/src/skf-test-skill/scripts/gap-ledger.py +456 -176
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +516 -79
- package/src/skf-test-skill/scripts/locate-export-segments.py +685 -76
- package/src/skf-test-skill/scripts/reconcile-coverage.py +192 -39
- package/src/skf-test-skill/scripts/score-signatures.py +1205 -65
- package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
- package/src/skf-test-skill/scripts/validate-inventory.py +350 -35
- package/src/skf-test-skill/templates/test-report-template.md +1 -0
- package/src/skf-update-skill/SKILL.md +3 -2
- package/src/skf-update-skill/references/detect-changes.md +1 -1
- package/src/skf-update-skill/references/gap-driven.md +54 -33
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +15 -10
- package/src/skf-update-skill/references/re-extract.md +3 -1
- package/src/skf-update-skill/references/report.md +6 -5
- package/src/skf-update-skill/references/write.md +42 -20
- package/src/skf-verify-stack/SKILL.md +4 -3
- package/tools/cli/lib/tool-check.js +158 -2
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# /// script
|
|
2
2
|
# requires-python = ">=3.11"
|
|
3
|
-
# dependencies = []
|
|
3
|
+
# dependencies = ["pyyaml"]
|
|
4
4
|
# ///
|
|
5
5
|
"""SKF Build Change Manifest — aggregate category A/B/C/D detection results.
|
|
6
6
|
|
|
@@ -64,11 +64,13 @@ Subcommands:
|
|
|
64
64
|
|
|
65
65
|
metadata --metadata <metadata.json> --lists <file> --stats <file>
|
|
66
66
|
--version <version> --generation-date <iso>
|
|
67
|
-
[--source-commit <sha>] [--source-ref <ref>]
|
|
67
|
+
[--source-commit <sha>] [--source-ref <ref>] [--skf-root <dir>]
|
|
68
|
+
[--extraction <file>] [--qmd-version <version>]
|
|
68
69
|
The skill's metadata.json an update leaves (see "Metadata" below),
|
|
69
70
|
written over --metadata through a temporary file and a rename.
|
|
70
71
|
Prints {"status": "written", "metadata": <path>, "version",
|
|
71
|
-
"exports": N, "scripts": N, "assets": N}
|
|
72
|
+
"exports": N, "scripts": N, "assets": N, "tool_versions": {...},
|
|
73
|
+
"dropped_stats": [...], "warnings": [...]}.
|
|
72
74
|
|
|
73
75
|
records [--extraction <file>] [--export-details <file>]
|
|
74
76
|
[--files-from <file>] [--patches <dir>] -o <file>
|
|
@@ -79,13 +81,18 @@ Subcommands:
|
|
|
79
81
|
(the exports marked public: false), "warnings"}.
|
|
80
82
|
|
|
81
83
|
gap-records --manifest <file> --provenance-map <file> [--source-root <dir>]
|
|
82
|
-
[--drift-status ok|skipped|overridden]
|
|
83
|
-
[--
|
|
84
|
+
[--brief <file>] [--drift-status ok|skipped|overridden]
|
|
85
|
+
[--judgments <file>] [--remediation-records <file>]
|
|
86
|
+
[--evidence <file> --tier <tier>]
|
|
84
87
|
(--plan [--files-out <file>] | -o <file>)
|
|
85
88
|
A gap-driven repair's spot-checks, routing and verification records
|
|
86
89
|
(see "Gap records" below): --plan prints what to answer and which
|
|
87
|
-
files
|
|
88
|
-
|
|
90
|
+
files §4a's extraction scans (targeted re-extraction, and the
|
|
91
|
+
signatures of the exports a spot-check located); without it, the
|
|
92
|
+
records are written to -o through a temporary file and a rename.
|
|
93
|
+
With --brief (the skill brief), no file a gap cites or its
|
|
94
|
+
Remediation names that the brief's scope leaves out is checked,
|
|
95
|
+
scanned or repaired from.
|
|
89
96
|
|
|
90
97
|
baseline-gaps --provenance-map <file> --extraction <file> --files <file>
|
|
91
98
|
--source-root <dir> -o <file>
|
|
@@ -241,14 +248,32 @@ Apply (update-skill write.md §3):
|
|
|
241
248
|
|
|
242
249
|
Reads the provenance map the update started from (--provenance-map;
|
|
243
250
|
without it, a degraded run's new map) and prints nothing but a summary:
|
|
244
|
-
the new map goes to -o. Every entry
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
251
|
+
the new map goes to -o. Every entry it adds or rewrites carries
|
|
252
|
+
signature_source, set from the tool that produced the signature:
|
|
253
|
+
ast-grep gives T1 (with confidence T1 and the recipe's ast_node_type),
|
|
254
|
+
a signature read by eye (source-read) gives T1-low (confidence T1-low,
|
|
255
|
+
ast_node_type null). Every entry the run leaves alone keeps its exact
|
|
256
|
+
value, but for a signature_source it lacks or holds as a value other
|
|
257
|
+
than T1, T1-low, T2 or T3 (compared case-insensitively), as a map
|
|
258
|
+
written before the key existed does: once the mode's changes are made,
|
|
259
|
+
each such entry takes its confidence when that names T1-low, T2 or T3,
|
|
260
|
+
else T1 for an extraction_method of ast-grep or ast_bridge (compared
|
|
261
|
+
stripped and case-insensitively) and T1-low for any other (T1 needs an
|
|
262
|
+
ast-grep match). The summary's signature_source_backfilled counts these
|
|
263
|
+
fills by value ({"T1": 1, "T1-low": 2}, {} when none), with one warning
|
|
264
|
+
that lists each unrecognized value replaced (name: old -> new, the
|
|
265
|
+
first 10, then "and N more"); their names are not in its `updated`. A
|
|
266
|
+
rewritten entry keeps any key the update does not set (notes,
|
|
267
|
+
deprecated). A new entry's source_library is the map's library when its
|
|
268
|
+
entries name one (--skill-name when they name none). When they name
|
|
269
|
+
several, it is the library the entries of its file settle, else the one
|
|
270
|
+
the entries under the nearest folder above the file that holds entries
|
|
271
|
+
settle: the one library they name, or of several, the one that is a
|
|
272
|
+
path prefix of all the others (storybook beside storybook/test). The
|
|
273
|
+
record of a MOVED file, or of an export whose change names an old_file,
|
|
274
|
+
tries its new path, then its old. When neither settles it, or the entry
|
|
275
|
+
has no file, it is the map's first library, and one warning counts and
|
|
276
|
+
names those entries (the first 10, then "and N more").
|
|
252
277
|
|
|
253
278
|
incremental (normal mode) reads --manifest (build's output), the renamed
|
|
254
279
|
exports of --category-c and --ccc-pairs (or of --input, the category
|
|
@@ -274,9 +299,10 @@ Apply (update-skill write.md §3):
|
|
|
274
299
|
MOVED_EXPORT whose fresh record is the runner's (ast-grep) also takes
|
|
275
300
|
its confidence, extraction_method and ast_node_type (the entry's when
|
|
276
301
|
the record's is null), and its signature_source when the record holds
|
|
277
|
-
each of params and return_type the entry holds (else the entry's
|
|
278
|
-
|
|
279
|
-
the
|
|
302
|
+
each of params and return_type the entry holds (else the entry's, or
|
|
303
|
+
T1-low when the entry has none it recognizes: the signature it keeps
|
|
304
|
+
was not the runner's), never its export_type, params or return_type;
|
|
305
|
+
one read by eye moves the line and file only.
|
|
280
306
|
|
|
281
307
|
Public-api skills (incremental only): a NEW_EXPORT or MODIFIED_EXPORT
|
|
282
308
|
the map does not hold, an ADDED file's record and a MOVED file's record
|
|
@@ -299,31 +325,117 @@ Apply (update-skill write.md §3):
|
|
|
299
325
|
gap-driven reads --reextract-records ({"mode": "gap-driven",
|
|
300
326
|
"verification": [...], "files": [...]}) and --merge-records ({"exports":
|
|
301
327
|
[{export_name, export_type, params, return_type}]}, merge's output for
|
|
302
|
-
an export no
|
|
328
|
+
an export no record covers). Each verification record names gap_id,
|
|
303
329
|
export_name, gap_category, severity, verification, in_map (the map holds
|
|
304
330
|
an entry of that name), map_entry ({source_file, source_line} of the one
|
|
305
331
|
entry the spot-check took, else null), source_citation, new_location,
|
|
306
|
-
unknown_reason,
|
|
307
|
-
|
|
308
|
-
|
|
332
|
+
unknown_reason, citation_check, map_locations, outside_scope (see "Gap
|
|
333
|
+
records"), pinned_definition_lines, reachability, and the located
|
|
334
|
+
export's export_type, params, return_type, extraction_method and
|
|
335
|
+
ast_node_type (from §4a's record at its line, see "Gap records"; null
|
|
336
|
+
when none holds it). It finds an export's entry by its name, else by the
|
|
337
|
+
entry's reexported_as (a renamed re-export the map holds under its
|
|
338
|
+
declared name, which the gap names by its public one):
|
|
339
|
+
verified (map_entry) unchanged, but a NEW_EXPORT's or
|
|
340
|
+
MODIFIED_EXPORT's signature (below) when its
|
|
341
|
+
record was found
|
|
342
|
+
moved (map_entry) source_line (and source_file) from new_location,
|
|
343
|
+
and a NEW_EXPORT's or MODIFIED_EXPORT's
|
|
344
|
+
signature (below) when its record was found
|
|
309
345
|
re-extracted the entry from its `files` record (gap-driven.md §4a);
|
|
310
|
-
none when its reachability is internal-unreachable
|
|
346
|
+
none when its reachability is internal-unreachable;
|
|
347
|
+
a record with a `local` name (matched by the
|
|
348
|
+
public name a re-export gives it) writes
|
|
349
|
+
export_name the declared `local` name, at its
|
|
350
|
+
line, and reexported_as the public one
|
|
311
351
|
verified/moved, not in_map, a NEW_EXPORT or MODIFIED_EXPORT whose
|
|
312
352
|
reachability is not internal-unreachable
|
|
313
353
|
one source-read entry at the citation's line
|
|
314
354
|
(verified) or new_location (moved), at most one
|
|
315
|
-
per name and file,
|
|
316
|
-
|
|
355
|
+
per name and file, with the signature (below)
|
|
356
|
+
and the source-read labels whatever its record's
|
|
357
|
+
tool
|
|
358
|
+
unknown, outside-scope unchanged and no entry added, whatever its
|
|
359
|
+
severity (never refused), with a warning that
|
|
360
|
+
names its gap_id, its unknown_reason and its
|
|
361
|
+
words, its outside_scope (the files the brief's
|
|
362
|
+
scope leaves out), the cause its citation_check
|
|
363
|
+
keeps for a citation the scope takes (with its
|
|
364
|
+
words) and the fix: when the skill covers that
|
|
365
|
+
file, change the brief's scope so it takes it
|
|
366
|
+
(its scope.include or scope.exclude), then
|
|
367
|
+
re-run update-skill; otherwise re-run Test
|
|
368
|
+
Skill (the test compared a declaration the
|
|
369
|
+
skill does not cover). A citation_check that
|
|
370
|
+
read nothing (UNREAD) takes its own fix in
|
|
371
|
+
their place, as below
|
|
317
372
|
unknown, not in_map, a NEW_EXPORT or MODIFIED_EXPORT, a Medium, Low or
|
|
318
373
|
Info severity an entry from --merge-records with no
|
|
319
374
|
source_file or source_line
|
|
320
375
|
unknown, not in_map, any other severity (a missing one included)
|
|
321
376
|
refused: exit 3, nothing written
|
|
322
377
|
missing, unknown in_map, unknown on a MOVED_EXPORT
|
|
323
|
-
unchanged, with a warning for a person
|
|
378
|
+
unchanged, with a warning for a person that
|
|
379
|
+
names an unknown's unknown_reason and its
|
|
380
|
+
words (UNKNOWN_REASONS); for a NEW_EXPORT or
|
|
381
|
+
MODIFIED_EXPORT whose severity is not Medium,
|
|
382
|
+
Low or Info (a missing one included), the
|
|
383
|
+
warning also names its gap_id and the one
|
|
384
|
+
fix a re-run acts on: for an ambiguous one,
|
|
385
|
+
fix its entries by hand, or give the gap a
|
|
386
|
+
`file:line` Source at its definition when
|
|
387
|
+
that file holds only one of them (the
|
|
388
|
+
re-run's map_match then finds that one); for
|
|
389
|
+
any other, fix the entry by hand (its
|
|
390
|
+
source_file and source_line), since the
|
|
391
|
+
re-run checks the same entry again. A
|
|
392
|
+
missing MOVED_EXPORT, whatever its severity,
|
|
393
|
+
takes that warning too (the gone file and
|
|
394
|
+
the hand fix): rule R5 moves a line, never
|
|
395
|
+
a file. A MOVED_EXPORT's warning names the
|
|
396
|
+
pinned_definition_lines only when the test
|
|
397
|
+
report listed any. A
|
|
398
|
+
cited-elsewhere one, whatever its severity,
|
|
399
|
+
takes in their place a warning that names
|
|
400
|
+
its gap_id, its source_citation, its
|
|
401
|
+
citation_check (with its words), its
|
|
402
|
+
map_locations and the fix for each side
|
|
403
|
+
that may be wrong: when the map is right,
|
|
404
|
+
re-run Test Skill (the test compared
|
|
405
|
+
another file's declaration), and if the gap
|
|
406
|
+
comes back (not for a missing one), exclude
|
|
407
|
+
the cited file in the brief's scope when the
|
|
408
|
+
skill does not cover it; when the export
|
|
409
|
+
moved, fix the entry by hand (with several
|
|
410
|
+
map_locations, whichever of those entries
|
|
411
|
+
moved), then re-run update-skill. A
|
|
412
|
+
citation_check outside-scope (no check ran:
|
|
413
|
+
the brief's scope leaves the cited file out)
|
|
414
|
+
takes no advice to exclude it, and its fix
|
|
415
|
+
for a move is to change the brief's scope so
|
|
416
|
+
it takes that file (its scope.include or
|
|
417
|
+
scope.exclude) and fix the entry by hand. A
|
|
418
|
+
citation_check that read nothing (UNREAD:
|
|
419
|
+
no-source-root, verifier-missing,
|
|
420
|
+
spot-check-failed) takes its own fix in
|
|
421
|
+
their place: give update-skill the source
|
|
422
|
+
tree, re-install SKF, or fix what the check
|
|
423
|
+
failed on, then re-run update-skill
|
|
324
424
|
rescoped the entry removed
|
|
425
|
+
The signature: when §4a found the export's record (its verification
|
|
426
|
+
record has an extraction_method), every part from it, a params or
|
|
427
|
+
return_type it gives as null removed from the entry; with none, an
|
|
428
|
+
entry the map holds keeps its signature, and an entry this run adds
|
|
429
|
+
takes --merge-records'. An entry the map holds keeps its confidence and
|
|
430
|
+
extraction_method; when its own extraction_method is ast-grep or
|
|
431
|
+
ast_bridge, it takes the signature_source of its record's labels (T1
|
|
432
|
+
for an ast-grep record, T1-low for a source-read one) and, when the
|
|
433
|
+
record's export_type is another, the record's ast_node_type, while a
|
|
434
|
+
source-read entry keeps T1-low and no node kind. Each signature_source
|
|
435
|
+
it changes is listed in `relabeled`, {name, file, from, to}, each with a
|
|
436
|
+
warning. An internal-unreachable NEW_EXPORT takes none.
|
|
325
437
|
Under the drift override (--drift-head and --drift-pinned) each warning
|
|
326
|
-
names the drift.
|
|
438
|
+
names the drift, after the outcome's cause.
|
|
327
439
|
|
|
328
440
|
full (degraded mode) makes every fresh record an entry.
|
|
329
441
|
|
|
@@ -351,8 +463,9 @@ Apply (update-skill write.md §3):
|
|
|
351
463
|
--metadata-in exports[] less each name whose entry this run removed (a
|
|
352
464
|
DELETED_EXPORT, a DELETED file's or a MOVED file's old path's, a
|
|
353
465
|
rescoped one, a changed URL's) or renamed, unless an entry of the new
|
|
354
|
-
map still holds it, then each name it added an entry for
|
|
355
|
-
|
|
466
|
+
map still holds it, then each name it added an entry for (the entry's
|
|
467
|
+
reexported_as when it has one, the public name) or renamed an entry to,
|
|
468
|
+
each once; a renamed name takes its old name's place. A name
|
|
356
469
|
left out as off the public surface (the summary's not_public) has no
|
|
357
470
|
entry, so it is not added. A full update, which rebuilds the map, also
|
|
358
471
|
drops each name the new map does not hold, and a reference app
|
|
@@ -379,15 +492,45 @@ Metadata (update-skill write.md §2):
|
|
|
379
492
|
component library's own counts, a reference app's notes, a field a
|
|
380
493
|
gap-driven patch added), the stats helper's fields an earlier run wrote
|
|
381
494
|
that this run's does not (effective_denominator,
|
|
382
|
-
pattern_surfaces_documented) dropped
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
495
|
+
pattern_surfaces_documented) dropped and listed in `dropped_stats`,
|
|
496
|
+
each with a warning. When this run's effective_denominator differs
|
|
497
|
+
from the old one (one of them absent included), it also drops each old
|
|
498
|
+
stats key named effective_denominator_* (an explanation of the old
|
|
499
|
+
count), listed in `dropped_stats` with the warning "metadata:
|
|
500
|
+
stats.<key> dropped: it explained an effective_denominator this run
|
|
501
|
+
changed or removed"; when the two are the same, such a key stays with
|
|
502
|
+
no warning, since it still describes the denominator. Each other old
|
|
503
|
+
stats key it keeps adds the warning "metadata: stats.<key> kept from an
|
|
504
|
+
earlier run; check it still describes this run's counts", unless
|
|
505
|
+
create-skill writes it for the skill's scope_type (STATS_CREATE_FIELDS:
|
|
506
|
+
a component-library's Component Library stats of
|
|
507
|
+
compile-assembly-rules.md, a reference-app's notes).
|
|
508
|
+
|
|
509
|
+
tool_versions names the tools this run used: the old tool_versions
|
|
510
|
+
({"ast_grep": null, "qmd": null} when absent, or when it is not an
|
|
511
|
+
object, which adds a warning), with skf set to the version
|
|
512
|
+
skf-render-quick-metadata.py's probe_skf_version reads under
|
|
513
|
+
--skf-root (package.json, else VERSION), never the old value, or to
|
|
514
|
+
"unknown" with a warning, one when no --skf-root is given and another
|
|
515
|
+
naming the folder when it holds no version; ast_grep set to the
|
|
516
|
+
ast_grep.version of --extraction (the recipe runner's output), or to
|
|
517
|
+
"unknown" with a warning when that is null but an export of it has
|
|
518
|
+
extraction_method ast-grep; qmd set to --qmd-version, a leading "qmd "
|
|
519
|
+
(as `qmd --version` prints it) removed. A tool this run did not use
|
|
520
|
+
(no --qmd-version, no --extraction, or one with no export ast-grep
|
|
521
|
+
matched) keeps its old value. It removes last_update and update_type,
|
|
522
|
+
which the provenance map's update block records, and keeps every
|
|
523
|
+
other field. It refuses (exit 1, nothing written, one stderr line) a
|
|
524
|
+
--stats whose scripts_count or assets_count is not the length of
|
|
525
|
+
--lists' scripts or assets, a --generation-date of another form, an
|
|
526
|
+
--extraction that is not a JSON object, a file it cannot read, and a
|
|
527
|
+
skf-render-quick-metadata.py beside it that cannot be loaded.
|
|
387
528
|
|
|
388
529
|
Summary on stdout: {"status": "written", "map": <-o>, "entries":
|
|
389
530
|
{"added", "updated", "removed"}, "not_public": [{name, file}],
|
|
390
|
-
"
|
|
531
|
+
"relabeled": [{name, file, from, to}] (gap-driven),
|
|
532
|
+
"signature_source_backfilled": {tier: count}, "file_entries":
|
|
533
|
+
{"added", "updated", "removed"}, "warnings": [...]},
|
|
391
534
|
or {"status": "refused",
|
|
392
535
|
"blocking_unresolved": [{export_name, severity}]} with exit 3.
|
|
393
536
|
|
|
@@ -415,13 +558,26 @@ it scans):
|
|
|
415
558
|
no seed holds is not added, with a warning. confidence_breakdown counts
|
|
416
559
|
the exports by confidence, and T2 the exports with a qmd_evidence.
|
|
417
560
|
|
|
561
|
+
Aliases. Each --extraction entry_point_diff `public` and
|
|
562
|
+
`other_definitions` item (an other in-scope definition of a public
|
|
563
|
+
name), and each member of a namespace item, whose `local` differs from
|
|
564
|
+
its `name` gives its name to the record of its `local` name and file,
|
|
565
|
+
when one is seeded:
|
|
566
|
+
that record's `aliases` lists each such public name once, sorted (an
|
|
567
|
+
export re-exported under another name, `export { MockUniversalStore as
|
|
568
|
+
experimental_MockUniversalStore }`, keeps its declared name and carries
|
|
569
|
+
the public one). A record no item renames has no `aliases`, and with
|
|
570
|
+
no entry_point_diff (Quick tier, a runner exit 3, an export read by
|
|
571
|
+
eye) no record has any.
|
|
572
|
+
|
|
418
573
|
Public surface. When --extraction's scope.type is public-api (in any
|
|
419
574
|
case) and it has an entry_point_diff, each export of a language family
|
|
420
575
|
whose entry_points.by_language status is `barrel` is marked `public`:
|
|
421
576
|
true when its name and file are a pair of the runner's entry_point_diff
|
|
422
|
-
`public` (its name, or the name `local` gives it
|
|
423
|
-
`extraction_gaps`, or when a `public` item of its name
|
|
424
|
-
no file; false otherwise. A `public` item with `via`
|
|
577
|
+
`public` or `other_definitions` (its name, or the name `local` gives it
|
|
578
|
+
in that file) or `extraction_gaps`, or when a `public` item of its name
|
|
579
|
+
and family has no file; false otherwise. A `public` item with `via`
|
|
580
|
+
namespace and a
|
|
425
581
|
`members` list (what the submodule passes on, which users reach as
|
|
426
582
|
`pkg.sub.name`, and which counts.exports_public_api counts in its place)
|
|
427
583
|
gives its members' pairs (each member's name, or its `local` one, and
|
|
@@ -444,14 +600,38 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
|
444
600
|
STRUCTURAL_FIX and metadata update entries are forwarded to merge as
|
|
445
601
|
they are (`forwarded`); every other entry is export-bearing. A severity
|
|
446
602
|
is blocking unless it is Medium, Low or Info (NON_BLOCKING), compared
|
|
447
|
-
case-insensitively, so a missing one is blocking.
|
|
603
|
+
case-insensitively, so a missing one is blocking.
|
|
604
|
+
|
|
605
|
+
The brief's scope (--brief, the skill brief) bounds the repair (#824),
|
|
606
|
+
by skf-classify-changed-files.py's load_scope and Scope.takes, loaded
|
|
607
|
+
from this folder (Category A's in-scope test: an exclude wins over an
|
|
608
|
+
include, and an empty include takes only a path with the suffix of a
|
|
609
|
+
path the gaps or the map's entries name or of the brief's language,
|
|
610
|
+
so a file with no suffix is not taken), with Category A's
|
|
611
|
+
rule that a file the map's entries[] cite is always taken. Each path is
|
|
612
|
+
tested relative to --source-root, POSIX, its `.` and `..` parts
|
|
613
|
+
resolved (an absolute path under the root loses the root's prefix).
|
|
614
|
+
An entry's outside_scope is its cited file and each of its
|
|
615
|
+
resolved_paths the scope leaves out: none of them is checked, scanned
|
|
616
|
+
or repaired from, so a declaration of another package the skill does
|
|
617
|
+
not cover is never read for it. A map entry's own file is checked
|
|
618
|
+
wherever it sits (found and MOVED_EXPORT routing does not test it).
|
|
619
|
+
Without --brief (a skill with no brief) there is no scope test and
|
|
620
|
+
nothing is left out. A --brief that names no file or cannot be read
|
|
621
|
+
is an input error (exit 1). In order:
|
|
448
622
|
|
|
449
623
|
1. Drift gate (--drift-status overridden): each export-bearing entry
|
|
450
624
|
but a MOVED_EXPORT needs the tree, with the first reason that fits:
|
|
451
625
|
a DELETED_EXPORT "a public API recount from the tree (rule R1)", a
|
|
452
626
|
provenance_completeness entry "a line from the tree (rule R3)", one
|
|
453
|
-
whose map_match is not-found
|
|
454
|
-
|
|
627
|
+
whose map_match is not-found with no candidate that has a
|
|
628
|
+
source_file "a line and a signature from the tree", any other (a
|
|
629
|
+
not-found with such a candidate, a citation of another file than
|
|
630
|
+
the map's entries, among them) "a signature from the tree". An
|
|
631
|
+
entry the brief's scope keeps from a repair before anything is read
|
|
632
|
+
for it (step 3 records it outside-scope, or cited-elsewhere with
|
|
633
|
+
citation_check outside-scope, with no check run and nothing to
|
|
634
|
+
scan) passes. Any other such entry: {"status":
|
|
455
635
|
"drift-blocked", "drift_blocked": [{gap_id, name, change_category,
|
|
456
636
|
severity, reason}]}, and no spot-check runs.
|
|
457
637
|
2. A DELETED_EXPORT without a `rescope` that holds its `amendment` and
|
|
@@ -460,57 +640,147 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
|
460
640
|
lines rules, loaded from this folder, under --source-root:
|
|
461
641
|
DELETED_EXPORT rescoped, no check
|
|
462
642
|
MOVED_EXPORT checked at its map entry when map_match is
|
|
463
|
-
found (unknown otherwise)
|
|
643
|
+
found (unknown otherwise), a bookkeeping
|
|
644
|
+
variant its file never names against its
|
|
645
|
+
base (the verifier's fold; no other
|
|
646
|
+
category folds); under the
|
|
464
647
|
override a `moved` becomes unknown with
|
|
465
648
|
unknown_reason drift-override; its
|
|
466
649
|
pinned_definition_lines are the numbers of
|
|
467
650
|
its remediation's "definition line(s) (...)"
|
|
468
651
|
NEW_EXPORT, MODIFIED_EXPORT
|
|
469
652
|
map_match found checked at the map entry (its export_type)
|
|
470
|
-
ambiguous unknown
|
|
653
|
+
ambiguous unknown, unknown_reason ambiguous (several
|
|
654
|
+
map entries of its name: no check runs)
|
|
471
655
|
not-found checked at its source_citation (no export
|
|
472
|
-
type) when it has one
|
|
473
|
-
|
|
656
|
+
type) when it has one, but never at a
|
|
657
|
+
cited file its outside_scope holds: no
|
|
658
|
+
check runs there, and it is unknown,
|
|
659
|
+
unknown_reason cited-elsewhere with
|
|
660
|
+
citation_check outside-scope when the map
|
|
661
|
+
holds entries of its name with a
|
|
662
|
+
source_file (never re-extracted), else
|
|
663
|
+
unknown_reason outside-scope, routed on
|
|
664
|
+
as below. When the map holds
|
|
665
|
+
entries of its name (in_map, which a
|
|
666
|
+
not-found has only for a citation of
|
|
667
|
+
another file than theirs), one of them
|
|
668
|
+
with a source_file, and no check pins a
|
|
669
|
+
line (unknown or missing): unknown,
|
|
670
|
+
unknown_reason cited-elsewhere, whatever
|
|
671
|
+
its severity (a provenance_completeness
|
|
672
|
+
one too), and never re-extracted, so no
|
|
673
|
+
other file's declaration is read for it;
|
|
674
|
+
its record's citation_check keeps the
|
|
675
|
+
check's cause (missing, or its
|
|
676
|
+
unknown_reason) and its map_locations the
|
|
677
|
+
`file:line` (or file) of each candidate
|
|
678
|
+
with a source_file, the map_locations null
|
|
679
|
+
on any other record. Candidates with no
|
|
680
|
+
source_file (placeholders) do not count.
|
|
681
|
+
Otherwise,
|
|
682
|
+
when its outside_scope is not empty and
|
|
683
|
+
the scope takes none of its
|
|
684
|
+
resolved_paths: unknown, unknown_reason
|
|
685
|
+
outside-scope, whatever its severity, with
|
|
686
|
+
nothing to scan (never in the plan); its
|
|
687
|
+
citation_check keeps the cause of a check
|
|
688
|
+
at a citation the scope takes (missing, or
|
|
689
|
+
its unknown_reason), null with none. The
|
|
690
|
+
citation_check is null on every record but
|
|
691
|
+
a cited-elsewhere or outside-scope one.
|
|
692
|
+
Otherwise, when that pins no line
|
|
693
|
+
(unknown or missing), or with no
|
|
474
694
|
citation: targeted re-extraction (§4a) for a
|
|
475
695
|
provenance_completeness entry with a source
|
|
476
696
|
root, for a blocking one whatever its
|
|
477
697
|
resolved_paths, and for a missing-export or
|
|
478
698
|
missing-type one with resolved_paths; else
|
|
479
|
-
unknown
|
|
699
|
+
unknown. Targeted re-extraction scans only
|
|
700
|
+
the resolved_paths the scope takes
|
|
480
701
|
A check gives `verified` (the line is a definition line, or a module
|
|
481
702
|
or package entry whose file exists), `moved` (one definition line,
|
|
482
703
|
not the recorded one: new_location), `missing` (no such file) or
|
|
483
|
-
`unknown` (several
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
unknown, with a `provenance: spot-
|
|
490
|
-
warning
|
|
704
|
+
`unknown`, its unknown_reason `several-definitions` (several
|
|
705
|
+
definition lines, none the recorded one) or `no-definition` (none).
|
|
706
|
+
A file the rules cover no language of takes the answered
|
|
707
|
+
`declaring_line`: the recorded line is verified, another is moved,
|
|
708
|
+
null is unknown (`unchecked-language`). With no --source-root every
|
|
709
|
+
check is unknown (`no-source-root`); with no verifier every check is
|
|
710
|
+
unknown (`verifier-missing`), with a `provenance: spot-checks not
|
|
711
|
+
run` warning; a check the verifier fails on (a file it cannot read,
|
|
712
|
+
or any error) is unknown (`spot-check-failed`), with a `provenance:
|
|
713
|
+
spot-check failed for {name}: {error}` warning; a map entry with no
|
|
714
|
+
file or line is unknown (`no-line`). UNKNOWN_REASONS gives each its
|
|
715
|
+
words. Every other outcome has a null unknown_reason. A map_match
|
|
716
|
+
whose entry carries its own export_name (skf-parse-gaps matched it
|
|
717
|
+
by its reexported_as) is checked under that name.
|
|
718
|
+
A NEW_EXPORT or MODIFIED_EXPORT the map holds entries of its name
|
|
719
|
+
for (its map_match found or ambiguous, or not-found with a
|
|
720
|
+
candidate that has a source_file) never goes to targeted
|
|
721
|
+
re-extraction, whatever its severity: one that ends missing or
|
|
722
|
+
unknown keeps its map entries unchanged and adds none, and apply
|
|
723
|
+
warns (see "Apply"). One recorded unknown, outside-scope (its record's
|
|
724
|
+
outside_scope lists the files the scope leaves out) adds no entry
|
|
725
|
+
either, a placeholder included, whatever its severity, and apply
|
|
726
|
+
warns.
|
|
727
|
+
A NEW_EXPORT or MODIFIED_EXPORT a check gives verified or moved, at
|
|
728
|
+
its map entry or at a citation the map holds no entry of its name
|
|
729
|
+
for, is located: its signature_at is the checked file and the
|
|
730
|
+
recorded line (verified) or the new one (moved). One whose citation
|
|
731
|
+
names another file than the map's entries of its name is not (apply
|
|
732
|
+
warns, as before), nor one whose file is absolute or outside
|
|
733
|
+
--source-root, links resolved (a cited file the brief's scope leaves
|
|
734
|
+
out is never checked, so never located), nor a NEW_EXPORT
|
|
735
|
+
--judgments answers internal-unreachable (one not yet answered is
|
|
736
|
+
located, so its scan may go unused).
|
|
491
737
|
--plan stops here: {"status": "planned", "reextract": {"entries":
|
|
492
|
-
[{gap_id, name, severity, resolved_paths
|
|
493
|
-
their resolved_paths
|
|
494
|
-
|
|
738
|
+
[{gap_id, name, severity, resolved_paths (those the scope takes)}],
|
|
739
|
+
"files": [the union of their resolved_paths, then of the located
|
|
740
|
+
exports' files],
|
|
741
|
+
"signatures": [{gap_id, name, file, line} of each located export]},
|
|
742
|
+
"warnings"}, the files also written to --files-out as a JSON array.
|
|
495
743
|
4. Targeted re-extraction (--remediation-records, `records`' output
|
|
496
744
|
over those files): each routed entry with resolved_paths takes the
|
|
497
745
|
first export of its name in the blocks of those files, in their
|
|
498
|
-
order
|
|
746
|
+
order, else the first whose `aliases` hold it (matched by name, or
|
|
747
|
+
by the public name the runner gives its local name in that file,
|
|
748
|
+
see "Records"): `re-extracted`, new_location its record's location,
|
|
499
749
|
resolution_source remediation-paths, and the record copied into
|
|
500
750
|
`files` as `records` wrote it, less its `public` mark (a normal
|
|
501
|
-
update's)
|
|
751
|
+
update's) and its `aliases`, with the answered `docstring` added. A
|
|
752
|
+
record matched by an alias is copied with `name` the public name and
|
|
753
|
+
`local` its own name. The aliases come from the runner's
|
|
754
|
+
entry_point_diff, so at Forge tier and above only: at Quick tier, or
|
|
755
|
+
when the runner exits 3 and §4a reads the files by eye, a gap naming
|
|
756
|
+
a renamed re-export by its public name matches no record.
|
|
502
757
|
An entry with no match, or no path to scan, is unresolved, except a
|
|
503
758
|
missing-export or missing-type one that is not blocking, which is
|
|
504
|
-
unknown.
|
|
759
|
+
unknown. An entry with no match whose outside_scope is not empty is
|
|
760
|
+
unknown, unknown_reason outside-scope (a file the scope leaves out
|
|
761
|
+
may define it), whatever its severity and category, never
|
|
762
|
+
unresolved. Any unresolved entry: {"status": "unresolved",
|
|
505
763
|
"unresolved": [{gap_id, name, severity, remediation_paths,
|
|
506
764
|
rejected_paths, files_scanned, exports_found_in_scan}]}.
|
|
765
|
+
Each located export takes the one record of its name (or of its map
|
|
766
|
+
entry's own export_name) in the block of its file at its line, else
|
|
767
|
+
the one with no line: the record copied into `files` as above, under
|
|
768
|
+
the gap's name, less its `public` mark and `aliases`, with no
|
|
769
|
+
docstring asked
|
|
770
|
+
(one copy per name and location, or per record with no location),
|
|
771
|
+
and its type, params, return_type, extraction_method and
|
|
772
|
+
ast_node_type on the verification record. One with no such record
|
|
773
|
+
keeps them null, with a `provenance: {name}: located at
|
|
774
|
+
{file}:{line}, but no record of it there; signature not repaired`
|
|
775
|
+
warning (§4a's by-eye pass may add it to --export-details). Files to
|
|
776
|
+
match, of either kind, with no --remediation-records are an error.
|
|
507
777
|
5. The public-reachability gate: every NEW_EXPORT that is verified,
|
|
508
778
|
moved or re-extracted takes the answered `reachability`, `public` or
|
|
509
779
|
`internal-unreachable`. An internal one keeps its outcome in its
|
|
510
|
-
verification record, its record is not in
|
|
511
|
-
it under `reclassified` instead of its
|
|
512
|
-
`reclassified` (merge queues it as a
|
|
513
|
-
entry for it).
|
|
780
|
+
verification record (with no signature), its record is not in
|
|
781
|
+
`files`, `counts` tallies it under `reclassified` instead of its
|
|
782
|
+
outcome, and it is listed in `reclassified` (merge queues it as a
|
|
783
|
+
metadata update; apply writes no entry for it).
|
|
514
784
|
|
|
515
785
|
Any answer missing in step 3 or 5 (a `docstring` too, for each
|
|
516
786
|
re-extracted entry): {"status": "needs-judgment", "needs_judgment":
|
|
@@ -522,26 +792,34 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
|
522
792
|
"unlabeled", "T2": 0} (each manifest entry by the extraction_method of
|
|
523
793
|
its re-extracted record or, in the map, of its map entry; a pinned cited
|
|
524
794
|
export the map lacks and the gate passed is T1-low), "verification":
|
|
525
|
-
[one record per export-bearing entry:
|
|
526
|
-
severity, verification, in_map, map_entry,
|
|
527
|
-
source_citation, new_location, unknown_reason,
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
795
|
+
[one record per export-bearing entry: gap_id, export_name,
|
|
796
|
+
gap_category, severity, verification, in_map, map_entry,
|
|
797
|
+
provenance_citation, source_citation, new_location, unknown_reason,
|
|
798
|
+
citation_check, map_locations, outside_scope (the files a gap cites or
|
|
799
|
+
its Remediation names that the brief's scope leaves out, whatever the
|
|
800
|
+
record's outcome; null when it leaves out none, or with no --brief),
|
|
801
|
+
pinned_definition_lines, resolution_source, reachability, and
|
|
802
|
+
export_type, params, return_type, extraction_method and ast_node_type
|
|
803
|
+
(a located export's record's, step 4; else null, and apply takes the
|
|
804
|
+
signature of an entry it adds from merge.md's merge-records.json)],
|
|
805
|
+
"files": [...]}, and prints {"status": "written",
|
|
531
806
|
"output", "gap_count", "files_extracted", "exports_extracted",
|
|
532
807
|
"confidence_breakdown", "counts" (per verification outcome, and
|
|
533
808
|
`reclassified`), "reclassified", "forwarded", "targeted_reextraction",
|
|
534
|
-
"warnings"}. When targeted re-extraction
|
|
535
|
-
|
|
536
|
-
|
|
809
|
+
"warnings"}. When §4a ran (targeted re-extraction, or a located
|
|
810
|
+
export's signature), `targeted_reextraction` is {resolved_count,
|
|
811
|
+
files_scanned (the targeted re-extraction's files),
|
|
812
|
+
signature_files_scanned (the located exports' files), exports_matched,
|
|
813
|
+
tier}, also appended to --evidence as one JSON line keyed
|
|
814
|
+
`targeted_reextraction`.
|
|
537
815
|
|
|
538
816
|
Baseline gaps (update-skill detect-changes Category B, step 1):
|
|
539
817
|
|
|
540
818
|
The recipe runner never reports a module, an alias that renames, a
|
|
541
|
-
dunder such as `__version__` or another
|
|
542
|
-
would read each such map entry as a deleted export. baseline-gaps
|
|
543
|
-
for step 2 to read by eye, each entry of a file --files lists
|
|
544
|
-
modified files, Category A's modified-files.json) whose name and file
|
|
819
|
+
dunder such as `__version__` or another Python or Rust `_` name, so the
|
|
820
|
+
diff would read each such map entry as a deleted export. baseline-gaps
|
|
821
|
+
lists, for step 2 to read by eye, each entry of a file --files lists
|
|
822
|
+
(the modified files, Category A's modified-files.json) whose name and file
|
|
545
823
|
no export of --extraction (the runner's JSON, or {"exports": []} when
|
|
546
824
|
it could not run) holds, under its name or its re-export target, and
|
|
547
825
|
whose file still declares it. The selection and the declaration rule
|
|
@@ -563,9 +841,10 @@ Exit codes:
|
|
|
563
841
|
0 operation succeeded (for gap-records, whatever its `status`)
|
|
564
842
|
1 user error (malformed JSON, bad path, malformed provenance file, a
|
|
565
843
|
helper file that is not the output it names, a gap-records answer of
|
|
566
|
-
the wrong type, targeted re-extraction
|
|
567
|
-
--remediation-records, or, for
|
|
568
|
-
no folder or a verifier that
|
|
844
|
+
the wrong type, files to match (targeted re-extraction or a located
|
|
845
|
+
export's signature) with no --remediation-records, or, for
|
|
846
|
+
baseline-gaps, a source root that is no folder or a verifier that
|
|
847
|
+
cannot be loaded)
|
|
569
848
|
3 apply refused: a blocking gap reached the provenance write with no
|
|
570
849
|
source line (nothing written)
|
|
571
850
|
"""
|
|
@@ -578,6 +857,7 @@ import hashlib
|
|
|
578
857
|
import importlib.util
|
|
579
858
|
import json
|
|
580
859
|
import os
|
|
860
|
+
import posixpath
|
|
581
861
|
import re
|
|
582
862
|
import subprocess
|
|
583
863
|
import sys
|
|
@@ -1330,6 +1610,8 @@ REPLACE_WAIT_SECONDS = 5.0 # how long a Windows rename onto a file held open is
|
|
|
1330
1610
|
# The update block apply sets at the top level of the map, in this order.
|
|
1331
1611
|
UPDATE_BLOCK_KEYS = ("last_update", "update_type", "test_report_run_id", "files_changed", "exports_affected",
|
|
1332
1612
|
"confidence_tier", "manual_sections_preserved")
|
|
1613
|
+
# The parts of a signature a gap-driven repair writes from a verification record, else from merge's records.
|
|
1614
|
+
SIGNATURE_PARTS = ("export_type", "params", "return_type")
|
|
1333
1615
|
|
|
1334
1616
|
|
|
1335
1617
|
def _labels(method: object, node: object) -> dict:
|
|
@@ -1457,16 +1739,17 @@ def _without_none(fields: dict, keep: tuple = ()) -> dict:
|
|
|
1457
1739
|
|
|
1458
1740
|
def _new_entry(fields: dict, library: str) -> dict:
|
|
1459
1741
|
"""An entry in create-skill's key order, from the fields a record gives."""
|
|
1460
|
-
order = ("export_name", "export_type", "source_library", "params", "return_type", "source_file",
|
|
1742
|
+
order = ("export_name", "reexported_as", "export_type", "source_library", "params", "return_type", "source_file",
|
|
1461
1743
|
"source_line", "confidence", "extraction_method", "ast_node_type", "signature_source")
|
|
1462
1744
|
merged = {**fields, "source_library": fields.get("source_library") or library}
|
|
1463
1745
|
out = {k: merged[k] for k in order if k in merged and (merged[k] is not None or k == "ast_node_type")}
|
|
1464
1746
|
return out
|
|
1465
1747
|
|
|
1466
1748
|
|
|
1467
|
-
def _rewrite(entry: dict, fields: dict) -> dict:
|
|
1468
|
-
"""The entry with `fields` set in place; keys the update does not set keep
|
|
1469
|
-
|
|
1749
|
+
def _rewrite(entry: dict, fields: dict, drop: tuple = ()) -> dict:
|
|
1750
|
+
"""The entry with `fields` set in place and the keys `drop` names removed; keys the update does not set keep
|
|
1751
|
+
their values."""
|
|
1752
|
+
out = {k: v for k, v in entry.items() if k not in drop}
|
|
1470
1753
|
for key, value in fields.items():
|
|
1471
1754
|
if value is None and key not in ("ast_node_type",):
|
|
1472
1755
|
continue
|
|
@@ -1490,19 +1773,87 @@ class _Map:
|
|
|
1490
1773
|
"provenance_version": "2.0", "skill_name": skill_name, "skill_type": "single", "entries": []}
|
|
1491
1774
|
if not isinstance(self.doc.get("entries"), list):
|
|
1492
1775
|
raise ValueError("provenance `entries` must be an array")
|
|
1493
|
-
libraries
|
|
1494
|
-
|
|
1495
|
-
self.
|
|
1776
|
+
# the map's libraries in entry order, and each file's, as the map stood before the run (#808)
|
|
1777
|
+
self.libraries: list[str] = []
|
|
1778
|
+
self.libraries_by_file: dict[str, set[str]] = {}
|
|
1779
|
+
for entry in self.doc["entries"]:
|
|
1780
|
+
library = entry.get("source_library") if isinstance(entry, dict) else None
|
|
1781
|
+
if not (isinstance(library, str) and library):
|
|
1782
|
+
continue
|
|
1783
|
+
if library not in self.libraries:
|
|
1784
|
+
self.libraries.append(library)
|
|
1785
|
+
if _file_of(entry):
|
|
1786
|
+
self.libraries_by_file.setdefault(_file_of(entry), set()).add(library)
|
|
1787
|
+
self.library = self.libraries[0] if self.libraries else skill_name
|
|
1788
|
+
# each folder's libraries (the entries under it, "" the root) and _library_at per path, built on first use
|
|
1789
|
+
self._libraries_under: dict[str, set[str]] | None = None
|
|
1790
|
+
self._library_of: dict[str, str | None] = {}
|
|
1791
|
+
# (name, file) of each new entry library_for gave the first library for want of a better one
|
|
1792
|
+
self.library_unresolved: list[tuple[str | None, str | None]] = []
|
|
1496
1793
|
self.added: list[str] = []
|
|
1497
1794
|
self.updated: list[str] = []
|
|
1498
1795
|
self.removed: list[str] = []
|
|
1499
1796
|
self.renamed: list[tuple[str, str]] = [] # (old name, new name)
|
|
1500
1797
|
self.touched_files: set[str] = set()
|
|
1501
1798
|
|
|
1502
|
-
def
|
|
1503
|
-
"""The
|
|
1504
|
-
|
|
1505
|
-
|
|
1799
|
+
def library_for(self, path: str | None, name: str | None = None, *, old: str | None = None) -> str:
|
|
1800
|
+
"""The source_library of a new entry for `name` in `path` (the record of a MOVED file, or of an export
|
|
1801
|
+
whose change names an old_file, passes that old path as `old`, tried second). A map naming one library (or
|
|
1802
|
+
none: --skill-name) gives it; in one naming several, the library of the file, else of the nearest folder
|
|
1803
|
+
(see _library_at). With neither, or no path, the map's first library, and the entry goes to
|
|
1804
|
+
`library_unresolved` for the summary's warning."""
|
|
1805
|
+
if len(self.libraries) <= 1:
|
|
1806
|
+
return self.library
|
|
1807
|
+
for candidate in (path, old):
|
|
1808
|
+
library = self._library_at(_norm_path(candidate))
|
|
1809
|
+
if library is not None:
|
|
1810
|
+
return library
|
|
1811
|
+
self.library_unresolved.append((name, _norm_path(path) or _norm_path(old)))
|
|
1812
|
+
return self.library
|
|
1813
|
+
|
|
1814
|
+
@staticmethod
|
|
1815
|
+
def _root_library(libraries: set[str]) -> str | None:
|
|
1816
|
+
"""The one library of the set, or of several the one that is a path prefix of all the others (`storybook`
|
|
1817
|
+
beside `storybook/test`); None when no library is."""
|
|
1818
|
+
return next((library for library in sorted(libraries)
|
|
1819
|
+
if all(other == library or other.startswith(library + "/") for other in libraries)), None)
|
|
1820
|
+
|
|
1821
|
+
def _library_at(self, path: str | None) -> str | None:
|
|
1822
|
+
"""The library the map's entries of `path` settle (see _root_library), else the one the entries under the
|
|
1823
|
+
nearest folder above it that holds entries settle; None when neither does. Memoized per path."""
|
|
1824
|
+
if not path:
|
|
1825
|
+
return None
|
|
1826
|
+
if path in self._library_of:
|
|
1827
|
+
return self._library_of[path]
|
|
1828
|
+
if self._libraries_under is None:
|
|
1829
|
+
self._libraries_under = {}
|
|
1830
|
+
for file, libraries in self.libraries_by_file.items():
|
|
1831
|
+
folders = file.split("/")[:-1]
|
|
1832
|
+
for depth in range(len(folders) + 1):
|
|
1833
|
+
prefix = "/".join(folders[:depth]) + "/" if depth else ""
|
|
1834
|
+
self._libraries_under.setdefault(prefix, set()).update(libraries)
|
|
1835
|
+
library = self._root_library(self.libraries_by_file.get(path, set()))
|
|
1836
|
+
if library is None:
|
|
1837
|
+
folders = path.split("/")[:-1]
|
|
1838
|
+
for depth in range(len(folders), -1, -1):
|
|
1839
|
+
under = self._libraries_under.get("/".join(folders[:depth]) + "/" if depth else "")
|
|
1840
|
+
if under:
|
|
1841
|
+
library = self._root_library(under)
|
|
1842
|
+
break
|
|
1843
|
+
self._library_of[path] = library
|
|
1844
|
+
return library
|
|
1845
|
+
|
|
1846
|
+
def find(self, name: str, path: str | None, line: int | None = None, *, reexport: bool = False) -> int | None:
|
|
1847
|
+
"""The index of the entry with this name (and file, and line when several share both); with `reexport`,
|
|
1848
|
+
when no entry has the name, of the entry whose reexported_as is the name (a gap names a renamed re-export
|
|
1849
|
+
by its public name)."""
|
|
1850
|
+
def hits_of(key) -> list[int]:
|
|
1851
|
+
return [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and key(e) == name
|
|
1852
|
+
and (path is None or _file_of(e) == path)]
|
|
1853
|
+
|
|
1854
|
+
hits = hits_of(_name_of)
|
|
1855
|
+
if not hits and reexport:
|
|
1856
|
+
hits = hits_of(lambda e: e.get("reexported_as"))
|
|
1506
1857
|
if len(hits) > 1 and line is not None:
|
|
1507
1858
|
hits = [i for i in hits if _line_of(self.doc["entries"][i].get("source_line")) == line]
|
|
1508
1859
|
return hits[0] if len(hits) == 1 else None
|
|
@@ -1516,9 +1867,9 @@ class _Map:
|
|
|
1516
1867
|
if entry.get("source_file"):
|
|
1517
1868
|
self.touched_files.add(_norm_path(entry["source_file"]))
|
|
1518
1869
|
|
|
1519
|
-
def update(self, index: int, fields: dict) -> None:
|
|
1870
|
+
def update(self, index: int, fields: dict, drop: tuple = ()) -> None:
|
|
1520
1871
|
before = self.doc["entries"][index]
|
|
1521
|
-
after = _rewrite(before, fields)
|
|
1872
|
+
after = _rewrite(before, fields, drop)
|
|
1522
1873
|
if after != before:
|
|
1523
1874
|
self.doc["entries"][index] = after
|
|
1524
1875
|
self.updated.append(_name_of(after))
|
|
@@ -1561,7 +1912,7 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
|
|
|
1561
1912
|
not_public.append({"name": name, "file": path})
|
|
1562
1913
|
return True
|
|
1563
1914
|
|
|
1564
|
-
def refresh(index: int | None, name: str, path: str, *, add: bool) -> None:
|
|
1915
|
+
def refresh(index: int | None, name: str, path: str, *, add: bool, old: str | None = None) -> None:
|
|
1565
1916
|
record = fresh.record(name, path)
|
|
1566
1917
|
if record is None:
|
|
1567
1918
|
warnings.append(f"provenance: {name} in {path}: no extraction record, entry "
|
|
@@ -1569,7 +1920,7 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
|
|
|
1569
1920
|
return
|
|
1570
1921
|
if index is None:
|
|
1571
1922
|
if add and not off_surface(name, path):
|
|
1572
|
-
pmap.add(_new_entry(record, pmap.
|
|
1923
|
+
pmap.add(_new_entry(record, pmap.library_for(path, name, old=old)))
|
|
1573
1924
|
return
|
|
1574
1925
|
pmap.update(index, _without_none(record, keep=("ast_node_type",)))
|
|
1575
1926
|
|
|
@@ -1591,7 +1942,7 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
|
|
|
1591
1942
|
if carried is None and off_surface(name, key_path):
|
|
1592
1943
|
continue
|
|
1593
1944
|
entry = _rewrite(carried, _without_none(record, keep=("ast_node_type",))) if carried \
|
|
1594
|
-
else _new_entry(record, pmap.
|
|
1945
|
+
else _new_entry(record, pmap.library_for(key_path, name, old=old))
|
|
1595
1946
|
pmap.add(entry)
|
|
1596
1947
|
continue
|
|
1597
1948
|
if status == "ADDED":
|
|
@@ -1624,9 +1975,13 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
|
|
|
1624
1975
|
if all(record.get(part) is not None for part in ("params", "return_type")
|
|
1625
1976
|
if entry.get(part) is not None):
|
|
1626
1977
|
fields["signature_source"] = record["signature_source"]
|
|
1978
|
+
elif _tier(entry.get("signature_source")) is None:
|
|
1979
|
+
# the signature it keeps was read by eye: set that, or the backfill would derive T1
|
|
1980
|
+
# from the ast-grep label set here
|
|
1981
|
+
fields["signature_source"] = "T1-low"
|
|
1627
1982
|
pmap.update(index, fields)
|
|
1628
1983
|
elif kind in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1629
|
-
refresh(index, name, path, add=True)
|
|
1984
|
+
refresh(index, name, path, add=True, old=_norm_path(change.get("old_file")))
|
|
1630
1985
|
for rename in _get_list(categories or {}, "category_c", "renamed_exports"):
|
|
1631
1986
|
if not isinstance(rename, dict):
|
|
1632
1987
|
continue
|
|
@@ -1663,7 +2018,7 @@ def _apply_docs_only(pmap: _Map, records: dict) -> tuple[int, int]:
|
|
|
1663
2018
|
"confidence": "T3", "signature_source": "T3"}
|
|
1664
2019
|
carried = old.get(_name_of(record))
|
|
1665
2020
|
entry = _rewrite(carried, _without_none(fields)) if carried else _new_entry(
|
|
1666
|
-
_without_none(fields), pmap.
|
|
2021
|
+
_without_none(fields), pmap.library_for(url, _name_of(record)))
|
|
1667
2022
|
pmap.add(entry)
|
|
1668
2023
|
return len(changed), len(exports)
|
|
1669
2024
|
|
|
@@ -1673,8 +2028,9 @@ def _drift_note(drift: tuple | None) -> str:
|
|
|
1673
2028
|
|
|
1674
2029
|
|
|
1675
2030
|
def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, drift: tuple | None,
|
|
1676
|
-
warnings: list[str]) -> tuple[int, int] | list[dict]:
|
|
1677
|
-
"""Gap-driven mode: one write per verification record. A list return is the refusal
|
|
2031
|
+
warnings: list[str], relabeled: list[dict]) -> tuple[int, int] | list[dict]:
|
|
2032
|
+
"""Gap-driven mode: one write per verification record. A list return is the refusal; each entry whose
|
|
2033
|
+
signature_source the repair changes goes to `relabeled`."""
|
|
1678
2034
|
verification = [r for r in records.get("verification") or [] if isinstance(r, dict) and _name_of(r)]
|
|
1679
2035
|
reextracted = _Fresh(records.get("files") or [], None, None)
|
|
1680
2036
|
merged = {}
|
|
@@ -1685,30 +2041,160 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
|
|
|
1685
2041
|
def in_map(r: dict) -> bool:
|
|
1686
2042
|
return bool(r.get("in_map")) or bool(r.get("map_entry"))
|
|
1687
2043
|
|
|
2044
|
+
def signature(r: dict) -> tuple[dict, tuple]:
|
|
2045
|
+
"""(the signature parts to set, the parts to remove) of an export: with the record §4a's extraction gave at
|
|
2046
|
+
its line (its verification record has an extraction_method), every part from it, a null params or
|
|
2047
|
+
return_type removed; with none, merge's."""
|
|
2048
|
+
if r.get("extraction_method") is not None:
|
|
2049
|
+
return _without_none({part: r.get(part) for part in SIGNATURE_PARTS}), \
|
|
2050
|
+
tuple(part for part in ("params", "return_type") if r.get(part) is None)
|
|
2051
|
+
extra = merged.get(_name_of(r), {})
|
|
2052
|
+
return _without_none({part: extra.get(part) for part in SIGNATURE_PARTS}), ()
|
|
2053
|
+
|
|
2054
|
+
def resign(r: dict, index: int, fields: dict) -> None:
|
|
2055
|
+
"""Write an entry the map holds with `fields` and, when §4a found the export's record, its signature; an
|
|
2056
|
+
ast-grep entry takes the record's signature_source, and its node kind when the record's export_type is
|
|
2057
|
+
another (a source-read entry keeps T1-low and no kind); confidence and extraction_method stay."""
|
|
2058
|
+
entry = pmap.doc["entries"][index]
|
|
2059
|
+
drop: tuple = ()
|
|
2060
|
+
method = r.get("extraction_method")
|
|
2061
|
+
if r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT") and method is not None \
|
|
2062
|
+
and r.get("reachability") != "internal-unreachable":
|
|
2063
|
+
parts, gone = signature(r)
|
|
2064
|
+
fields = {**fields, **parts}
|
|
2065
|
+
drop = tuple(part for part in gone if entry.get(part) is not None)
|
|
2066
|
+
if entry.get("extraction_method") in ("ast-grep", "ast_bridge"):
|
|
2067
|
+
fields["signature_source"] = _labels(method, None)["signature_source"]
|
|
2068
|
+
if parts.get("export_type") not in (None, entry.get("export_type")):
|
|
2069
|
+
fields["ast_node_type"] = r.get("ast_node_type")
|
|
2070
|
+
old, new = entry.get("signature_source"), fields["signature_source"]
|
|
2071
|
+
if old is not None and old != new:
|
|
2072
|
+
path = _file_of(entry)
|
|
2073
|
+
relabeled.append({"name": _name_of(entry), "file": path, "from": old, "to": new})
|
|
2074
|
+
warnings.append(f"provenance: {_name_of(entry)} in {path}: signature_source {old} relabeled "
|
|
2075
|
+
f"{new}, from the {method} record of its signature")
|
|
2076
|
+
pmap.update(index, fields, drop)
|
|
2077
|
+
|
|
2078
|
+
note = _drift_note(drift)
|
|
2079
|
+
|
|
2080
|
+
def why(r: dict) -> str:
|
|
2081
|
+
"""The cause of an `unknown` outcome its record gives, `<unknown_reason>: <words>`, else ''."""
|
|
2082
|
+
reason = r.get("unknown_reason")
|
|
2083
|
+
return f"{reason}: {UNKNOWN_REASONS[reason]}" if r.get("verification") == "unknown" \
|
|
2084
|
+
and reason in UNKNOWN_REASONS else ""
|
|
2085
|
+
|
|
2086
|
+
def unchecked(r: dict) -> str | None:
|
|
2087
|
+
"""The warning for an export the map holds that its spot-check could not locate, which stays unchanged: a
|
|
2088
|
+
blocking NEW_EXPORT or MODIFIED_EXPORT (missing, or unknown for a cause its unknown_reason names), or a
|
|
2089
|
+
MOVED_EXPORT whose map entry's file is gone (missing, rule R5), whatever its severity. It gives the gap, the
|
|
2090
|
+
export, the outcome, its cause and the one fix a re-run acts on; None for any other record."""
|
|
2091
|
+
outcome, category = r.get("verification"), r.get("gap_category")
|
|
2092
|
+
gone = outcome == "missing" and category == "MOVED_EXPORT" # R5 cannot point an entry at another file
|
|
2093
|
+
if not in_map(r) or not gone and (
|
|
2094
|
+
outcome not in ("missing", "unknown") or not _blocking(r.get("severity"))
|
|
2095
|
+
or category not in ("NEW_EXPORT", "MODIFIED_EXPORT")):
|
|
2096
|
+
return None
|
|
2097
|
+
name, gap = _name_of(r), r.get("gap_id")
|
|
2098
|
+
path = _location(r.get("map_entry"))[0]
|
|
2099
|
+
cause = f"its map entry's file{f' {path}' if path else ''} is gone" if outcome == "missing" else why(r)
|
|
2100
|
+
lines = r.get("pinned_definition_lines") if gone else None
|
|
2101
|
+
detail = "; ".join(part for part in (
|
|
2102
|
+
cause, note, f"definition lines per the test report: {lines}" if lines else "") if part)
|
|
2103
|
+
if r.get("unknown_reason") == "ambiguous": # a Source at the definition lets the re-run's lookup pick one
|
|
2104
|
+
fix = ("its entries are left unchanged. Fix its entries by hand, or give the gap a `file:line` Source at "
|
|
2105
|
+
"its definition when that file holds only one of them, then re-run update-skill")
|
|
2106
|
+
else: # the re-run checks the same map entry again: only a hand fix changes it
|
|
2107
|
+
fix = ("its entry is left unchanged. Fix the entry by hand (set its source_file and source_line to the "
|
|
2108
|
+
"line that defines it), then re-run update-skill")
|
|
2109
|
+
return f"provenance: {name}{f' ({gap})' if gap else ''}: {outcome}{f' ({detail})' if detail else ''}; {fix}"
|
|
2110
|
+
|
|
2111
|
+
def elsewhere(r: dict) -> str | None:
|
|
2112
|
+
"""The warning for a NEW_EXPORT or MODIFIED_EXPORT whose gap cites another file than the map's entries of its
|
|
2113
|
+
name (unknown, cited-elsewhere), whatever its severity: the gap, the export, the cited file and its check's
|
|
2114
|
+
cause, the record's map_locations and the fix for each side that may be wrong; None for any other record."""
|
|
2115
|
+
if r.get("verification") != "unknown" or r.get("unknown_reason") != "cited-elsewhere":
|
|
2116
|
+
return None
|
|
2117
|
+
name, gap = _name_of(r), r.get("gap_id")
|
|
2118
|
+
cited_file, cited_line = _location(r.get("source_citation"))
|
|
2119
|
+
cited = _where(cited_file, cited_line) or cited_file
|
|
2120
|
+
check = r.get("citation_check") if isinstance(r.get("citation_check"), str) else None
|
|
2121
|
+
words = "no such file" if check == "missing" else OUTSIDE_SCOPE_CHECK if check == "outside-scope" \
|
|
2122
|
+
else UNKNOWN_REASONS.get(check or "")
|
|
2123
|
+
where = [w for w in r.get("map_locations") or [] if isinstance(w, str) and w]
|
|
2124
|
+
detail = "; ".join(part for part in (
|
|
2125
|
+
why(r), f"it cites {cited}" if cited else "",
|
|
2126
|
+
f"its check there: {check}{f' ({words})' if words else ''}" if check else "",
|
|
2127
|
+
f"the map holds it at {', '.join(where)}" if where else "", note) if part)
|
|
2128
|
+
several = len(where) > 1
|
|
2129
|
+
kept = "its entries are left unchanged" if several else "its entry is left unchanged"
|
|
2130
|
+
head = f"provenance: {name}{f' ({gap})' if gap else ''}: unknown ({detail}); {kept}, and nothing is " \
|
|
2131
|
+
"re-extracted for it."
|
|
2132
|
+
if check in UNREAD: # nothing was read at the cited file: make the check read it first
|
|
2133
|
+
return f"{head} Nothing was read at the cited file: {UNREAD[check]}, then re-run update-skill"
|
|
2134
|
+
if check == "outside-scope": # the scope already leaves the cited file out: only a move there brings it in
|
|
2135
|
+
moved = f"when an export moved there, {TAKE_FILE} and fix by hand whichever of those entries moved" \
|
|
2136
|
+
if several else f"when the export moved there, {TAKE_FILE} and fix the map entry by hand"
|
|
2137
|
+
else:
|
|
2138
|
+
moved = "when an export moved, fix by hand whichever of those entries moved" if several \
|
|
2139
|
+
else "when the export moved, fix the map entry by hand"
|
|
2140
|
+
scope = "" if check in ("missing", "outside-scope") else (
|
|
2141
|
+
", and if the gap comes back, exclude the cited file in the brief's scope when the skill does not cover it")
|
|
2142
|
+
return (f"{head} When the map is right, re-run Test Skill, since the test compared another file's "
|
|
2143
|
+
f"declaration{scope}; {moved} (set its source_file and source_line to the line that defines it), "
|
|
2144
|
+
f"then re-run update-skill")
|
|
2145
|
+
|
|
2146
|
+
def scoped_out(r: dict) -> bool:
|
|
2147
|
+
"""True for an `unknown` record whose gap the brief's scope keeps from a repair (outside-scope)."""
|
|
2148
|
+
return r.get("verification") == "unknown" and r.get("unknown_reason") == "outside-scope"
|
|
2149
|
+
|
|
2150
|
+
def outside(r: dict) -> str | None:
|
|
2151
|
+
"""The warning for a NEW_EXPORT or MODIFIED_EXPORT recorded unknown, outside-scope, whatever its severity: the
|
|
2152
|
+
gap, the export, the files the brief's scope leaves out and the fix for each case; None for any other
|
|
2153
|
+
record."""
|
|
2154
|
+
if not scoped_out(r):
|
|
2155
|
+
return None
|
|
2156
|
+
name, gap = _name_of(r), r.get("gap_id")
|
|
2157
|
+
left = [p for p in r.get("outside_scope") or [] if isinstance(p, str) and p]
|
|
2158
|
+
# the cause of a check at an in-scope citation that pinned no line (its file the scope takes)
|
|
2159
|
+
check = r.get("citation_check") if isinstance(r.get("citation_check"), str) else None
|
|
2160
|
+
words = "no such file" if check == "missing" else UNKNOWN_REASONS.get(check or "")
|
|
2161
|
+
cited = _where(*_location(r.get("source_citation"))) or _location(r.get("source_citation"))[0]
|
|
2162
|
+
checked = f"its check there: {check}{f' ({words})' if words else ''}" if check else ""
|
|
2163
|
+
detail = "; ".join(part for part in (
|
|
2164
|
+
why(r), f"left out: {', '.join(left)}" if left else "", f"it cites {cited}" if check and cited else "",
|
|
2165
|
+
checked, note) if part)
|
|
2166
|
+
head = f"provenance: {name}{f' ({gap})' if gap else ''}: unknown ({detail}); nothing is added or re-extracted " \
|
|
2167
|
+
"for it."
|
|
2168
|
+
if check in UNREAD: # nothing was read at the cited file the scope takes: make the check read it first
|
|
2169
|
+
return f"{head} Nothing was read at the cited file: {UNREAD[check]}, then re-run update-skill"
|
|
2170
|
+
covers = "When the skill covers one of those files, change the brief's scope so it takes it" if len(left) > 1 \
|
|
2171
|
+
else "When the skill covers that file, change the brief's scope so it takes it"
|
|
2172
|
+
return (f"{head} {covers} (its scope.include or scope.exclude), then re-run update-skill; otherwise re-run "
|
|
2173
|
+
"Test Skill, since the test compared a declaration the skill does not cover")
|
|
2174
|
+
|
|
1688
2175
|
blocking = [{"export_name": _name_of(r), "severity": r.get("severity")} for r in verification
|
|
1689
|
-
if r.get("verification") == "unknown" and not in_map(r)
|
|
1690
|
-
and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT")
|
|
1691
|
-
and str(r.get("severity") or "").strip().lower() not in NON_BLOCKING]
|
|
2176
|
+
if r.get("verification") == "unknown" and not in_map(r) and not scoped_out(r)
|
|
2177
|
+
and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT") and _blocking(r.get("severity"))]
|
|
1692
2178
|
if blocking:
|
|
1693
2179
|
return blocking
|
|
1694
|
-
note = _drift_note(drift)
|
|
1695
2180
|
pinned_keys: set[tuple] = set()
|
|
1696
2181
|
for r in verification:
|
|
1697
2182
|
name, outcome, category = _name_of(r), r.get("verification"), r.get("gap_category")
|
|
1698
2183
|
entry_file, entry_line = _location(r.get("map_entry"))
|
|
1699
|
-
index = pmap.find(name, entry_file, entry_line) if r.get("map_entry") else None
|
|
2184
|
+
index = pmap.find(name, entry_file, entry_line, reexport=True) if r.get("map_entry") else None
|
|
1700
2185
|
lines = r.get("pinned_definition_lines")
|
|
1701
|
-
|
|
2186
|
+
# rule R5's lines, only when the test report listed any (a citation that did not resolve lists none)
|
|
2187
|
+
r5 = f"; definition lines per the test report: {lines}" if category == "MOVED_EXPORT" and lines else ""
|
|
1702
2188
|
if outcome == "rescoped":
|
|
1703
2189
|
if index is not None:
|
|
1704
2190
|
pmap.remove([index])
|
|
1705
2191
|
elif outcome == "verified" and index is not None:
|
|
1706
2192
|
if drift and category == "MOVED_EXPORT":
|
|
1707
2193
|
warnings.append(f"provenance: {name}: verified at HEAD only ({note}{r5})")
|
|
2194
|
+
resign(r, index, {})
|
|
1708
2195
|
elif outcome == "moved" and index is not None:
|
|
1709
2196
|
path, line = _location(r.get("new_location"))
|
|
1710
|
-
if path and line is not None
|
|
1711
|
-
pmap.update(index, {"source_file": path, "source_line": line})
|
|
2197
|
+
resign(r, index, {"source_file": path, "source_line": line} if path and line is not None else {})
|
|
1712
2198
|
elif outcome == "re-extracted":
|
|
1713
2199
|
if r.get("reachability") == "internal-unreachable":
|
|
1714
2200
|
continue # not public API: merge queued it as a metadata update
|
|
@@ -1717,9 +2203,13 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
|
|
|
1717
2203
|
if record is None:
|
|
1718
2204
|
warnings.append(f"provenance: {name}: re-extracted, but no record of it in the run's files")
|
|
1719
2205
|
continue
|
|
1720
|
-
|
|
2206
|
+
local = (reextracted.worker.get((name, path)) or {}).get("local")
|
|
2207
|
+
if isinstance(local, str) and local and local != name:
|
|
2208
|
+
# matched by the public name a re-export gives it: the entry keeps its declared name
|
|
2209
|
+
record = {**record, "export_name": local, "reexported_as": name}
|
|
2210
|
+
existing = pmap.find(name, path, reexport=True)
|
|
1721
2211
|
if existing is None:
|
|
1722
|
-
pmap.add(_new_entry(record, pmap.
|
|
2212
|
+
pmap.add(_new_entry(record, pmap.library_for(path, name)))
|
|
1723
2213
|
else:
|
|
1724
2214
|
pmap.update(existing, _without_none(record, keep=("ast_node_type",)))
|
|
1725
2215
|
elif outcome in ("verified", "moved") and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
@@ -1727,29 +2217,29 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
|
|
|
1727
2217
|
continue
|
|
1728
2218
|
where = r.get("source_citation") if outcome == "verified" else r.get("new_location")
|
|
1729
2219
|
path, line = _location(where)
|
|
1730
|
-
if not path or line is None or (name, path) in pinned_keys
|
|
2220
|
+
if not path or line is None or (name, path) in pinned_keys \
|
|
2221
|
+
or pmap.find(name, path, reexport=True) is not None:
|
|
1731
2222
|
continue
|
|
1732
2223
|
pinned_keys.add((name, path))
|
|
1733
|
-
|
|
2224
|
+
# the signature its record gives, else merge's; the source-read labels whatever the record's tool
|
|
1734
2225
|
pmap.add(_new_entry(_without_none({
|
|
1735
|
-
"export_name": name,
|
|
1736
|
-
"export_type": extra.get("export_type") or r.get("export_type"),
|
|
1737
|
-
"params": extra.get("params"), "return_type": extra.get("return_type"),
|
|
2226
|
+
"export_name": name, **signature(r)[0],
|
|
1738
2227
|
"source_file": path, "source_line": line, **_labels("source-read", None)},
|
|
1739
|
-
keep=("ast_node_type",)), pmap.
|
|
1740
|
-
elif outcome == "unknown" and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT")
|
|
1741
|
-
|
|
2228
|
+
keep=("ast_node_type",)), pmap.library_for(path, name)))
|
|
2229
|
+
elif outcome == "unknown" and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT") \
|
|
2230
|
+
and not scoped_out(r): # an outside-scope one takes no placeholder: it may be another package's
|
|
1742
2231
|
pmap.add(_new_entry(_without_none({
|
|
1743
|
-
"export_name": name,
|
|
1744
|
-
"
|
|
1745
|
-
**_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library))
|
|
2232
|
+
"export_name": name, **signature(r)[0],
|
|
2233
|
+
**_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library_for(None, name)))
|
|
1746
2234
|
else:
|
|
1747
2235
|
if r.get("unknown_reason") == "drift-override":
|
|
1748
2236
|
warnings.append(f"provenance: {name}: unknown ({note}; no line taken from HEAD{r5})")
|
|
2237
|
+
elif (warning := elsewhere(r) or outside(r) or unchecked(r)) is not None:
|
|
2238
|
+
warnings.append(warning)
|
|
1749
2239
|
elif drift:
|
|
1750
|
-
warnings.append(f"provenance: {name}: {outcome} ({note}{r5})")
|
|
2240
|
+
warnings.append(f"provenance: {name}: {outcome} ({why(r) + '; ' if why(r) else ''}{note}{r5})")
|
|
1751
2241
|
else:
|
|
1752
|
-
warnings.append(f"provenance: {name}: {outcome}{r5}")
|
|
2242
|
+
warnings.append(f"provenance: {name}: {outcome}{f' ({why(r)})' if why(r) else ''}{r5}")
|
|
1753
2243
|
return len(pmap.touched_files), len(verification)
|
|
1754
2244
|
|
|
1755
2245
|
|
|
@@ -1830,6 +2320,9 @@ def exports_after(metadata: object, pmap: _Map, update_type: str) -> list:
|
|
|
1830
2320
|
return []
|
|
1831
2321
|
current = metadata.get("exports") if isinstance(metadata.get("exports"), list) else []
|
|
1832
2322
|
held = {_name_of(e) for e in pmap.doc["entries"] if isinstance(e, dict)} - {None}
|
|
2323
|
+
# an entry added under its declared name, re-exported under a public one: exports[] lists the public name
|
|
2324
|
+
public = {_name_of(e): e["reexported_as"] for e in pmap.doc["entries"]
|
|
2325
|
+
if isinstance(e, dict) and isinstance(e.get("reexported_as"), str) and e["reexported_as"]}
|
|
1833
2326
|
renamed = dict(pmap.renamed)
|
|
1834
2327
|
gone = ({n for n in pmap.removed if n} | set(renamed)) - held
|
|
1835
2328
|
if update_type == "full":
|
|
@@ -1849,8 +2342,8 @@ def exports_after(metadata: object, pmap: _Map, update_type: str) -> list:
|
|
|
1849
2342
|
put(new, new if new in held else None)
|
|
1850
2343
|
else:
|
|
1851
2344
|
put(item, name)
|
|
1852
|
-
for name in pmap.added + [new for _old, new in pmap.renamed]:
|
|
1853
|
-
put(name, name if name in held else None)
|
|
2345
|
+
for name in [public.get(n, n) for n in pmap.added] + [new for _old, new in pmap.renamed]:
|
|
2346
|
+
put(name, name if name in held or name in public.values() else None)
|
|
1854
2347
|
return out
|
|
1855
2348
|
|
|
1856
2349
|
|
|
@@ -1914,6 +2407,50 @@ def file_rows_after(metadata: dict, pmap: _Map, source_root: Path | None) -> dic
|
|
|
1914
2407
|
return lists
|
|
1915
2408
|
|
|
1916
2409
|
|
|
2410
|
+
# the signature_source tiers skf-render-metadata-stats.py bins, in their spelling (it compares them case-insensitively)
|
|
2411
|
+
SIGNATURE_SOURCES = ("T1", "T1-low", "T2", "T3")
|
|
2412
|
+
WARNING_LIST_CAP = 10 # the items a warning lists before "and N more"
|
|
2413
|
+
|
|
2414
|
+
|
|
2415
|
+
def _capped(items: list[str]) -> str:
|
|
2416
|
+
"""The items joined with ", ", the first WARNING_LIST_CAP of them and "and N more" for the rest."""
|
|
2417
|
+
more = len(items) - WARNING_LIST_CAP
|
|
2418
|
+
return ", ".join(items[:WARNING_LIST_CAP] + ([f"and {more} more"] if more > 0 else []))
|
|
2419
|
+
|
|
2420
|
+
|
|
2421
|
+
def _tier(value: object) -> str | None:
|
|
2422
|
+
"""A signature_source or confidence tier in its SIGNATURE_SOURCES spelling; None for any other value."""
|
|
2423
|
+
if not isinstance(value, str):
|
|
2424
|
+
return None
|
|
2425
|
+
return next((tier for tier in SIGNATURE_SOURCES if tier.upper() == value.strip().upper()), None)
|
|
2426
|
+
|
|
2427
|
+
|
|
2428
|
+
def _backfill_signature_source(pmap: _Map) -> tuple[dict[str, int], list[tuple[object, object, str]]]:
|
|
2429
|
+
"""Give each entry whose signature_source is missing or not a tier the stats helper bins (a map written before
|
|
2430
|
+
the key existed, #802) the one its labels give: its confidence when that names T1-low, T2 or T3, else T1 for
|
|
2431
|
+
an extraction_method of ast-grep or ast_bridge (compared stripped and case-insensitively) and T1-low for any
|
|
2432
|
+
other (relabel-rule.md: T1 needs evidence an ast-grep rule matched). Returns (the fills counted by value, in
|
|
2433
|
+
tier order; (name, old value, new value) of each unrecognized value replaced, which the map no longer holds);
|
|
2434
|
+
the names are not listed as updated."""
|
|
2435
|
+
counts: dict[str, int] = {}
|
|
2436
|
+
replaced: list[tuple[object, object, str]] = []
|
|
2437
|
+
for entry in pmap.doc["entries"]:
|
|
2438
|
+
if not isinstance(entry, dict) or _tier(entry.get("signature_source")) is not None:
|
|
2439
|
+
continue
|
|
2440
|
+
confidence, method = _tier(entry.get("confidence")), entry.get("extraction_method")
|
|
2441
|
+
if confidence in ("T1-low", "T2", "T3"):
|
|
2442
|
+
value = confidence
|
|
2443
|
+
elif isinstance(method, str) and method.strip().lower() in ("ast-grep", "ast_bridge"):
|
|
2444
|
+
value = "T1"
|
|
2445
|
+
else:
|
|
2446
|
+
value = "T1-low"
|
|
2447
|
+
if entry.get("signature_source") is not None:
|
|
2448
|
+
replaced.append((entry.get("export_name"), entry["signature_source"], value))
|
|
2449
|
+
entry["signature_source"] = value
|
|
2450
|
+
counts[value] = counts.get(value, 0) + 1
|
|
2451
|
+
return {tier: counts[tier] for tier in SIGNATURE_SOURCES if tier in counts}, replaced
|
|
2452
|
+
|
|
2453
|
+
|
|
1917
2454
|
def apply_update(*, update_type: str, provenance: dict | None, skill_name: str, manifest: dict | None = None,
|
|
1918
2455
|
categories: dict | None = None, extraction: dict | None = None, details: dict | None = None,
|
|
1919
2456
|
records: dict | None = None, merge_records: dict | None = None, compare: dict | None = None,
|
|
@@ -1936,9 +2473,10 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
|
|
|
1936
2473
|
pmap = _Map(provenance, skill_name)
|
|
1937
2474
|
warnings: list[str] = []
|
|
1938
2475
|
not_public: list[dict] = []
|
|
2476
|
+
relabeled: list[dict] = []
|
|
1939
2477
|
mode = records.get("mode")
|
|
1940
2478
|
if update_type == "gap-driven":
|
|
1941
|
-
outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings)
|
|
2479
|
+
outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings, relabeled)
|
|
1942
2480
|
if isinstance(outcome, list):
|
|
1943
2481
|
return {}, {"status": "refused", "blocking_unresolved": outcome}
|
|
1944
2482
|
files, exports = outcome
|
|
@@ -1951,7 +2489,7 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
|
|
|
1951
2489
|
index = pmap.find(name, path)
|
|
1952
2490
|
record = fresh.record(name, path)
|
|
1953
2491
|
if index is None:
|
|
1954
|
-
pmap.add(_new_entry(record, pmap.
|
|
2492
|
+
pmap.add(_new_entry(record, pmap.library_for(path, name)))
|
|
1955
2493
|
else:
|
|
1956
2494
|
pmap.update(index, _without_none(record, keep=("ast_node_type",)))
|
|
1957
2495
|
files = len({path for _name, path in fresh.all_keys()})
|
|
@@ -1963,6 +2501,23 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
|
|
|
1963
2501
|
f"({fresh.surface_problem}): every new export was added")
|
|
1964
2502
|
if not_public:
|
|
1965
2503
|
warnings.append(_not_public_warning(not_public))
|
|
2504
|
+
if pmap.library_unresolved:
|
|
2505
|
+
count = len(pmap.library_unresolved)
|
|
2506
|
+
named = _capped([f"{name} in {path}" if path else f"{name} (no file)"
|
|
2507
|
+
for name, path in pmap.library_unresolved])
|
|
2508
|
+
warnings.append(f"provenance: {count} new entr{'y' if count == 1 else 'ies'} took source_library "
|
|
2509
|
+
f"{pmap.library}, the map's first, as the map names several libraries "
|
|
2510
|
+
f"({', '.join(pmap.libraries)}) and each entry's file or folder does not settle which: "
|
|
2511
|
+
f"{named}; check each entry's source_library")
|
|
2512
|
+
backfilled, replaced = _backfill_signature_source(pmap)
|
|
2513
|
+
if backfilled:
|
|
2514
|
+
count = sum(backfilled.values())
|
|
2515
|
+
listed = [f"{name}: {old if isinstance(old, str) else json.dumps(old)} -> {new}" for name, old, new in replaced]
|
|
2516
|
+
values = f"; unrecognized values replaced: {_capped(listed)}" if replaced else ""
|
|
2517
|
+
warnings.append(f"provenance: backfilled signature_source on {count} entr{'y' if count == 1 else 'ies'} "
|
|
2518
|
+
f"that had none or an unrecognized value (a map written before the key existed), from each "
|
|
2519
|
+
f"one's confidence and extraction_method: "
|
|
2520
|
+
f"{', '.join(f'{n} {value}' for value, n in backfilled.items())}{values}")
|
|
1966
2521
|
file_changes = _apply_files(pmap, compare, new_files, promoted, source_root)
|
|
1967
2522
|
doc = pmap.doc
|
|
1968
2523
|
for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
|
|
@@ -1983,6 +2538,8 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
|
|
|
1983
2538
|
"status": "written",
|
|
1984
2539
|
"entries": {"added": pmap.added, "updated": pmap.updated, "removed": pmap.removed},
|
|
1985
2540
|
"not_public": not_public,
|
|
2541
|
+
"relabeled": relabeled,
|
|
2542
|
+
"signature_source_backfilled": backfilled,
|
|
1986
2543
|
"file_entries": file_changes,
|
|
1987
2544
|
"warnings": warnings,
|
|
1988
2545
|
}
|
|
@@ -1997,11 +2554,98 @@ GENERATION_DATE_RE = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ")
|
|
|
1997
2554
|
STATS_HELPER_FIELDS = ("exports_documented", "exports_public_api", "exports_internal", "exports_total",
|
|
1998
2555
|
"public_api_coverage", "total_coverage", "scripts_count", "assets_count",
|
|
1999
2556
|
"pattern_surfaces_documented", "effective_denominator")
|
|
2557
|
+
# the other stats fields create-skill writes, by the skill's scope_type: a component library's own counts (the
|
|
2558
|
+
# Component Library stats of compile-assembly-rules.md, its stats_extra) and a reference app's notes
|
|
2559
|
+
STATS_CREATE_FIELDS = {
|
|
2560
|
+
"component-library": ("components_registered", "components_documented", "props_interfaces_extracted",
|
|
2561
|
+
"components_unique", "demo_files_excluded", "design_variants"),
|
|
2562
|
+
"reference-app": ("notes",),
|
|
2563
|
+
}
|
|
2564
|
+
# a stats key that explains effective_denominator, dropped when this run's denominator differs from the old one
|
|
2565
|
+
DENOMINATOR_NOTE_PREFIX = "effective_denominator_"
|
|
2566
|
+
SKF_NO_ROOT = ("metadata: tool_versions.skf is unknown: no --skf-root was given; pass the installed SKF's folder "
|
|
2567
|
+
"({project-root}/_bmad/skf)")
|
|
2568
|
+
AST_GREP_UNKNOWN = ("metadata: tool_versions.ast_grep is unknown: --extraction holds exports ast-grep matched but no "
|
|
2569
|
+
"ast_grep.version")
|
|
2570
|
+
_QMD_PREFIX = re.compile(r"^qmd\s+", re.IGNORECASE) # what `qmd --version` prints first
|
|
2571
|
+
|
|
2572
|
+
|
|
2573
|
+
def _tool_version(value: object) -> str | None:
|
|
2574
|
+
return value.strip() if isinstance(value, str) and value.strip() else None
|
|
2575
|
+
|
|
2576
|
+
|
|
2577
|
+
def _skf_unknown(skf_root: str | Path) -> str:
|
|
2578
|
+
return (f"metadata: tool_versions.skf is unknown: {Path(skf_root).as_posix()} holds no package.json or VERSION "
|
|
2579
|
+
f"file with a version")
|
|
2580
|
+
|
|
2581
|
+
|
|
2582
|
+
def _kept_warning(key: str) -> str:
|
|
2583
|
+
return f"metadata: stats.{key} kept from an earlier run; check it still describes this run's counts"
|
|
2584
|
+
|
|
2585
|
+
|
|
2586
|
+
def tool_versions_after(old: object, *, skf_version: str | None, skf_root: str | Path | None,
|
|
2587
|
+
ast_grep_version: str | None, qmd_version: str | None) -> tuple[dict, list[str]]:
|
|
2588
|
+
"""(the tool_versions an update leaves, its warnings) (see "Metadata" in the module docstring)."""
|
|
2589
|
+
warnings: list[str] = []
|
|
2590
|
+
if isinstance(old, dict):
|
|
2591
|
+
tools = dict(old)
|
|
2592
|
+
else:
|
|
2593
|
+
tools = {"ast_grep": None, "qmd": None}
|
|
2594
|
+
if old is not None:
|
|
2595
|
+
kind = {bool: "a boolean", str: "a string", list: "an array"}.get(type(old), "a number")
|
|
2596
|
+
warnings.append(f"metadata: tool_versions was {kind}, not an object; replaced with the versions this run "
|
|
2597
|
+
f"records")
|
|
2598
|
+
ast_grep = _tool_version(ast_grep_version)
|
|
2599
|
+
if ast_grep is not None:
|
|
2600
|
+
tools["ast_grep"] = ast_grep
|
|
2601
|
+
if ast_grep == "unknown":
|
|
2602
|
+
warnings.append(AST_GREP_UNKNOWN)
|
|
2603
|
+
qmd = _tool_version(qmd_version)
|
|
2604
|
+
if qmd is not None:
|
|
2605
|
+
qmd = _tool_version(_QMD_PREFIX.sub("", qmd, count=1))
|
|
2606
|
+
if qmd is not None:
|
|
2607
|
+
tools["qmd"] = qmd
|
|
2608
|
+
skf = _tool_version(skf_version)
|
|
2609
|
+
if skf is None or skf == "unknown":
|
|
2610
|
+
skf = "unknown"
|
|
2611
|
+
warnings.append(SKF_NO_ROOT if skf_root is None else _skf_unknown(skf_root))
|
|
2612
|
+
tools["skf"] = skf
|
|
2613
|
+
return tools, warnings
|
|
2614
|
+
|
|
2615
|
+
|
|
2616
|
+
def stats_after(old_stats: dict, new_stats: dict, scope_type: object) -> tuple[dict, list[str], list[str]]:
|
|
2617
|
+
"""(the stats an update leaves, dropped_stats, their warnings) (see "Metadata" in the module docstring)."""
|
|
2618
|
+
exempt = STATS_CREATE_FIELDS.get(scope_type, ()) if isinstance(scope_type, str) else ()
|
|
2619
|
+
denominator = new_stats.get("effective_denominator")
|
|
2620
|
+
recounted = old_stats.get("effective_denominator") != denominator
|
|
2621
|
+
kept: dict = {}
|
|
2622
|
+
dropped: list[str] = []
|
|
2623
|
+
warnings: list[str] = []
|
|
2624
|
+
for key, value in old_stats.items():
|
|
2625
|
+
if key in new_stats:
|
|
2626
|
+
continue
|
|
2627
|
+
if key in STATS_HELPER_FIELDS:
|
|
2628
|
+
dropped.append(key)
|
|
2629
|
+
warnings.append(f"metadata: stats.{key} dropped: this run's stats helper did not write it")
|
|
2630
|
+
continue
|
|
2631
|
+
if key.startswith(DENOMINATOR_NOTE_PREFIX) and recounted:
|
|
2632
|
+
dropped.append(key)
|
|
2633
|
+
warnings.append(f"metadata: stats.{key} dropped: it explained an effective_denominator this run changed "
|
|
2634
|
+
f"or removed")
|
|
2635
|
+
continue
|
|
2636
|
+
kept[key] = value
|
|
2637
|
+
# an explanation of a denominator this run left as it was still describes it
|
|
2638
|
+
if key not in exempt and not (key.startswith(DENOMINATOR_NOTE_PREFIX) and denominator is not None):
|
|
2639
|
+
warnings.append(_kept_warning(key))
|
|
2640
|
+
return {**new_stats, **kept}, dropped, warnings
|
|
2000
2641
|
|
|
2001
2642
|
|
|
2002
2643
|
def update_metadata(metadata: object, lists: object, stats: object, *, version: str, generation_date: str,
|
|
2003
|
-
source_commit: str | None = None, source_ref: str | None = None
|
|
2004
|
-
|
|
2644
|
+
source_commit: str | None = None, source_ref: str | None = None,
|
|
2645
|
+
skf_version: str | None = None, skf_root: str | Path | None = None,
|
|
2646
|
+
ast_grep_version: str | None = None, qmd_version: str | None = None) -> tuple[dict, dict]:
|
|
2647
|
+
"""(the metadata.json an update leaves, {"dropped_stats": [...], "warnings": [...]}) (see "Metadata" in the
|
|
2648
|
+
module docstring). Raises ValueError."""
|
|
2005
2649
|
if not isinstance(metadata, dict):
|
|
2006
2650
|
raise ValueError("--metadata file must hold a JSON object (the skill's metadata.json)")
|
|
2007
2651
|
if not isinstance(lists, dict) or not all(isinstance(lists.get(k), list) for k in ("exports", "scripts",
|
|
@@ -2033,11 +2677,14 @@ def update_metadata(metadata: object, lists: object, stats: object, *, version:
|
|
|
2033
2677
|
out[kind] = lists[kind]
|
|
2034
2678
|
else:
|
|
2035
2679
|
out.pop(kind, None)
|
|
2680
|
+
out["tool_versions"], warnings = tool_versions_after(
|
|
2681
|
+
metadata.get("tool_versions"), skf_version=skf_version, skf_root=skf_root,
|
|
2682
|
+
ast_grep_version=ast_grep_version, qmd_version=qmd_version)
|
|
2036
2683
|
old_stats = metadata.get("stats") if isinstance(metadata.get("stats"), dict) else {}
|
|
2037
|
-
|
|
2038
|
-
|
|
2684
|
+
out["stats"], dropped, stats_warnings = stats_after(old_stats, stats["stats"], metadata.get("scope_type"))
|
|
2685
|
+
warnings += stats_warnings
|
|
2039
2686
|
out["confidence_distribution"] = stats["confidence_distribution"]
|
|
2040
|
-
return out
|
|
2687
|
+
return out, {"dropped_stats": dropped, "warnings": warnings}
|
|
2041
2688
|
|
|
2042
2689
|
|
|
2043
2690
|
NOT_PUBLIC_NAMED = 10 # the names the not_public warning spells out; the summary lists them all
|
|
@@ -2177,6 +2824,20 @@ def build_records(extraction: dict | None, details: dict | None, files: list | N
|
|
|
2177
2824
|
if patch.get(field) is not None:
|
|
2178
2825
|
seeded[key][field] = patch[field]
|
|
2179
2826
|
fill_if_null(key, patch)
|
|
2827
|
+
diff = extraction.get("entry_point_diff") if isinstance(extraction, dict) else None
|
|
2828
|
+
aliases: dict[tuple, set] = {}
|
|
2829
|
+
rows = [*(diff.get("public") or []), *(diff.get("other_definitions") or [])] if isinstance(diff, dict) else []
|
|
2830
|
+
for item in rows:
|
|
2831
|
+
members = item.get("members") if isinstance(item, dict) and item.get("via") == "namespace" else None
|
|
2832
|
+
for named in [item, *(members if isinstance(members, list) else [])]:
|
|
2833
|
+
if not isinstance(named, dict):
|
|
2834
|
+
continue
|
|
2835
|
+
name, local = named.get("name"), named.get("local")
|
|
2836
|
+
key = (local, _norm_path(named.get("file")))
|
|
2837
|
+
if isinstance(name, str) and name and isinstance(local, str) and local != name and key in seeded:
|
|
2838
|
+
aliases.setdefault(key, set()).add(name) # the public name the runner gives `local` in that file
|
|
2839
|
+
for key, names in aliases.items():
|
|
2840
|
+
seeded[key]["aliases"] = sorted(names)
|
|
2180
2841
|
surface, _problem = public_surface(extraction) # apply warns when a public-api run gives no surface
|
|
2181
2842
|
for key in seeded if surface is not None else ():
|
|
2182
2843
|
mark = surface.public(key[0], key[1], languages.get(key))
|
|
@@ -2217,6 +2878,32 @@ DRIFT_IN_MAP = "a signature from the tree"
|
|
|
2217
2878
|
REACHABILITY = ("public", "internal-unreachable")
|
|
2218
2879
|
VERIFIER_MISSING = "provenance: spot-checks not run: skf-verify-provenance-completeness.py is missing; re-install SKF"
|
|
2219
2880
|
OUTCOMES = ("verified", "moved", "missing", "re-extracted", "rescoped", "unknown")
|
|
2881
|
+
# Why an entry ends `unknown` (its unknown_reason, see "Gap records"), with the words apply's warning gives it.
|
|
2882
|
+
UNKNOWN_REASONS = {
|
|
2883
|
+
"ambiguous": "several map entries of its name",
|
|
2884
|
+
"no-definition": "no line of its file defines it, by the definition-line rules",
|
|
2885
|
+
"several-definitions": "several lines of its file define it, none the recorded one",
|
|
2886
|
+
"unchecked-language": "the definition-line rules cover no language of its file, and no line was answered as "
|
|
2887
|
+
"declaring it",
|
|
2888
|
+
"no-line": "its map entry gives no file or line to check",
|
|
2889
|
+
"spot-check-failed": "the spot-check failed",
|
|
2890
|
+
"verifier-missing": "skf-verify-provenance-completeness.py is missing",
|
|
2891
|
+
"no-source-root": "no source tree to check",
|
|
2892
|
+
"drift-override": "the drift override kept its line from being moved",
|
|
2893
|
+
"cited-elsewhere": "the gap cites another file than the provenance map's entries of its name",
|
|
2894
|
+
"outside-scope": "the brief's scope leaves out a file the gap cites or its Remediation names, so the export is "
|
|
2895
|
+
"not read there",
|
|
2896
|
+
}
|
|
2897
|
+
# A cited-elsewhere entry's citation_check when the brief's scope leaves the cited file out, so no check ran there.
|
|
2898
|
+
OUTSIDE_SCOPE_CHECK = "not run: the brief's scope leaves it out"
|
|
2899
|
+
# The fix for a file the brief's scope leaves out that the skill covers: an include may miss it, or an exclude take it.
|
|
2900
|
+
TAKE_FILE = "change the brief's scope so it takes that file (its scope.include or scope.exclude)"
|
|
2901
|
+
# A cited-elsewhere entry's citation_check that read nothing at the cited file, with the fix that lets a re-run read it.
|
|
2902
|
+
UNREAD = {
|
|
2903
|
+
"no-source-root": "give update-skill the source tree",
|
|
2904
|
+
"verifier-missing": "re-install SKF",
|
|
2905
|
+
"spot-check-failed": "fix what its spot-check failed on, which the `spot-check failed` warning names",
|
|
2906
|
+
}
|
|
2220
2907
|
_DEFINITION_LINES = re.compile(r"definition lines?\s*\(([\d,\s]+)\)", re.IGNORECASE)
|
|
2221
2908
|
_VERIFIER = None
|
|
2222
2909
|
|
|
@@ -2240,6 +2927,95 @@ def _verifier():
|
|
|
2240
2927
|
return _VERIFIER or None
|
|
2241
2928
|
|
|
2242
2929
|
|
|
2930
|
+
CLASSIFIER_FILE = "skf-classify-changed-files.py" # load_scope: the brief's scope (--brief)
|
|
2931
|
+
|
|
2932
|
+
|
|
2933
|
+
def _brief_scope(path: Path, cited: set[str] | frozenset = frozenset()):
|
|
2934
|
+
"""The skill brief's scope by skf-classify-changed-files.py's load_scope, loaded from this folder: its
|
|
2935
|
+
takes(path) is Category A's in-scope test (an exclude wins over an include; an empty include takes a path only
|
|
2936
|
+
when its suffix is the suffix of a `cited` path, the files the gaps and the map's entries name, or of the brief's
|
|
2937
|
+
language, so a file with no suffix is never taken, and an exclude still leaves a path out).
|
|
2938
|
+
Raises ValueError, on one line, for a brief that names no file or cannot be read, and for a scope test that
|
|
2939
|
+
cannot be loaded."""
|
|
2940
|
+
try:
|
|
2941
|
+
spec = importlib.util.spec_from_file_location(
|
|
2942
|
+
"skf_classify_changed_files_for_gap_records", Path(__file__).resolve().parent / CLASSIFIER_FILE)
|
|
2943
|
+
if spec is None or spec.loader is None:
|
|
2944
|
+
raise ImportError(f"{CLASSIFIER_FILE} not found")
|
|
2945
|
+
classifier = importlib.util.module_from_spec(spec)
|
|
2946
|
+
spec.loader.exec_module(classifier)
|
|
2947
|
+
except Exception as exc: # noqa: BLE001 - a classifier (or the resolver it loads) missing or failing
|
|
2948
|
+
raise ValueError(f"cannot load the brief's scope test, {CLASSIFIER_FILE} beside "
|
|
2949
|
+
f"{Path(__file__).name}: {exc}; re-install SKF") from exc
|
|
2950
|
+
try:
|
|
2951
|
+
return classifier.load_scope(path, cited)
|
|
2952
|
+
except ValueError as exc: # a brief it cannot find or read (a YAML error spans several lines)
|
|
2953
|
+
raise ValueError(re.sub(r"\s*[\r\n]\s*", " ", str(exc).strip())) from exc
|
|
2954
|
+
except Exception as exc: # noqa: BLE001 - an out-of-date classifier, or a resolver beside it that fails
|
|
2955
|
+
raise ValueError(f"cannot load the brief's scope test, {CLASSIFIER_FILE} beside "
|
|
2956
|
+
f"{Path(__file__).name}: {exc}; re-install SKF") from exc
|
|
2957
|
+
|
|
2958
|
+
|
|
2959
|
+
def _takes(scope, path: str) -> bool:
|
|
2960
|
+
"""True when the brief's scope takes `path`. Raises ValueError when its test fails (the resolver it calls cannot
|
|
2961
|
+
be loaded or is out of date)."""
|
|
2962
|
+
try:
|
|
2963
|
+
return bool(scope.takes(path))
|
|
2964
|
+
except Exception as exc: # noqa: BLE001 - whatever the scope test raises, the run reports it on one line
|
|
2965
|
+
raise ValueError(f"the brief's scope test failed on {path}: {exc}; re-install SKF") from exc
|
|
2966
|
+
|
|
2967
|
+
|
|
2968
|
+
def _scope_rel(path: object, root: Path | None) -> str | None:
|
|
2969
|
+
"""`path` as the scope test reads it: POSIX, relative to the source root (an absolute path under it loses the
|
|
2970
|
+
root's prefix), its `.` and `..` parts resolved lexically (posixpath.normpath); None for no path."""
|
|
2971
|
+
text = _norm_path(path)
|
|
2972
|
+
if text is None:
|
|
2973
|
+
return None
|
|
2974
|
+
if root is not None:
|
|
2975
|
+
bases = [root.as_posix()]
|
|
2976
|
+
try:
|
|
2977
|
+
bases.append(root.resolve().as_posix())
|
|
2978
|
+
except OSError:
|
|
2979
|
+
pass
|
|
2980
|
+
for base in dict.fromkeys(b.rstrip("/") for b in bases if b.rstrip("/")):
|
|
2981
|
+
if text.startswith(base + "/"):
|
|
2982
|
+
text = text[len(base) + 1:]
|
|
2983
|
+
break
|
|
2984
|
+
return posixpath.normpath(text)
|
|
2985
|
+
|
|
2986
|
+
|
|
2987
|
+
def _scope_test(scope, provenance: dict | None, root: Path | None):
|
|
2988
|
+
"""The test gap-records applies to each file a gap cites or its Remediation names: True for a file the
|
|
2989
|
+
provenance map's entries[] cite (Category A takes each file the map tracks whatever the scope says) and for one
|
|
2990
|
+
the brief's scope takes, each path read by _scope_rel."""
|
|
2991
|
+
tracked = {_scope_rel(_file_of(e), root) for e in _entries_of(provenance)} - {None}
|
|
2992
|
+
|
|
2993
|
+
def takes(path: str) -> bool:
|
|
2994
|
+
rel = _scope_rel(path, root)
|
|
2995
|
+
return rel is not None and (rel in tracked or _takes(scope, rel))
|
|
2996
|
+
|
|
2997
|
+
return takes
|
|
2998
|
+
|
|
2999
|
+
|
|
3000
|
+
def _map_paths(provenance: object) -> set[str]:
|
|
3001
|
+
"""The files the provenance map's entries[] cite (load_scope's `cited`, with the gaps' paths)."""
|
|
3002
|
+
return {p for p in (_file_of(e) for e in _entries_of(provenance)) if p}
|
|
3003
|
+
|
|
3004
|
+
|
|
3005
|
+
def _gap_paths(manifest: object) -> set[str]:
|
|
3006
|
+
"""Every file a manifest's entries name: each citation's file and resolved path (load_scope's `cited`)."""
|
|
3007
|
+
entries = manifest.get("entries") if isinstance(manifest, dict) else None
|
|
3008
|
+
paths = set()
|
|
3009
|
+
for entry in entries if isinstance(entries, list) else []:
|
|
3010
|
+
if not isinstance(entry, dict):
|
|
3011
|
+
continue
|
|
3012
|
+
citation = entry.get("source_citation") if isinstance(entry.get("source_citation"), dict) else {}
|
|
3013
|
+
for path in [citation.get("file"), *(entry.get("resolved_paths") or [])]:
|
|
3014
|
+
if _norm_path(path):
|
|
3015
|
+
paths.add(_norm_path(path))
|
|
3016
|
+
return paths
|
|
3017
|
+
|
|
3018
|
+
|
|
2243
3019
|
def _blocking(severity: object) -> bool:
|
|
2244
3020
|
return str(severity or "").strip().lower() not in NON_BLOCKING
|
|
2245
3021
|
|
|
@@ -2268,40 +3044,50 @@ def _drift_reason(entry: dict) -> str | None:
|
|
|
2268
3044
|
return None
|
|
2269
3045
|
if entry.get("provenance_completeness"):
|
|
2270
3046
|
return DRIFT_COMPLETENESS
|
|
2271
|
-
|
|
2272
|
-
|
|
3047
|
+
match = entry.get("map_match") if isinstance(entry.get("map_match"), dict) else {}
|
|
3048
|
+
status = match.get("status")
|
|
3049
|
+
# a not-found with a candidate that has a file is a citation of another file than the map's entries: held (#824)
|
|
3050
|
+
held = status in ("found", "ambiguous") or status == "not-found" and any(
|
|
3051
|
+
isinstance(c, dict) and _norm_path(c.get("source_file")) for c in match.get("candidates") or [])
|
|
3052
|
+
return DRIFT_IN_MAP if held else DRIFT_NOT_IN_MAP
|
|
2273
3053
|
|
|
2274
3054
|
|
|
2275
3055
|
def _spot_check(root: Path | None, path: str | None, name: str, line: int | None, export_type: object,
|
|
2276
|
-
answers: dict) -> tuple[str, int | None, bool, str | None]:
|
|
3056
|
+
answers: dict, fold: bool = False) -> tuple[str, int | None, bool, str | None, str | None]:
|
|
2277
3057
|
"""(outcome, the line a `moved` export moves to, whether it waits on a `declaring_line` answer, why the
|
|
2278
|
-
check failed).
|
|
3058
|
+
check failed, why it is `unknown`: an UNKNOWN_REASONS key). `fold` (rule R5's MOVED_EXPORT only) checks a
|
|
3059
|
+
bookkeeping variant its file never names against its base; any other gap judges the name as written, so a
|
|
3060
|
+
fabricated variant is never located at its base's line."""
|
|
3061
|
+
if root is None:
|
|
3062
|
+
return "unknown", None, False, None, "no-source-root"
|
|
2279
3063
|
verifier = _verifier()
|
|
2280
|
-
if verifier is None
|
|
2281
|
-
return "unknown", None, False, None
|
|
3064
|
+
if verifier is None:
|
|
3065
|
+
return "unknown", None, False, None, "verifier-missing"
|
|
3066
|
+
if not path or line is None:
|
|
3067
|
+
return "unknown", None, False, None, "no-line"
|
|
2282
3068
|
try:
|
|
2283
3069
|
report = verifier.definition_lines_report(path, name, root, line,
|
|
2284
|
-
export_type if isinstance(export_type, str) else None)
|
|
3070
|
+
export_type if isinstance(export_type, str) else None, fold)
|
|
2285
3071
|
except Exception as exc: # noqa: BLE001 - a file it cannot read, or any verifier failure: this entry only
|
|
2286
|
-
return "unknown", None, False, f"{type(exc).__name__}: {exc}"
|
|
3072
|
+
return "unknown", None, False, f"{type(exc).__name__}: {exc}", "spot-check-failed"
|
|
2287
3073
|
check = report.get("line_check")
|
|
2288
3074
|
if check == "file-missing":
|
|
2289
|
-
return "missing", None, False, None
|
|
3075
|
+
return "missing", None, False, None, None
|
|
2290
3076
|
if check == "skipped-export-type":
|
|
2291
|
-
return "verified", None, False, None
|
|
3077
|
+
return "verified", None, False, None, None
|
|
2292
3078
|
if check == "skipped-language":
|
|
2293
3079
|
if "declaring_line" not in answers:
|
|
2294
|
-
return "unknown", None, True, None
|
|
3080
|
+
return "unknown", None, True, None, None
|
|
2295
3081
|
declared = answers["declaring_line"]
|
|
2296
3082
|
if declared is None:
|
|
2297
|
-
return "unknown", None, False, None
|
|
2298
|
-
return ("verified", None, False, None) if declared == line else ("moved", declared, False, None)
|
|
3083
|
+
return "unknown", None, False, None, "unchecked-language"
|
|
3084
|
+
return ("verified", None, False, None, None) if declared == line else ("moved", declared, False, None, None)
|
|
2299
3085
|
if report.get("line_is_definition"):
|
|
2300
|
-
return "verified", None, False, None
|
|
3086
|
+
return "verified", None, False, None, None
|
|
2301
3087
|
defs = report.get("definition_lines") or []
|
|
2302
3088
|
if len(defs) == 1 and defs[0] != line:
|
|
2303
|
-
return "moved", defs[0], False, None
|
|
2304
|
-
return "unknown", None, False, None
|
|
3089
|
+
return "moved", defs[0], False, None, None
|
|
3090
|
+
return "unknown", None, False, None, "several-definitions" if defs else "no-definition"
|
|
2305
3091
|
|
|
2306
3092
|
|
|
2307
3093
|
def _pinned_lines(remediation: object) -> list[int] | None:
|
|
@@ -2311,6 +3097,18 @@ def _pinned_lines(remediation: object) -> list[int] | None:
|
|
|
2311
3097
|
return lines or None
|
|
2312
3098
|
|
|
2313
3099
|
|
|
3100
|
+
def _inside(root: Path | None, path: str | None) -> bool:
|
|
3101
|
+
"""True for a relative path that stays under the source root, links resolved."""
|
|
3102
|
+
rel = Path(path) if path else None
|
|
3103
|
+
if root is None or rel is None or rel.is_absolute() or ".." in rel.parts:
|
|
3104
|
+
return False
|
|
3105
|
+
try:
|
|
3106
|
+
(root / rel).resolve().relative_to(root.resolve())
|
|
3107
|
+
except (OSError, ValueError):
|
|
3108
|
+
return False
|
|
3109
|
+
return True
|
|
3110
|
+
|
|
3111
|
+
|
|
2314
3112
|
def _where(path: object, line: object) -> str | None:
|
|
2315
3113
|
file, number = _norm_path(path), _line_of(line)
|
|
2316
3114
|
return f"{file}:{number}" if file and number is not None else None
|
|
@@ -2319,7 +3117,7 @@ def _where(path: object, line: object) -> str | None:
|
|
|
2319
3117
|
class _GapEntry:
|
|
2320
3118
|
"""One export-bearing manifest entry as gap-records routes it."""
|
|
2321
3119
|
|
|
2322
|
-
def __init__(self, entry: dict, answers: dict):
|
|
3120
|
+
def __init__(self, entry: dict, answers: dict, takes=None):
|
|
2323
3121
|
self.entry = entry
|
|
2324
3122
|
self.answers = answers
|
|
2325
3123
|
self.name = _name_of(entry) or ""
|
|
@@ -2330,16 +3128,32 @@ class _GapEntry:
|
|
|
2330
3128
|
self.map_entry = {"source_file": found.get("source_file"), "source_line": found.get("source_line")} \
|
|
2331
3129
|
if found else None
|
|
2332
3130
|
self.map_export_type = found.get("export_type") if found else None
|
|
3131
|
+
# the map entry's own name when skf-parse-gaps matched it by its reexported_as (the gap names the public one)
|
|
3132
|
+
self.declared = _name_of(found) if found and _name_of(found) else self.name
|
|
2333
3133
|
self.in_map = bool(match.get("candidates"))
|
|
3134
|
+
# where map_match's candidates sit, `file:line` (or the file); a placeholder with no file names none
|
|
3135
|
+
self.map_locations = list(dict.fromkeys(
|
|
3136
|
+
_where(c.get("source_file"), c.get("source_line")) or _norm_path(c.get("source_file"))
|
|
3137
|
+
for c in match.get("candidates") or [] if isinstance(c, dict) and _norm_path(c.get("source_file"))))
|
|
3138
|
+
# a cited-elsewhere or outside-scope entry's citation check: missing, its reason, or outside-scope
|
|
3139
|
+
self.citation_check: str | None = None
|
|
3140
|
+
self.cited_cause: str | None = None # why a check at an in-scope citation pinned no line: missing, its reason
|
|
2334
3141
|
citation = entry.get("source_citation")
|
|
2335
3142
|
self.citation = citation if isinstance(citation, dict) else None
|
|
3143
|
+
self.cited_file = _norm_path(self.citation.get("file")) if self.citation else None
|
|
3144
|
+
# the cited file and each resolved path the brief's scope leaves out (#824): none is checked, scanned or
|
|
3145
|
+
# repaired from; with no brief there is no scope test (`takes` None), and nothing is left out
|
|
3146
|
+
self.outside_scope: list[str] = [] if takes is None else list(dict.fromkeys(
|
|
3147
|
+
p for p in (self.cited_file, *(_norm_path(p) for p in entry.get("resolved_paths") or []))
|
|
3148
|
+
if p and not takes(p)))
|
|
2336
3149
|
self.recorded: tuple[str | None, int | None] = (None, None) # the file and line a check read
|
|
2337
3150
|
self.outcome: str | None = None
|
|
2338
3151
|
self.new_line: int | None = None
|
|
2339
3152
|
self.unknown_reason: str | None = None
|
|
2340
3153
|
self.reextract = False
|
|
2341
|
-
self.record: dict | None = None # the §4a record it matched
|
|
3154
|
+
self.record: dict | None = None # the §4a record it matched, or took its signature from
|
|
2342
3155
|
self.record_path: str | None = None
|
|
3156
|
+
self.signature_at: tuple[str, int] | None = None # where a check located it: its signature's record
|
|
2343
3157
|
self.waits = False # on a declaring_line answer
|
|
2344
3158
|
self.failure: str | None = None # why its spot-check failed
|
|
2345
3159
|
|
|
@@ -2347,14 +3161,48 @@ class _GapEntry:
|
|
|
2347
3161
|
def blocking(self) -> bool:
|
|
2348
3162
|
return _blocking(self.entry.get("severity"))
|
|
2349
3163
|
|
|
3164
|
+
@property
|
|
3165
|
+
def scoped_out(self) -> bool:
|
|
3166
|
+
"""True for an entry route() records outside-scope (or cited-elsewhere with citation_check outside-scope)
|
|
3167
|
+
before anything is read for it: no in-scope citation to check and no in-scope file to scan. The drift gate
|
|
3168
|
+
passes it, since nothing it writes comes from the tree."""
|
|
3169
|
+
if self.category not in ("NEW_EXPORT", "MODIFIED_EXPORT") or self.match_status != "not-found":
|
|
3170
|
+
return False
|
|
3171
|
+
cited_out = self.cited_file is not None and self.cited_file in self.outside_scope
|
|
3172
|
+
if self.citation and not cited_out: # an in-scope citation is checked
|
|
3173
|
+
return False
|
|
3174
|
+
if self.map_locations: # with the cited file left out: cited-elsewhere, never checked; with none, as before
|
|
3175
|
+
return cited_out
|
|
3176
|
+
return bool(self.outside_scope) and not self.paths
|
|
3177
|
+
|
|
2350
3178
|
@property
|
|
2351
3179
|
def paths(self) -> list[str]:
|
|
2352
|
-
|
|
3180
|
+
"""The resolved paths targeted re-extraction scans: each one the brief's scope takes."""
|
|
3181
|
+
return [p for p in (_norm_path(p) for p in self.entry.get("resolved_paths") or [])
|
|
3182
|
+
if p and p not in self.outside_scope]
|
|
2353
3183
|
|
|
2354
3184
|
def check(self, root: Path | None, path: object, line: object, export_type: object) -> None:
|
|
2355
3185
|
self.recorded = (_norm_path(path), _line_of(line))
|
|
2356
|
-
self.outcome, self.new_line, self.waits, self.failure = _spot_check(
|
|
2357
|
-
root, self.recorded[0], self.
|
|
3186
|
+
self.outcome, self.new_line, self.waits, self.failure, self.unknown_reason = _spot_check(
|
|
3187
|
+
root, self.recorded[0], self.declared, self.recorded[1], export_type, self.answers,
|
|
3188
|
+
self.category == "MOVED_EXPORT")
|
|
3189
|
+
|
|
3190
|
+
def locate(self, root: Path | None) -> None:
|
|
3191
|
+
"""Record where a check located the export (the checked file, at the recorded line or the one it moved
|
|
3192
|
+
to), so its signature comes from §4a's record there; never a file outside the source root."""
|
|
3193
|
+
line = self.new_line if self.outcome == "moved" else self.recorded[1]
|
|
3194
|
+
if self.outcome in ("verified", "moved") and line is not None and _inside(root, self.recorded[0]):
|
|
3195
|
+
self.signature_at = (self.recorded[0], line)
|
|
3196
|
+
|
|
3197
|
+
def take_signature(self, blocks: dict[str, list[dict]]) -> None:
|
|
3198
|
+
"""Take the one record of the entry's name in the block of the file it was located in that is at its line,
|
|
3199
|
+
else the one with no line; none otherwise."""
|
|
3200
|
+
path, line = self.signature_at
|
|
3201
|
+
named = [r for r in blocks.get(path, []) if _name_of(r) in (self.name, self.declared)]
|
|
3202
|
+
at = [r for r in named if _line_of(r.get("location")) == line] \
|
|
3203
|
+
or [r for r in named if _line_of(r.get("location")) is None]
|
|
3204
|
+
if len(at) == 1:
|
|
3205
|
+
self.record, self.record_path = at[0], path
|
|
2358
3206
|
|
|
2359
3207
|
def route(self, root: Path | None, drift: bool) -> None:
|
|
2360
3208
|
"""Spot-check the entry, or send it to targeted re-extraction (see "Gap records" in the module docstring)."""
|
|
@@ -2373,14 +3221,36 @@ class _GapEntry:
|
|
|
2373
3221
|
return
|
|
2374
3222
|
if self.match_status == "found":
|
|
2375
3223
|
self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
|
|
3224
|
+
self.locate(root)
|
|
2376
3225
|
return
|
|
2377
|
-
if self.match_status == "ambiguous":
|
|
2378
|
-
self.outcome = "unknown"
|
|
3226
|
+
if self.match_status == "ambiguous": # several map entries of its name: no check, apply warns
|
|
3227
|
+
self.outcome, self.unknown_reason = "unknown", "ambiguous"
|
|
2379
3228
|
return
|
|
2380
|
-
if self.citation:
|
|
3229
|
+
if self.citation and self.cited_file is not None and self.cited_file in self.outside_scope:
|
|
3230
|
+
# the brief's scope leaves the cited file out: the skill does not cover its declaration, so no check runs
|
|
3231
|
+
# there (#824); the fallbacks below scan only the files the scope takes
|
|
3232
|
+
if self.map_locations: # the map holds its name at other files: cited elsewhere, and never checked
|
|
3233
|
+
self.outcome, self.unknown_reason, self.citation_check = "unknown", "cited-elsewhere", "outside-scope"
|
|
3234
|
+
return
|
|
3235
|
+
self.outcome, self.unknown_reason = "unknown", "outside-scope"
|
|
3236
|
+
elif self.citation:
|
|
2381
3237
|
self.check(root, self.citation.get("file"), self.citation.get("line"), None)
|
|
2382
3238
|
if self.waits or self.outcome in ("verified", "moved"):
|
|
3239
|
+
if not self.in_map: # a citation of another file than the map's entries of its name takes none
|
|
3240
|
+
self.locate(root)
|
|
2383
3241
|
return
|
|
3242
|
+
self.cited_cause = "missing" if self.outcome == "missing" else self.unknown_reason
|
|
3243
|
+
if self.match_status == "not-found" and self.map_locations:
|
|
3244
|
+
# the map holds its name at other files: the cited file's declaration may be another package's, so it is
|
|
3245
|
+
# never re-extracted there (#824); the record keeps the check's cause, and apply warns with what to fix
|
|
3246
|
+
self.citation_check = "missing" if self.outcome == "missing" else self.unknown_reason
|
|
3247
|
+
self.outcome, self.unknown_reason = "unknown", "cited-elsewhere"
|
|
3248
|
+
return
|
|
3249
|
+
if self.outside_scope and not self.paths:
|
|
3250
|
+
# no file the brief's scope takes is left to scan: never in the plan (#824); the record keeps the cause of
|
|
3251
|
+
# a check at an in-scope citation, which apply's warning names
|
|
3252
|
+
self.outcome, self.unknown_reason, self.citation_check = "unknown", "outside-scope", self.cited_cause
|
|
3253
|
+
return
|
|
2384
3254
|
if self.entry.get("provenance_completeness") and root is not None:
|
|
2385
3255
|
self.reextract = True
|
|
2386
3256
|
elif self.blocking:
|
|
@@ -2391,14 +3261,20 @@ class _GapEntry:
|
|
|
2391
3261
|
self.outcome = "unknown"
|
|
2392
3262
|
|
|
2393
3263
|
def match(self, blocks: dict[str, list[dict]]) -> tuple[int, int]:
|
|
2394
|
-
"""Take the first export of the entry's name in the blocks of its resolved files
|
|
2395
|
-
|
|
3264
|
+
"""Take the first export of the entry's name in the blocks of its resolved files, else the first whose
|
|
3265
|
+
`aliases` hold its name (the public name the runner gives its local name in that file); (files scanned,
|
|
3266
|
+
exports found in them)."""
|
|
2396
3267
|
exports = 0
|
|
3268
|
+
aliased: tuple[dict, str] | None = None
|
|
2397
3269
|
for path in self.paths:
|
|
2398
3270
|
for record in blocks.get(path, []):
|
|
2399
3271
|
exports += 1
|
|
2400
3272
|
if self.record is None and _name_of(record) == self.name:
|
|
2401
3273
|
self.record, self.record_path = record, path
|
|
3274
|
+
elif aliased is None and isinstance(record.get("aliases"), list) and self.name in record["aliases"]:
|
|
3275
|
+
aliased = (record, path)
|
|
3276
|
+
if self.record is None and aliased is not None:
|
|
3277
|
+
self.record, self.record_path = aliased
|
|
2402
3278
|
if self.record is not None:
|
|
2403
3279
|
self.outcome = "re-extracted"
|
|
2404
3280
|
return len(self.paths), exports
|
|
@@ -2427,8 +3303,15 @@ class _GapEntry:
|
|
|
2427
3303
|
return (self.record or {}).get("location")
|
|
2428
3304
|
return None
|
|
2429
3305
|
|
|
3306
|
+
def signature(self) -> dict | None:
|
|
3307
|
+
"""The record a located export takes its signature from, or None."""
|
|
3308
|
+
return self.record if self.signature_at and not self.internal else None
|
|
3309
|
+
|
|
2430
3310
|
def verification(self) -> dict:
|
|
3311
|
+
signed = self.signature() or {}
|
|
2431
3312
|
recorded = _where(*self.recorded)
|
|
3313
|
+
elsewhere = self.outcome == "unknown" and self.unknown_reason == "cited-elsewhere"
|
|
3314
|
+
outside = self.outcome == "unknown" and self.unknown_reason == "outside-scope"
|
|
2432
3315
|
if self.outcome == "re-extracted":
|
|
2433
3316
|
citation = self.new_location()
|
|
2434
3317
|
elif self.outcome in ("verified", "moved", "missing", "rescoped") and recorded:
|
|
@@ -2445,13 +3328,20 @@ class _GapEntry:
|
|
|
2445
3328
|
"provenance_citation": citation,
|
|
2446
3329
|
"source_citation": self.citation,
|
|
2447
3330
|
"new_location": self.new_location(),
|
|
2448
|
-
"unknown_reason": self.unknown_reason,
|
|
3331
|
+
"unknown_reason": self.unknown_reason if self.outcome == "unknown" else None,
|
|
3332
|
+
"citation_check": self.citation_check if elsewhere or outside else None,
|
|
3333
|
+
"map_locations": self.map_locations if elsewhere else None,
|
|
3334
|
+
"outside_scope": self.outside_scope or None, # whatever the outcome: each file the scope left out
|
|
2449
3335
|
"pinned_definition_lines": _pinned_lines(self.entry.get("remediation"))
|
|
2450
3336
|
if self.category == "MOVED_EXPORT" else None,
|
|
2451
3337
|
"resolution_source": "remediation-paths" if self.outcome == "re-extracted" else None,
|
|
2452
3338
|
"reachability": self.answers.get("reachability") if self.category == "NEW_EXPORT" and self.located
|
|
2453
3339
|
else None,
|
|
2454
|
-
"export_type":
|
|
3340
|
+
"export_type": signed.get("type", signed.get("export_type")),
|
|
3341
|
+
"params": signed.get("params", signed.get("parameters")),
|
|
3342
|
+
"return_type": signed.get("return_type"),
|
|
3343
|
+
"extraction_method": signed.get("extraction_method"),
|
|
3344
|
+
"ast_node_type": signed.get("ast_node_type"),
|
|
2455
3345
|
}
|
|
2456
3346
|
|
|
2457
3347
|
def label(self, provenance: list[dict]) -> str:
|
|
@@ -2464,7 +3354,7 @@ class _GapEntry:
|
|
|
2464
3354
|
elif self.map_entry is not None:
|
|
2465
3355
|
file, line = _norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"])
|
|
2466
3356
|
for entry in provenance:
|
|
2467
|
-
if (_name_of(entry) == self.
|
|
3357
|
+
if (_name_of(entry) == self.declared and _file_of(entry) == file
|
|
2468
3358
|
and _line_of(entry.get("source_line")) == line):
|
|
2469
3359
|
method = entry.get("extraction_method")
|
|
2470
3360
|
break
|
|
@@ -2479,8 +3369,9 @@ class _GapEntry:
|
|
|
2479
3369
|
|
|
2480
3370
|
def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path | None, drift_status: str = "ok",
|
|
2481
3371
|
judgments: dict | None = None, remediation: dict | None = None, plan: bool = False,
|
|
2482
|
-
tier: str | None = None) -> tuple[dict | None, dict]:
|
|
2483
|
-
"""(reextract-records.json or None, the summary): see "Gap records" in the module docstring.
|
|
3372
|
+
tier: str | None = None, scope=None) -> tuple[dict | None, dict]:
|
|
3373
|
+
"""(reextract-records.json or None, the summary): see "Gap records" in the module docstring. `scope` is the
|
|
3374
|
+
brief's (_brief_scope), or None for a skill with no brief."""
|
|
2484
3375
|
if not isinstance(manifest, dict) or not isinstance(manifest.get("entries"), list):
|
|
2485
3376
|
raise ValueError("the change manifest has no `entries` array: is it translate's output?")
|
|
2486
3377
|
entries = [e for e in manifest["entries"] if isinstance(e, dict)]
|
|
@@ -2488,13 +3379,14 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
|
|
|
2488
3379
|
root = source_root if source_root is not None and source_root.is_dir() else None
|
|
2489
3380
|
forwarded = [{"gap_id": e.get("gap_id"), "change_category": e.get("change_category")}
|
|
2490
3381
|
for e in entries if e.get("change_category") not in EXPORT_CATEGORIES]
|
|
2491
|
-
|
|
3382
|
+
takes = None if scope is None else _scope_test(scope, provenance, source_root)
|
|
3383
|
+
routed = [_GapEntry(e, judgments.get(e.get("gap_id")) or {}, takes) for e in entries
|
|
2492
3384
|
if e.get("change_category") in EXPORT_CATEGORIES]
|
|
2493
3385
|
drift = drift_status == "overridden"
|
|
2494
3386
|
if drift:
|
|
2495
3387
|
blocked = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "change_category": g.category,
|
|
2496
3388
|
"severity": g.entry.get("severity"), "reason": _drift_reason(g.entry)}
|
|
2497
|
-
for g in routed if _drift_reason(g.entry)]
|
|
3389
|
+
for g in routed if _drift_reason(g.entry) and not g.scoped_out] # nothing is read for it
|
|
2498
3390
|
if blocked:
|
|
2499
3391
|
return None, {"status": "drift-blocked", "drift_blocked": blocked}
|
|
2500
3392
|
bare = [g.name for g in routed if g.category == "DELETED_EXPORT" and not (
|
|
@@ -2513,17 +3405,22 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
|
|
|
2513
3405
|
if waiting:
|
|
2514
3406
|
return None, {"status": "needs-judgment", "needs_judgment": waiting, "warnings": warnings}
|
|
2515
3407
|
to_scan = [g for g in routed if g.reextract]
|
|
2516
|
-
|
|
3408
|
+
located = [g for g in routed if g.signature_at and not g.internal] # an internal export takes no signature
|
|
3409
|
+
files = list(dict.fromkeys([p for g in to_scan for p in g.paths] + [g.signature_at[0] for g in located]))
|
|
2517
3410
|
if plan:
|
|
2518
3411
|
return None, {"status": "planned", "reextract": {
|
|
2519
3412
|
"entries": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
|
|
2520
|
-
"resolved_paths": g.paths} for g in to_scan], "files": files
|
|
3413
|
+
"resolved_paths": g.paths} for g in to_scan], "files": files,
|
|
3414
|
+
"signatures": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "file": g.signature_at[0],
|
|
3415
|
+
"line": g.signature_at[1]} for g in located]}, "warnings": warnings}
|
|
2521
3416
|
blocks: dict[str, list[dict]] = {}
|
|
2522
3417
|
scanned = 0
|
|
3418
|
+
targeted_files = {p for g in to_scan for p in g.paths}
|
|
3419
|
+
signature_files = {g.signature_at[0] for g in located}
|
|
2523
3420
|
if files:
|
|
2524
3421
|
if remediation is None:
|
|
2525
|
-
raise ValueError("
|
|
2526
|
-
"of `records` over them")
|
|
3422
|
+
raise ValueError("gap-records has files to match (targeted re-extraction, or a located export's "
|
|
3423
|
+
"signature): pass --remediation-records, the output of `records` over them")
|
|
2527
3424
|
if not isinstance(remediation, dict) or not isinstance(remediation.get("files"), list):
|
|
2528
3425
|
raise ValueError("--remediation-records has no `files` array: is it the output of `records`?")
|
|
2529
3426
|
for block in remediation["files"]:
|
|
@@ -2536,6 +3433,9 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
|
|
|
2536
3433
|
files_scanned, found = g.match(blocks)
|
|
2537
3434
|
if g.record is not None:
|
|
2538
3435
|
continue
|
|
3436
|
+
if g.outside_scope: # none of the files the scope takes holds it, and a file it leaves out may (#824)
|
|
3437
|
+
g.outcome, g.unknown_reason, g.citation_check = "unknown", "outside-scope", g.cited_cause
|
|
3438
|
+
continue
|
|
2539
3439
|
if not g.blocking and g.entry.get("category") in DOCUMENT_CATEGORIES:
|
|
2540
3440
|
g.outcome = "unknown"
|
|
2541
3441
|
continue
|
|
@@ -2545,21 +3445,36 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
|
|
|
2545
3445
|
"files_scanned": files_scanned, "exports_found_in_scan": found})
|
|
2546
3446
|
if unresolved:
|
|
2547
3447
|
return None, {"status": "unresolved", "unresolved": unresolved, "warnings": warnings}
|
|
3448
|
+
for g in located:
|
|
3449
|
+
g.take_signature(blocks)
|
|
2548
3450
|
needs = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": g.needs(),
|
|
2549
3451
|
"file": _location(g.new_location())[0] if g.outcome != "verified" else g.recorded[0],
|
|
2550
3452
|
"line": _location(g.new_location())[1] if g.outcome != "verified" else g.recorded[1]}
|
|
2551
3453
|
for g in routed if g.needs()]
|
|
2552
3454
|
if needs:
|
|
2553
3455
|
return None, {"status": "needs-judgment", "needs_judgment": needs, "warnings": warnings}
|
|
3456
|
+
warnings += [f"provenance: {g.name}: located at {g.signature_at[0]}:{g.signature_at[1]}, but no record of it "
|
|
3457
|
+
f"there; signature not repaired" for g in located if g.record is None]
|
|
2554
3458
|
kept: dict[str, list[dict]] = {}
|
|
2555
3459
|
for g in routed:
|
|
2556
|
-
if g.record is
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
3460
|
+
if g.record is None or g.internal:
|
|
3461
|
+
continue
|
|
3462
|
+
record = dict(g.record)
|
|
3463
|
+
record.pop("public", None) # a repair documents and maps every record: the mark is a normal update's
|
|
3464
|
+
record.pop("aliases", None) # the match is made: the copy names the one it took
|
|
3465
|
+
if _name_of(record) != g.name: # matched by its public name: documented under it, its own name as `local`
|
|
3466
|
+
record = {"name": g.name, "local": _name_of(record),
|
|
3467
|
+
**{k: v for k, v in record.items() if k not in ("name", "export_name", "local")}}
|
|
3468
|
+
if g.outcome == "re-extracted" and g.answers.get("docstring") is not None:
|
|
3469
|
+
record["docstring"] = g.answers["docstring"] # a located export's signature record asks none
|
|
3470
|
+
block = kept.setdefault(g.record_path, [])
|
|
3471
|
+
location = record.get("location")
|
|
3472
|
+
same = next((r for r in block if (_name_of(r) == _name_of(record) and r.get("location") == location
|
|
3473
|
+
if location is not None else r == record)), None)
|
|
3474
|
+
if same is None:
|
|
3475
|
+
block.append(record)
|
|
3476
|
+
elif same.get("docstring") is None and record.get("docstring") is not None:
|
|
3477
|
+
same["docstring"] = record["docstring"]
|
|
2563
3478
|
breakdown = {"T1": 0, "T1-low": 0, "unlabeled": 0, "T2": 0}
|
|
2564
3479
|
provenance_entries = _entries_of(provenance)
|
|
2565
3480
|
for g in routed:
|
|
@@ -2570,13 +3485,15 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
|
|
|
2570
3485
|
"files_extracted": scanned,
|
|
2571
3486
|
"exports_extracted": len(entries),
|
|
2572
3487
|
"confidence_breakdown": breakdown,
|
|
2573
|
-
"verification": [g.verification() for g in routed],
|
|
3488
|
+
"verification": [{"gap_id": g.entry.get("gap_id"), **g.verification()} for g in routed],
|
|
2574
3489
|
"files": [{"file_path": path, "exports": exports} for path, exports in kept.items()],
|
|
2575
3490
|
}
|
|
2576
3491
|
matched = [g for g in to_scan if g.record is not None]
|
|
2577
|
-
|
|
3492
|
+
scanned_paths = set(blocks)
|
|
3493
|
+
targeted = {"resolved_count": len(matched), "files_scanned": len(scanned_paths & targeted_files),
|
|
3494
|
+
"signature_files_scanned": len(scanned_paths & signature_files),
|
|
2578
3495
|
"exports_matched": len({(g.record_path, _name_of(g.record)) for g in matched}),
|
|
2579
|
-
"tier": tier} if to_scan else None
|
|
3496
|
+
"tier": tier} if to_scan or located else None
|
|
2580
3497
|
summary = {
|
|
2581
3498
|
"status": "written",
|
|
2582
3499
|
"gap_count": len(entries),
|
|
@@ -2766,13 +3683,62 @@ def _metadata_in(path: str | None) -> dict | None:
|
|
|
2766
3683
|
return metadata
|
|
2767
3684
|
|
|
2768
3685
|
|
|
3686
|
+
_QUICK_METADATA = None
|
|
3687
|
+
|
|
3688
|
+
|
|
3689
|
+
def _quick_metadata():
|
|
3690
|
+
"""skf-render-quick-metadata.py, loaded once from this folder as skf-extraction-inventory.py loads it: its
|
|
3691
|
+
probe_skf_version reads the installed SKF's version, so create-skill and update-skill read it one way. Raises
|
|
3692
|
+
ValueError, one line, when it cannot be loaded."""
|
|
3693
|
+
global _QUICK_METADATA
|
|
3694
|
+
if _QUICK_METADATA is None:
|
|
3695
|
+
path = Path(__file__).resolve().parent / "skf-render-quick-metadata.py"
|
|
3696
|
+
cannot = f"cannot load {path.name} beside {Path(__file__).name}"
|
|
3697
|
+
if not path.is_file():
|
|
3698
|
+
raise ValueError(f"{cannot}: it is missing; re-install SKF")
|
|
3699
|
+
spec = importlib.util.spec_from_file_location("skf_render_quick_metadata", path)
|
|
3700
|
+
if spec is None or spec.loader is None:
|
|
3701
|
+
raise ValueError(f"{cannot}: re-install SKF")
|
|
3702
|
+
module = importlib.util.module_from_spec(spec)
|
|
3703
|
+
try:
|
|
3704
|
+
spec.loader.exec_module(module)
|
|
3705
|
+
except (ImportError, SyntaxError, OSError) as exc:
|
|
3706
|
+
raise ValueError(f"{cannot} ({type(exc).__name__}: {' '.join(str(exc).split())}): re-install "
|
|
3707
|
+
f"SKF") from exc
|
|
3708
|
+
if not callable(getattr(module, "probe_skf_version", None)):
|
|
3709
|
+
raise ValueError(f"{cannot}: it has no probe_skf_version; re-install SKF")
|
|
3710
|
+
_QUICK_METADATA = module
|
|
3711
|
+
return _QUICK_METADATA
|
|
3712
|
+
|
|
3713
|
+
|
|
3714
|
+
def _runner_ast_grep(path: str | None) -> str | None:
|
|
3715
|
+
"""The ast-grep version the recipe runner's output (--extraction) records: "unknown" when it records none but
|
|
3716
|
+
holds an export ast-grep matched, and None when ast-grep did not run (no export it matched, as in the
|
|
3717
|
+
{"exports": []} step 2 writes when the runner cannot run)."""
|
|
3718
|
+
if not path:
|
|
3719
|
+
return None
|
|
3720
|
+
extraction = _load_json_file(Path(path), "--extraction file")
|
|
3721
|
+
if not isinstance(extraction, dict):
|
|
3722
|
+
raise ValueError("--extraction file must hold a JSON object (the recipe runner's output)")
|
|
3723
|
+
ast_grep = extraction.get("ast_grep")
|
|
3724
|
+
version = _tool_version(ast_grep.get("version")) if isinstance(ast_grep, dict) else None
|
|
3725
|
+
if version is not None:
|
|
3726
|
+
return version
|
|
3727
|
+
exports = extraction.get("exports") if isinstance(extraction.get("exports"), list) else []
|
|
3728
|
+
matched = any(isinstance(e, dict) and e.get("extraction_method") == "ast-grep" for e in exports)
|
|
3729
|
+
return "unknown" if matched else None
|
|
3730
|
+
|
|
3731
|
+
|
|
2769
3732
|
def _cmd_metadata(args: argparse.Namespace) -> int:
|
|
2770
3733
|
target = Path(args.metadata)
|
|
2771
3734
|
try:
|
|
2772
|
-
|
|
3735
|
+
skf_version = _quick_metadata().probe_skf_version(args.skf_root) if args.skf_root else None
|
|
3736
|
+
metadata, summary = update_metadata(
|
|
2773
3737
|
_load_json_file(target, "--metadata file"), _load_json_file(Path(args.lists), "--lists file"),
|
|
2774
3738
|
_load_json_file(Path(args.stats), "--stats file"), version=args.version,
|
|
2775
|
-
generation_date=args.generation_date, source_commit=args.source_commit, source_ref=args.source_ref
|
|
3739
|
+
generation_date=args.generation_date, source_commit=args.source_commit, source_ref=args.source_ref,
|
|
3740
|
+
skf_version=skf_version, skf_root=args.skf_root, ast_grep_version=_runner_ast_grep(args.extraction),
|
|
3741
|
+
qmd_version=args.qmd_version)
|
|
2776
3742
|
_write_json_atomic(target, metadata)
|
|
2777
3743
|
except ValueError as exc:
|
|
2778
3744
|
print(f"error: {exc}", file=sys.stderr)
|
|
@@ -2782,7 +3748,8 @@ def _cmd_metadata(args: argparse.Namespace) -> int:
|
|
|
2782
3748
|
return 1
|
|
2783
3749
|
print(json.dumps({"status": "written", "metadata": target.as_posix(), "version": metadata["version"],
|
|
2784
3750
|
"exports": len(metadata["exports"]), "scripts": len(metadata.get("scripts") or []),
|
|
2785
|
-
"assets": len(metadata.get("assets") or [])
|
|
3751
|
+
"assets": len(metadata.get("assets") or []), "tool_versions": metadata["tool_versions"],
|
|
3752
|
+
**summary}))
|
|
2786
3753
|
return 0
|
|
2787
3754
|
|
|
2788
3755
|
|
|
@@ -2873,15 +3840,22 @@ def _cmd_gap_records(args: argparse.Namespace) -> int:
|
|
|
2873
3840
|
raise ValueError("--evidence and --tier go together")
|
|
2874
3841
|
judgments = _gap_judgments(_load_json_file(Path(args.judgments), "--judgments file")) \
|
|
2875
3842
|
if args.judgments else {}
|
|
3843
|
+
manifest = _load_json_file(Path(args.manifest), "--manifest file")
|
|
3844
|
+
provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file")
|
|
3845
|
+
if args.brief is not None and not args.brief.strip():
|
|
3846
|
+
raise ValueError("--brief is empty (an unbound brief path?): pass the skill brief, or leave --brief out")
|
|
3847
|
+
scope = _brief_scope(Path(args.brief), _gap_paths(manifest) | _map_paths(provenance)) \
|
|
3848
|
+
if args.brief is not None else None
|
|
2876
3849
|
records, summary = gap_records(
|
|
2877
|
-
|
|
2878
|
-
|
|
3850
|
+
manifest,
|
|
3851
|
+
provenance,
|
|
2879
3852
|
source_root=Path(args.source_root) if args.source_root else None,
|
|
2880
3853
|
drift_status=args.drift_status,
|
|
2881
3854
|
judgments=judgments,
|
|
2882
3855
|
remediation=_optional_json(args.remediation_records, "--remediation-records file"),
|
|
2883
3856
|
plan=args.plan,
|
|
2884
3857
|
tier=args.tier,
|
|
3858
|
+
scope=scope,
|
|
2885
3859
|
)
|
|
2886
3860
|
if summary["status"] == "planned" and args.files_out:
|
|
2887
3861
|
_write_json_atomic(Path(args.files_out), summary["reextract"]["files"])
|
|
@@ -3061,9 +4035,15 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
3061
4035
|
description="Rewrite the skill's metadata.json in place. It sets version and generation_date, "
|
|
3062
4036
|
"source_commit and source_ref when given, exports, scripts and assets from --lists (a script "
|
|
3063
4037
|
"or asset list left out when empty), confidence_distribution from --stats and the stats "
|
|
3064
|
-
"helper's stats fields, dropping one an earlier run wrote that --stats lacks;
|
|
3065
|
-
"
|
|
3066
|
-
"
|
|
4038
|
+
"helper's stats fields, dropping one an earlier run wrote that --stats lacks; tool_versions.skf "
|
|
4039
|
+
"from --skf-root (unknown, with a warning, when it gives no version), and tool_versions.ast_grep "
|
|
4040
|
+
"and tool_versions.qmd from --extraction and --qmd-version when this run used them, else as "
|
|
4041
|
+
"they stand. When --stats' effective_denominator differs from the old one (or one is absent), it "
|
|
4042
|
+
"drops each old stats key named effective_denominator_*; each dropped key is in dropped_stats, "
|
|
4043
|
+
"with a warning. It keeps every other old stats key, with a warning for each one create-skill "
|
|
4044
|
+
"does not write for the skill's scope_type. It removes last_update and "
|
|
4045
|
+
"update_type and keeps every other field, a patched one included. It refuses a --stats whose "
|
|
4046
|
+
"scripts_count or assets_count is not the length of --lists' scripts or assets.")
|
|
3067
4047
|
p_meta.add_argument("--metadata", required=True, metavar="FILE",
|
|
3068
4048
|
help="the skill's metadata.json, rewritten in place")
|
|
3069
4049
|
p_meta.add_argument("--lists", required=True, metavar="FILE", help="the lists apply wrote (--lists-out)")
|
|
@@ -3073,6 +4053,14 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
3073
4053
|
p_meta.add_argument("--generation-date", required=True, help="the clock's UTC time, YYYY-MM-DDTHH:MM:SSZ")
|
|
3074
4054
|
p_meta.add_argument("--source-commit", default=None, help="the commit this update read (an empty one: null)")
|
|
3075
4055
|
p_meta.add_argument("--source-ref", default=None, help="the ref this update read (an empty one: null)")
|
|
4056
|
+
p_meta.add_argument("--skf-root", metavar="DIR",
|
|
4057
|
+
help="the installed SKF ({project-root}/_bmad/skf): tool_versions.skf, from its package.json "
|
|
4058
|
+
"or VERSION file")
|
|
4059
|
+
p_meta.add_argument("--extraction", metavar="FILE",
|
|
4060
|
+
help="the recipe runner's output this run wrote: tool_versions.ast_grep, its ast_grep.version")
|
|
4061
|
+
p_meta.add_argument("--qmd-version", default=None,
|
|
4062
|
+
help="what qmd --version printed, when this run queried QMD: tool_versions.qmd (a leading "
|
|
4063
|
+
"\"qmd \" removed)")
|
|
3076
4064
|
p_meta.set_defaults(func=_cmd_metadata)
|
|
3077
4065
|
|
|
3078
4066
|
p_records = sub.add_parser("records", help="write step 3's re-extraction records from the runner's and the "
|
|
@@ -3090,13 +4078,16 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
3090
4078
|
p_gap.add_argument("--manifest", required=True, metavar="FILE", help="skf-parse-gaps.py translate's manifest")
|
|
3091
4079
|
p_gap.add_argument("--provenance-map", required=True, metavar="FILE", help="the map the repair starts from")
|
|
3092
4080
|
p_gap.add_argument("--source-root", metavar="DIR", help="the source tree the spot-checks read")
|
|
4081
|
+
p_gap.add_argument("--brief", metavar="FILE",
|
|
4082
|
+
help="the skill brief (skill-brief.yaml): no file a gap cites or its Remediation names that its "
|
|
4083
|
+
"scope leaves out is checked, scanned or repaired from; without it there is no scope test")
|
|
3093
4084
|
p_gap.add_argument("--drift-status", choices=DRIFT_STATUSES, default="ok",
|
|
3094
4085
|
help="the workspace drift check's status; overridden runs the drift gate")
|
|
3095
4086
|
p_gap.add_argument("--judgments", metavar="FILE", help="the answers to its needs_judgment[], keyed by gap id")
|
|
3096
4087
|
p_gap.add_argument("--remediation-records", metavar="FILE",
|
|
3097
|
-
help="`records` output over the files
|
|
4088
|
+
help="`records` output over the files the plan listed")
|
|
3098
4089
|
p_gap.add_argument("--plan", action="store_true",
|
|
3099
|
-
help="print the answers it needs and the files
|
|
4090
|
+
help="print the answers it needs and the files §4a's extraction scans; write no records")
|
|
3100
4091
|
p_gap.add_argument("--files-out", metavar="FILE", help="with --plan: write those files here, a JSON array")
|
|
3101
4092
|
p_gap.add_argument("--evidence", metavar="FILE",
|
|
3102
4093
|
help="append the targeted_reextraction record to this JSON-lines file when it ran")
|