bmad-module-skill-forge 4.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +3 -3
- package/docs/_data/pinned.yaml +19 -13
- package/docs/architecture.md +2 -2
- package/docs/concepts.md +4 -4
- package/docs/examples.md +32 -25
- package/docs/getting-started.md +3 -3
- package/docs/how-it-works.md +2 -2
- package/docs/index.md +1 -1
- package/docs/skill-model.md +41 -39
- package/docs/verifying-a-skill.md +41 -20
- package/docs/why-skf.md +1 -1
- package/docs/workflows.md +2 -2
- package/package.json +1 -1
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +11 -3
- package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
- package/src/shared/scripts/skf-build-change-manifest.py +881 -168
- package/src/shared/scripts/skf-count-tokens.py +54 -10
- package/src/shared/scripts/skf-detect-workspaces.py +67 -0
- package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
- package/src/shared/scripts/skf-extract-public-api.py +439 -107
- package/src/shared/scripts/skf-extraction-inventory.py +10 -6
- package/src/shared/scripts/skf-extraction-snapshot.py +6 -6
- package/src/shared/scripts/skf-parse-gaps.py +135 -26
- package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
- package/src/shared/scripts/skf-resolve-customization.py +197 -0
- package/src/shared/scripts/skf-shard-body.py +239 -41
- package/src/shared/scripts/skf-validate-output.py +12 -3
- package/src/shared/scripts/skf-verify-provenance-completeness.py +470 -38
- package/src/shared/tool-requirements.yaml +8 -2
- package/src/skf-analyze-source/SKILL.md +2 -2
- package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
- package/src/skf-audit-skill/SKILL.md +2 -2
- package/src/skf-audit-skill/references/re-index.md +2 -2
- package/src/skf-brief-skill/SKILL.md +2 -2
- package/src/skf-brief-skill/assets/skill-brief-schema.md +2 -2
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-campaign/SKILL.md +2 -2
- package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
- package/src/skf-create-skill/SKILL.md +2 -2
- package/src/skf-create-skill/assets/skill-sections.md +1 -1
- package/src/skf-create-skill/references/extract.md +2 -1
- package/src/skf-create-skill/references/extraction-patterns.md +3 -3
- package/src/skf-create-skill/references/report.md +6 -1
- package/src/skf-create-stack-skill/SKILL.md +2 -2
- package/src/skf-create-stack-skill/references/validate.md +1 -1
- package/src/skf-drop-skill/SKILL.md +2 -2
- package/src/skf-export-skill/SKILL.md +2 -2
- package/src/skf-export-skill/assets/snippet-format.md +5 -5
- package/src/skf-export-skill/references/generate-snippet.md +22 -15
- package/src/skf-export-skill/references/load-skill.md +1 -1
- package/src/skf-export-skill/references/token-report.md +29 -9
- package/src/skf-forger/SKILL.md +2 -2
- package/src/skf-quick-skill/SKILL.md +2 -2
- package/src/skf-refine-architecture/SKILL.md +6 -8
- package/src/skf-rename-skill/SKILL.md +2 -2
- package/src/skf-setup/SKILL.md +2 -2
- package/src/skf-test-skill/SKILL.md +2 -2
- package/src/skf-test-skill/assets/output-section-formats.md +6 -4
- package/src/skf-test-skill/references/coherence-check.md +4 -3
- package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
- package/src/skf-test-skill/references/coverage-check-tiers.md +1 -1
- package/src/skf-test-skill/references/coverage-check.md +45 -69
- package/src/skf-test-skill/references/init.md +2 -1
- package/src/skf-test-skill/references/migration-section-rules.md +26 -14
- package/src/skf-test-skill/references/report.md +2 -2
- package/src/skf-test-skill/references/scoring-rules.md +5 -4
- package/src/skf-test-skill/references/source-access-protocol.md +6 -6
- package/src/skf-test-skill/scripts/gap-ledger.py +378 -163
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +171 -70
- package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
- package/src/skf-test-skill/scripts/reconcile-coverage.py +192 -39
- package/src/skf-test-skill/scripts/score-signatures.py +329 -54
- package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
- package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
- package/src/skf-test-skill/templates/test-report-template.md +1 -0
- package/src/skf-update-skill/SKILL.md +2 -2
- package/src/skf-update-skill/references/detect-changes.md +1 -1
- package/src/skf-update-skill/references/gap-driven.md +36 -21
- package/src/skf-update-skill/references/merge.md +15 -10
- package/src/skf-update-skill/references/re-extract.md +3 -1
- package/src/skf-update-skill/references/report.md +5 -4
- package/src/skf-update-skill/references/write.md +39 -18
- package/src/skf-verify-stack/SKILL.md +2 -2
- package/tools/cli/lib/tool-check.js +158 -2
|
@@ -64,11 +64,13 @@ Subcommands:
|
|
|
64
64
|
|
|
65
65
|
metadata --metadata <metadata.json> --lists <file> --stats <file>
|
|
66
66
|
--version <version> --generation-date <iso>
|
|
67
|
-
[--source-commit <sha>] [--source-ref <ref>]
|
|
67
|
+
[--source-commit <sha>] [--source-ref <ref>] [--skf-root <dir>]
|
|
68
|
+
[--extraction <file>] [--qmd-version <version>]
|
|
68
69
|
The skill's metadata.json an update leaves (see "Metadata" below),
|
|
69
70
|
written over --metadata through a temporary file and a rename.
|
|
70
71
|
Prints {"status": "written", "metadata": <path>, "version",
|
|
71
|
-
"exports": N, "scripts": N, "assets": N}
|
|
72
|
+
"exports": N, "scripts": N, "assets": N, "tool_versions": {...},
|
|
73
|
+
"dropped_stats": [...], "warnings": [...]}.
|
|
72
74
|
|
|
73
75
|
records [--extraction <file>] [--export-details <file>]
|
|
74
76
|
[--files-from <file>] [--patches <dir>] -o <file>
|
|
@@ -84,8 +86,9 @@ Subcommands:
|
|
|
84
86
|
(--plan [--files-out <file>] | -o <file>)
|
|
85
87
|
A gap-driven repair's spot-checks, routing and verification records
|
|
86
88
|
(see "Gap records" below): --plan prints what to answer and which
|
|
87
|
-
files
|
|
88
|
-
|
|
89
|
+
files §4a's extraction scans (targeted re-extraction, and the
|
|
90
|
+
signatures of the exports a spot-check located); without it, the
|
|
91
|
+
records are written to -o through a temporary file and a rename.
|
|
89
92
|
|
|
90
93
|
baseline-gaps --provenance-map <file> --extraction <file> --files <file>
|
|
91
94
|
--source-root <dir> -o <file>
|
|
@@ -241,14 +244,32 @@ Apply (update-skill write.md §3):
|
|
|
241
244
|
|
|
242
245
|
Reads the provenance map the update started from (--provenance-map;
|
|
243
246
|
without it, a degraded run's new map) and prints nothing but a summary:
|
|
244
|
-
the new map goes to -o. Every entry
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
247
|
+
the new map goes to -o. Every entry it adds or rewrites carries
|
|
248
|
+
signature_source, set from the tool that produced the signature:
|
|
249
|
+
ast-grep gives T1 (with confidence T1 and the recipe's ast_node_type),
|
|
250
|
+
a signature read by eye (source-read) gives T1-low (confidence T1-low,
|
|
251
|
+
ast_node_type null). Every entry the run leaves alone keeps its exact
|
|
252
|
+
value, but for a signature_source it lacks or holds as a value other
|
|
253
|
+
than T1, T1-low, T2 or T3 (compared case-insensitively), as a map
|
|
254
|
+
written before the key existed does: once the mode's changes are made,
|
|
255
|
+
each such entry takes its confidence when that names T1-low, T2 or T3,
|
|
256
|
+
else T1 for an extraction_method of ast-grep or ast_bridge (compared
|
|
257
|
+
stripped and case-insensitively) and T1-low for any other (T1 needs an
|
|
258
|
+
ast-grep match). The summary's signature_source_backfilled counts these
|
|
259
|
+
fills by value ({"T1": 1, "T1-low": 2}, {} when none), with one warning
|
|
260
|
+
that lists each unrecognized value replaced (name: old -> new, the
|
|
261
|
+
first 10, then "and N more"); their names are not in its `updated`. A
|
|
262
|
+
rewritten entry keeps any key the update does not set (notes,
|
|
263
|
+
deprecated). A new entry's source_library is the map's library when its
|
|
264
|
+
entries name one (--skill-name when they name none). When they name
|
|
265
|
+
several, it is the library the entries of its file settle, else the one
|
|
266
|
+
the entries under the nearest folder above the file that holds entries
|
|
267
|
+
settle: the one library they name, or of several, the one that is a
|
|
268
|
+
path prefix of all the others (storybook beside storybook/test). The
|
|
269
|
+
record of a MOVED file, or of an export whose change names an old_file,
|
|
270
|
+
tries its new path, then its old. When neither settles it, or the entry
|
|
271
|
+
has no file, it is the map's first library, and one warning counts and
|
|
272
|
+
names those entries (the first 10, then "and N more").
|
|
252
273
|
|
|
253
274
|
incremental (normal mode) reads --manifest (build's output), the renamed
|
|
254
275
|
exports of --category-c and --ccc-pairs (or of --input, the category
|
|
@@ -274,9 +295,10 @@ Apply (update-skill write.md §3):
|
|
|
274
295
|
MOVED_EXPORT whose fresh record is the runner's (ast-grep) also takes
|
|
275
296
|
its confidence, extraction_method and ast_node_type (the entry's when
|
|
276
297
|
the record's is null), and its signature_source when the record holds
|
|
277
|
-
each of params and return_type the entry holds (else the entry's
|
|
278
|
-
|
|
279
|
-
the
|
|
298
|
+
each of params and return_type the entry holds (else the entry's, or
|
|
299
|
+
T1-low when the entry has none it recognizes: the signature it keeps
|
|
300
|
+
was not the runner's), never its export_type, params or return_type;
|
|
301
|
+
one read by eye moves the line and file only.
|
|
280
302
|
|
|
281
303
|
Public-api skills (incremental only): a NEW_EXPORT or MODIFIED_EXPORT
|
|
282
304
|
the map does not hold, an ADDED file's record and a MOVED file's record
|
|
@@ -299,31 +321,91 @@ Apply (update-skill write.md §3):
|
|
|
299
321
|
gap-driven reads --reextract-records ({"mode": "gap-driven",
|
|
300
322
|
"verification": [...], "files": [...]}) and --merge-records ({"exports":
|
|
301
323
|
[{export_name, export_type, params, return_type}]}, merge's output for
|
|
302
|
-
an export no
|
|
324
|
+
an export no record covers). Each verification record names gap_id,
|
|
303
325
|
export_name, gap_category, severity, verification, in_map (the map holds
|
|
304
326
|
an entry of that name), map_entry ({source_file, source_line} of the one
|
|
305
327
|
entry the spot-check took, else null), source_citation, new_location,
|
|
306
|
-
unknown_reason,
|
|
307
|
-
|
|
308
|
-
|
|
328
|
+
unknown_reason, citation_check, map_locations (see "Gap records"),
|
|
329
|
+
pinned_definition_lines, reachability, and the located
|
|
330
|
+
export's export_type, params, return_type, extraction_method and
|
|
331
|
+
ast_node_type (from §4a's record at its line, see "Gap records"; null
|
|
332
|
+
when none holds it). It finds an export's entry by its name, else by the
|
|
333
|
+
entry's reexported_as (a renamed re-export the map holds under its
|
|
334
|
+
declared name, which the gap names by its public one):
|
|
335
|
+
verified (map_entry) unchanged, but a NEW_EXPORT's or
|
|
336
|
+
MODIFIED_EXPORT's signature (below) when its
|
|
337
|
+
record was found
|
|
338
|
+
moved (map_entry) source_line (and source_file) from new_location,
|
|
339
|
+
and a NEW_EXPORT's or MODIFIED_EXPORT's
|
|
340
|
+
signature (below) when its record was found
|
|
309
341
|
re-extracted the entry from its `files` record (gap-driven.md §4a);
|
|
310
|
-
none when its reachability is internal-unreachable
|
|
342
|
+
none when its reachability is internal-unreachable;
|
|
343
|
+
a record with a `local` name (matched by the
|
|
344
|
+
public name a re-export gives it) writes
|
|
345
|
+
export_name the declared `local` name, at its
|
|
346
|
+
line, and reexported_as the public one
|
|
311
347
|
verified/moved, not in_map, a NEW_EXPORT or MODIFIED_EXPORT whose
|
|
312
348
|
reachability is not internal-unreachable
|
|
313
349
|
one source-read entry at the citation's line
|
|
314
350
|
(verified) or new_location (moved), at most one
|
|
315
|
-
per name and file,
|
|
316
|
-
|
|
351
|
+
per name and file, with the signature (below)
|
|
352
|
+
and the source-read labels whatever its record's
|
|
353
|
+
tool
|
|
317
354
|
unknown, not in_map, a NEW_EXPORT or MODIFIED_EXPORT, a Medium, Low or
|
|
318
355
|
Info severity an entry from --merge-records with no
|
|
319
356
|
source_file or source_line
|
|
320
357
|
unknown, not in_map, any other severity (a missing one included)
|
|
321
358
|
refused: exit 3, nothing written
|
|
322
359
|
missing, unknown in_map, unknown on a MOVED_EXPORT
|
|
323
|
-
unchanged, with a warning for a person
|
|
360
|
+
unchanged, with a warning for a person that
|
|
361
|
+
names an unknown's unknown_reason and its
|
|
362
|
+
words (UNKNOWN_REASONS); for a NEW_EXPORT or
|
|
363
|
+
MODIFIED_EXPORT whose severity is not Medium,
|
|
364
|
+
Low or Info (a missing one included), the
|
|
365
|
+
warning also names its gap_id and the one
|
|
366
|
+
fix a re-run acts on: for an ambiguous one,
|
|
367
|
+
fix its entries by hand, or give the gap a
|
|
368
|
+
`file:line` Source at its definition when
|
|
369
|
+
that file holds only one of them (the
|
|
370
|
+
re-run's map_match then finds that one); for
|
|
371
|
+
any other, fix the entry by hand (its
|
|
372
|
+
source_file and source_line), since the
|
|
373
|
+
re-run checks the same entry again. A
|
|
374
|
+
cited-elsewhere one, whatever its severity,
|
|
375
|
+
takes in their place a warning that names
|
|
376
|
+
its gap_id, its source_citation, its
|
|
377
|
+
citation_check (with its words), its
|
|
378
|
+
map_locations and the fix for each side
|
|
379
|
+
that may be wrong: when the map is right,
|
|
380
|
+
re-run Test Skill (the test compared
|
|
381
|
+
another file's declaration), and if the gap
|
|
382
|
+
comes back (not for a missing one), exclude
|
|
383
|
+
the cited file in the brief's scope when the
|
|
384
|
+
skill does not cover it; when the export
|
|
385
|
+
moved, fix the entry by hand (with several
|
|
386
|
+
map_locations, whichever of those entries
|
|
387
|
+
moved), then re-run update-skill. A
|
|
388
|
+
citation_check that read nothing (UNREAD:
|
|
389
|
+
no-source-root, verifier-missing,
|
|
390
|
+
spot-check-failed) takes its own fix in
|
|
391
|
+
their place: give update-skill the source
|
|
392
|
+
tree, re-install SKF, or fix what the check
|
|
393
|
+
failed on, then re-run update-skill
|
|
324
394
|
rescoped the entry removed
|
|
395
|
+
The signature: when §4a found the export's record (its verification
|
|
396
|
+
record has an extraction_method), every part from it, a params or
|
|
397
|
+
return_type it gives as null removed from the entry; with none, an
|
|
398
|
+
entry the map holds keeps its signature, and an entry this run adds
|
|
399
|
+
takes --merge-records'. An entry the map holds keeps its confidence and
|
|
400
|
+
extraction_method; when its own extraction_method is ast-grep or
|
|
401
|
+
ast_bridge, it takes the signature_source of its record's labels (T1
|
|
402
|
+
for an ast-grep record, T1-low for a source-read one) and, when the
|
|
403
|
+
record's export_type is another, the record's ast_node_type, while a
|
|
404
|
+
source-read entry keeps T1-low and no node kind. Each signature_source
|
|
405
|
+
it changes is listed in `relabeled`, {name, file, from, to}, each with a
|
|
406
|
+
warning. An internal-unreachable NEW_EXPORT takes none.
|
|
325
407
|
Under the drift override (--drift-head and --drift-pinned) each warning
|
|
326
|
-
names the drift.
|
|
408
|
+
names the drift, after the outcome's cause.
|
|
327
409
|
|
|
328
410
|
full (degraded mode) makes every fresh record an entry.
|
|
329
411
|
|
|
@@ -351,8 +433,9 @@ Apply (update-skill write.md §3):
|
|
|
351
433
|
--metadata-in exports[] less each name whose entry this run removed (a
|
|
352
434
|
DELETED_EXPORT, a DELETED file's or a MOVED file's old path's, a
|
|
353
435
|
rescoped one, a changed URL's) or renamed, unless an entry of the new
|
|
354
|
-
map still holds it, then each name it added an entry for
|
|
355
|
-
|
|
436
|
+
map still holds it, then each name it added an entry for (the entry's
|
|
437
|
+
reexported_as when it has one, the public name) or renamed an entry to,
|
|
438
|
+
each once; a renamed name takes its old name's place. A name
|
|
356
439
|
left out as off the public surface (the summary's not_public) has no
|
|
357
440
|
entry, so it is not added. A full update, which rebuilds the map, also
|
|
358
441
|
drops each name the new map does not hold, and a reference app
|
|
@@ -379,15 +462,45 @@ Metadata (update-skill write.md §2):
|
|
|
379
462
|
component library's own counts, a reference app's notes, a field a
|
|
380
463
|
gap-driven patch added), the stats helper's fields an earlier run wrote
|
|
381
464
|
that this run's does not (effective_denominator,
|
|
382
|
-
pattern_surfaces_documented) dropped
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
465
|
+
pattern_surfaces_documented) dropped and listed in `dropped_stats`,
|
|
466
|
+
each with a warning. When this run's effective_denominator differs
|
|
467
|
+
from the old one (one of them absent included), it also drops each old
|
|
468
|
+
stats key named effective_denominator_* (an explanation of the old
|
|
469
|
+
count), listed in `dropped_stats` with the warning "metadata:
|
|
470
|
+
stats.<key> dropped: it explained an effective_denominator this run
|
|
471
|
+
changed or removed"; when the two are the same, such a key stays with
|
|
472
|
+
no warning, since it still describes the denominator. Each other old
|
|
473
|
+
stats key it keeps adds the warning "metadata: stats.<key> kept from an
|
|
474
|
+
earlier run; check it still describes this run's counts", unless
|
|
475
|
+
create-skill writes it for the skill's scope_type (STATS_CREATE_FIELDS:
|
|
476
|
+
a component-library's Component Library stats of
|
|
477
|
+
compile-assembly-rules.md, a reference-app's notes).
|
|
478
|
+
|
|
479
|
+
tool_versions names the tools this run used: the old tool_versions
|
|
480
|
+
({"ast_grep": null, "qmd": null} when absent, or when it is not an
|
|
481
|
+
object, which adds a warning), with skf set to the version
|
|
482
|
+
skf-render-quick-metadata.py's probe_skf_version reads under
|
|
483
|
+
--skf-root (package.json, else VERSION), never the old value, or to
|
|
484
|
+
"unknown" with a warning, one when no --skf-root is given and another
|
|
485
|
+
naming the folder when it holds no version; ast_grep set to the
|
|
486
|
+
ast_grep.version of --extraction (the recipe runner's output), or to
|
|
487
|
+
"unknown" with a warning when that is null but an export of it has
|
|
488
|
+
extraction_method ast-grep; qmd set to --qmd-version, a leading "qmd "
|
|
489
|
+
(as `qmd --version` prints it) removed. A tool this run did not use
|
|
490
|
+
(no --qmd-version, no --extraction, or one with no export ast-grep
|
|
491
|
+
matched) keeps its old value. It removes last_update and update_type,
|
|
492
|
+
which the provenance map's update block records, and keeps every
|
|
493
|
+
other field. It refuses (exit 1, nothing written, one stderr line) a
|
|
494
|
+
--stats whose scripts_count or assets_count is not the length of
|
|
495
|
+
--lists' scripts or assets, a --generation-date of another form, an
|
|
496
|
+
--extraction that is not a JSON object, a file it cannot read, and a
|
|
497
|
+
skf-render-quick-metadata.py beside it that cannot be loaded.
|
|
387
498
|
|
|
388
499
|
Summary on stdout: {"status": "written", "map": <-o>, "entries":
|
|
389
500
|
{"added", "updated", "removed"}, "not_public": [{name, file}],
|
|
390
|
-
"
|
|
501
|
+
"relabeled": [{name, file, from, to}] (gap-driven),
|
|
502
|
+
"signature_source_backfilled": {tier: count}, "file_entries":
|
|
503
|
+
{"added", "updated", "removed"}, "warnings": [...]},
|
|
391
504
|
or {"status": "refused",
|
|
392
505
|
"blocking_unresolved": [{export_name, severity}]} with exit 3.
|
|
393
506
|
|
|
@@ -415,13 +528,26 @@ it scans):
|
|
|
415
528
|
no seed holds is not added, with a warning. confidence_breakdown counts
|
|
416
529
|
the exports by confidence, and T2 the exports with a qmd_evidence.
|
|
417
530
|
|
|
531
|
+
Aliases. Each --extraction entry_point_diff `public` and
|
|
532
|
+
`other_definitions` item (an other in-scope definition of a public
|
|
533
|
+
name), and each member of a namespace item, whose `local` differs from
|
|
534
|
+
its `name` gives its name to the record of its `local` name and file,
|
|
535
|
+
when one is seeded:
|
|
536
|
+
that record's `aliases` lists each such public name once, sorted (an
|
|
537
|
+
export re-exported under another name, `export { MockUniversalStore as
|
|
538
|
+
experimental_MockUniversalStore }`, keeps its declared name and carries
|
|
539
|
+
the public one). A record no item renames has no `aliases`, and with
|
|
540
|
+
no entry_point_diff (Quick tier, a runner exit 3, an export read by
|
|
541
|
+
eye) no record has any.
|
|
542
|
+
|
|
418
543
|
Public surface. When --extraction's scope.type is public-api (in any
|
|
419
544
|
case) and it has an entry_point_diff, each export of a language family
|
|
420
545
|
whose entry_points.by_language status is `barrel` is marked `public`:
|
|
421
546
|
true when its name and file are a pair of the runner's entry_point_diff
|
|
422
|
-
`public` (its name, or the name `local` gives it
|
|
423
|
-
`extraction_gaps`, or when a `public` item of its name
|
|
424
|
-
no file; false otherwise. A `public` item with `via`
|
|
547
|
+
`public` or `other_definitions` (its name, or the name `local` gives it
|
|
548
|
+
in that file) or `extraction_gaps`, or when a `public` item of its name
|
|
549
|
+
and family has no file; false otherwise. A `public` item with `via`
|
|
550
|
+
namespace and a
|
|
425
551
|
`members` list (what the submodule passes on, which users reach as
|
|
426
552
|
`pkg.sub.name`, and which counts.exports_public_api counts in its place)
|
|
427
553
|
gives its members' pairs (each member's name, or its `local` one, and
|
|
@@ -450,8 +576,11 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
|
450
576
|
but a MOVED_EXPORT needs the tree, with the first reason that fits:
|
|
451
577
|
a DELETED_EXPORT "a public API recount from the tree (rule R1)", a
|
|
452
578
|
provenance_completeness entry "a line from the tree (rule R3)", one
|
|
453
|
-
whose map_match is not-found
|
|
454
|
-
|
|
579
|
+
whose map_match is not-found with no candidate that has a
|
|
580
|
+
source_file "a line and a signature from the tree", any other (a
|
|
581
|
+
not-found with such a candidate, a citation of another file than
|
|
582
|
+
the map's entries, among them) "a signature from the tree". Any
|
|
583
|
+
such entry: {"status":
|
|
455
584
|
"drift-blocked", "drift_blocked": [{gap_id, name, change_category,
|
|
456
585
|
severity, reason}]}, and no spot-check runs.
|
|
457
586
|
2. A DELETED_EXPORT without a `rescope` that holds its `amendment` and
|
|
@@ -467,10 +596,28 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
|
467
596
|
its remediation's "definition line(s) (...)"
|
|
468
597
|
NEW_EXPORT, MODIFIED_EXPORT
|
|
469
598
|
map_match found checked at the map entry (its export_type)
|
|
470
|
-
ambiguous unknown
|
|
599
|
+
ambiguous unknown, unknown_reason ambiguous (several
|
|
600
|
+
map entries of its name: no check runs)
|
|
471
601
|
not-found checked at its source_citation (no export
|
|
472
|
-
type) when it has one
|
|
473
|
-
|
|
602
|
+
type) when it has one. When the map holds
|
|
603
|
+
entries of its name (in_map, which a
|
|
604
|
+
not-found has only for a citation of
|
|
605
|
+
another file than theirs), one of them
|
|
606
|
+
with a source_file, and no check pins a
|
|
607
|
+
line (unknown or missing): unknown,
|
|
608
|
+
unknown_reason cited-elsewhere, whatever
|
|
609
|
+
its severity (a provenance_completeness
|
|
610
|
+
one too), and never re-extracted, so no
|
|
611
|
+
other file's declaration is read for it;
|
|
612
|
+
its record's citation_check keeps the
|
|
613
|
+
check's cause (missing, or its
|
|
614
|
+
unknown_reason) and its map_locations the
|
|
615
|
+
`file:line` (or file) of each candidate
|
|
616
|
+
with a source_file, both null on any other
|
|
617
|
+
record. Candidates with no source_file
|
|
618
|
+
(placeholders) do not count. Otherwise,
|
|
619
|
+
when that pins no line
|
|
620
|
+
(unknown or missing), or with no
|
|
474
621
|
citation: targeted re-extraction (§4a) for a
|
|
475
622
|
provenance_completeness entry with a source
|
|
476
623
|
root, for a blocking one whatever its
|
|
@@ -480,37 +627,78 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
|
480
627
|
A check gives `verified` (the line is a definition line, or a module
|
|
481
628
|
or package entry whose file exists), `moved` (one definition line,
|
|
482
629
|
not the recorded one: new_location), `missing` (no such file) or
|
|
483
|
-
`unknown` (several
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
unknown, with a `provenance: spot-
|
|
490
|
-
warning
|
|
630
|
+
`unknown`, its unknown_reason `several-definitions` (several
|
|
631
|
+
definition lines, none the recorded one) or `no-definition` (none).
|
|
632
|
+
A file the rules cover no language of takes the answered
|
|
633
|
+
`declaring_line`: the recorded line is verified, another is moved,
|
|
634
|
+
null is unknown (`unchecked-language`). With no --source-root every
|
|
635
|
+
check is unknown (`no-source-root`); with no verifier every check is
|
|
636
|
+
unknown (`verifier-missing`), with a `provenance: spot-checks not
|
|
637
|
+
run` warning; a check the verifier fails on (a file it cannot read,
|
|
638
|
+
or any error) is unknown (`spot-check-failed`), with a `provenance:
|
|
639
|
+
spot-check failed for {name}: {error}` warning; a map entry with no
|
|
640
|
+
file or line is unknown (`no-line`). UNKNOWN_REASONS gives each its
|
|
641
|
+
words. Every other outcome has a null unknown_reason. A map_match
|
|
642
|
+
whose entry carries its own export_name (skf-parse-gaps matched it
|
|
643
|
+
by its reexported_as) is checked under that name.
|
|
644
|
+
A NEW_EXPORT or MODIFIED_EXPORT the map holds entries of its name
|
|
645
|
+
for (its map_match found or ambiguous, or not-found with a
|
|
646
|
+
candidate that has a source_file) never goes to targeted
|
|
647
|
+
re-extraction, whatever its severity: one that ends missing or
|
|
648
|
+
unknown keeps its map entries unchanged and adds none, and apply
|
|
649
|
+
warns (see "Apply").
|
|
650
|
+
A NEW_EXPORT or MODIFIED_EXPORT a check gives verified or moved, at
|
|
651
|
+
its map entry or at a citation the map holds no entry of its name
|
|
652
|
+
for, is located: its signature_at is the checked file and the
|
|
653
|
+
recorded line (verified) or the new one (moved). One whose citation
|
|
654
|
+
names another file than the map's entries of its name is not (apply
|
|
655
|
+
warns, as before), nor one whose file is absolute or outside
|
|
656
|
+
--source-root, links resolved, nor a NEW_EXPORT --judgments answers
|
|
657
|
+
internal-unreachable (one not yet answered is located, so its scan
|
|
658
|
+
may go unused).
|
|
491
659
|
--plan stops here: {"status": "planned", "reextract": {"entries":
|
|
492
660
|
[{gap_id, name, severity, resolved_paths}], "files": [the union of
|
|
493
|
-
their resolved_paths
|
|
494
|
-
|
|
661
|
+
their resolved_paths, then of the located exports' files],
|
|
662
|
+
"signatures": [{gap_id, name, file, line} of each located export]},
|
|
663
|
+
"warnings"}, the files also written to --files-out as a JSON array.
|
|
495
664
|
4. Targeted re-extraction (--remediation-records, `records`' output
|
|
496
665
|
over those files): each routed entry with resolved_paths takes the
|
|
497
666
|
first export of its name in the blocks of those files, in their
|
|
498
|
-
order
|
|
667
|
+
order, else the first whose `aliases` hold it (matched by name, or
|
|
668
|
+
by the public name the runner gives its local name in that file,
|
|
669
|
+
see "Records"): `re-extracted`, new_location its record's location,
|
|
499
670
|
resolution_source remediation-paths, and the record copied into
|
|
500
671
|
`files` as `records` wrote it, less its `public` mark (a normal
|
|
501
|
-
update's)
|
|
672
|
+
update's) and its `aliases`, with the answered `docstring` added. A
|
|
673
|
+
record matched by an alias is copied with `name` the public name and
|
|
674
|
+
`local` its own name. The aliases come from the runner's
|
|
675
|
+
entry_point_diff, so at Forge tier and above only: at Quick tier, or
|
|
676
|
+
when the runner exits 3 and §4a reads the files by eye, a gap naming
|
|
677
|
+
a renamed re-export by its public name matches no record.
|
|
502
678
|
An entry with no match, or no path to scan, is unresolved, except a
|
|
503
679
|
missing-export or missing-type one that is not blocking, which is
|
|
504
680
|
unknown. Any unresolved entry: {"status": "unresolved",
|
|
505
681
|
"unresolved": [{gap_id, name, severity, remediation_paths,
|
|
506
682
|
rejected_paths, files_scanned, exports_found_in_scan}]}.
|
|
683
|
+
Each located export takes the one record of its name (or of its map
|
|
684
|
+
entry's own export_name) in the block of its file at its line, else
|
|
685
|
+
the one with no line: the record copied into `files` as above, under
|
|
686
|
+
the gap's name, less its `public` mark and `aliases`, with no
|
|
687
|
+
docstring asked
|
|
688
|
+
(one copy per name and location, or per record with no location),
|
|
689
|
+
and its type, params, return_type, extraction_method and
|
|
690
|
+
ast_node_type on the verification record. One with no such record
|
|
691
|
+
keeps them null, with a `provenance: {name}: located at
|
|
692
|
+
{file}:{line}, but no record of it there; signature not repaired`
|
|
693
|
+
warning (§4a's by-eye pass may add it to --export-details). Files to
|
|
694
|
+
match, of either kind, with no --remediation-records are an error.
|
|
507
695
|
5. The public-reachability gate: every NEW_EXPORT that is verified,
|
|
508
696
|
moved or re-extracted takes the answered `reachability`, `public` or
|
|
509
697
|
`internal-unreachable`. An internal one keeps its outcome in its
|
|
510
|
-
verification record, its record is not in
|
|
511
|
-
it under `reclassified` instead of its
|
|
512
|
-
`reclassified` (merge queues it as a
|
|
513
|
-
entry for it).
|
|
698
|
+
verification record (with no signature), its record is not in
|
|
699
|
+
`files`, `counts` tallies it under `reclassified` instead of its
|
|
700
|
+
outcome, and it is listed in `reclassified` (merge queues it as a
|
|
701
|
+
metadata update; apply writes no entry for it).
|
|
514
702
|
|
|
515
703
|
Any answer missing in step 3 or 5 (a `docstring` too, for each
|
|
516
704
|
re-extracted entry): {"status": "needs-judgment", "needs_judgment":
|
|
@@ -522,26 +710,32 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
|
522
710
|
"unlabeled", "T2": 0} (each manifest entry by the extraction_method of
|
|
523
711
|
its re-extracted record or, in the map, of its map entry; a pinned cited
|
|
524
712
|
export the map lacks and the gate passed is T1-low), "verification":
|
|
525
|
-
[one record per export-bearing entry:
|
|
526
|
-
severity, verification, in_map, map_entry,
|
|
527
|
-
source_citation, new_location, unknown_reason,
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
713
|
+
[one record per export-bearing entry: gap_id, export_name,
|
|
714
|
+
gap_category, severity, verification, in_map, map_entry,
|
|
715
|
+
provenance_citation, source_citation, new_location, unknown_reason,
|
|
716
|
+
citation_check, map_locations, pinned_definition_lines,
|
|
717
|
+
resolution_source, reachability, and
|
|
718
|
+
export_type, params, return_type, extraction_method and ast_node_type
|
|
719
|
+
(a located export's record's, step 4; else null, and apply takes the
|
|
720
|
+
signature of an entry it adds from merge.md's merge-records.json)],
|
|
721
|
+
"files": [...]}, and prints {"status": "written",
|
|
531
722
|
"output", "gap_count", "files_extracted", "exports_extracted",
|
|
532
723
|
"confidence_breakdown", "counts" (per verification outcome, and
|
|
533
724
|
`reclassified`), "reclassified", "forwarded", "targeted_reextraction",
|
|
534
|
-
"warnings"}. When targeted re-extraction
|
|
535
|
-
|
|
536
|
-
|
|
725
|
+
"warnings"}. When §4a ran (targeted re-extraction, or a located
|
|
726
|
+
export's signature), `targeted_reextraction` is {resolved_count,
|
|
727
|
+
files_scanned (the targeted re-extraction's files),
|
|
728
|
+
signature_files_scanned (the located exports' files), exports_matched,
|
|
729
|
+
tier}, also appended to --evidence as one JSON line keyed
|
|
730
|
+
`targeted_reextraction`.
|
|
537
731
|
|
|
538
732
|
Baseline gaps (update-skill detect-changes Category B, step 1):
|
|
539
733
|
|
|
540
734
|
The recipe runner never reports a module, an alias that renames, a
|
|
541
|
-
dunder such as `__version__` or another
|
|
542
|
-
would read each such map entry as a deleted export. baseline-gaps
|
|
543
|
-
for step 2 to read by eye, each entry of a file --files lists
|
|
544
|
-
modified files, Category A's modified-files.json) whose name and file
|
|
735
|
+
dunder such as `__version__` or another Python or Rust `_` name, so the
|
|
736
|
+
diff would read each such map entry as a deleted export. baseline-gaps
|
|
737
|
+
lists, for step 2 to read by eye, each entry of a file --files lists
|
|
738
|
+
(the modified files, Category A's modified-files.json) whose name and file
|
|
545
739
|
no export of --extraction (the runner's JSON, or {"exports": []} when
|
|
546
740
|
it could not run) holds, under its name or its re-export target, and
|
|
547
741
|
whose file still declares it. The selection and the declaration rule
|
|
@@ -563,9 +757,10 @@ Exit codes:
|
|
|
563
757
|
0 operation succeeded (for gap-records, whatever its `status`)
|
|
564
758
|
1 user error (malformed JSON, bad path, malformed provenance file, a
|
|
565
759
|
helper file that is not the output it names, a gap-records answer of
|
|
566
|
-
the wrong type, targeted re-extraction
|
|
567
|
-
--remediation-records, or, for
|
|
568
|
-
no folder or a verifier that
|
|
760
|
+
the wrong type, files to match (targeted re-extraction or a located
|
|
761
|
+
export's signature) with no --remediation-records, or, for
|
|
762
|
+
baseline-gaps, a source root that is no folder or a verifier that
|
|
763
|
+
cannot be loaded)
|
|
569
764
|
3 apply refused: a blocking gap reached the provenance write with no
|
|
570
765
|
source line (nothing written)
|
|
571
766
|
"""
|
|
@@ -1330,6 +1525,8 @@ REPLACE_WAIT_SECONDS = 5.0 # how long a Windows rename onto a file held open is
|
|
|
1330
1525
|
# The update block apply sets at the top level of the map, in this order.
|
|
1331
1526
|
UPDATE_BLOCK_KEYS = ("last_update", "update_type", "test_report_run_id", "files_changed", "exports_affected",
|
|
1332
1527
|
"confidence_tier", "manual_sections_preserved")
|
|
1528
|
+
# The parts of a signature a gap-driven repair writes from a verification record, else from merge's records.
|
|
1529
|
+
SIGNATURE_PARTS = ("export_type", "params", "return_type")
|
|
1333
1530
|
|
|
1334
1531
|
|
|
1335
1532
|
def _labels(method: object, node: object) -> dict:
|
|
@@ -1457,16 +1654,17 @@ def _without_none(fields: dict, keep: tuple = ()) -> dict:
|
|
|
1457
1654
|
|
|
1458
1655
|
def _new_entry(fields: dict, library: str) -> dict:
|
|
1459
1656
|
"""An entry in create-skill's key order, from the fields a record gives."""
|
|
1460
|
-
order = ("export_name", "export_type", "source_library", "params", "return_type", "source_file",
|
|
1657
|
+
order = ("export_name", "reexported_as", "export_type", "source_library", "params", "return_type", "source_file",
|
|
1461
1658
|
"source_line", "confidence", "extraction_method", "ast_node_type", "signature_source")
|
|
1462
1659
|
merged = {**fields, "source_library": fields.get("source_library") or library}
|
|
1463
1660
|
out = {k: merged[k] for k in order if k in merged and (merged[k] is not None or k == "ast_node_type")}
|
|
1464
1661
|
return out
|
|
1465
1662
|
|
|
1466
1663
|
|
|
1467
|
-
def _rewrite(entry: dict, fields: dict) -> dict:
|
|
1468
|
-
"""The entry with `fields` set in place; keys the update does not set keep
|
|
1469
|
-
|
|
1664
|
+
def _rewrite(entry: dict, fields: dict, drop: tuple = ()) -> dict:
|
|
1665
|
+
"""The entry with `fields` set in place and the keys `drop` names removed; keys the update does not set keep
|
|
1666
|
+
their values."""
|
|
1667
|
+
out = {k: v for k, v in entry.items() if k not in drop}
|
|
1470
1668
|
for key, value in fields.items():
|
|
1471
1669
|
if value is None and key not in ("ast_node_type",):
|
|
1472
1670
|
continue
|
|
@@ -1490,19 +1688,87 @@ class _Map:
|
|
|
1490
1688
|
"provenance_version": "2.0", "skill_name": skill_name, "skill_type": "single", "entries": []}
|
|
1491
1689
|
if not isinstance(self.doc.get("entries"), list):
|
|
1492
1690
|
raise ValueError("provenance `entries` must be an array")
|
|
1493
|
-
libraries
|
|
1494
|
-
|
|
1495
|
-
self.
|
|
1691
|
+
# the map's libraries in entry order, and each file's, as the map stood before the run (#808)
|
|
1692
|
+
self.libraries: list[str] = []
|
|
1693
|
+
self.libraries_by_file: dict[str, set[str]] = {}
|
|
1694
|
+
for entry in self.doc["entries"]:
|
|
1695
|
+
library = entry.get("source_library") if isinstance(entry, dict) else None
|
|
1696
|
+
if not (isinstance(library, str) and library):
|
|
1697
|
+
continue
|
|
1698
|
+
if library not in self.libraries:
|
|
1699
|
+
self.libraries.append(library)
|
|
1700
|
+
if _file_of(entry):
|
|
1701
|
+
self.libraries_by_file.setdefault(_file_of(entry), set()).add(library)
|
|
1702
|
+
self.library = self.libraries[0] if self.libraries else skill_name
|
|
1703
|
+
# each folder's libraries (the entries under it, "" the root) and _library_at per path, built on first use
|
|
1704
|
+
self._libraries_under: dict[str, set[str]] | None = None
|
|
1705
|
+
self._library_of: dict[str, str | None] = {}
|
|
1706
|
+
# (name, file) of each new entry library_for gave the first library for want of a better one
|
|
1707
|
+
self.library_unresolved: list[tuple[str | None, str | None]] = []
|
|
1496
1708
|
self.added: list[str] = []
|
|
1497
1709
|
self.updated: list[str] = []
|
|
1498
1710
|
self.removed: list[str] = []
|
|
1499
1711
|
self.renamed: list[tuple[str, str]] = [] # (old name, new name)
|
|
1500
1712
|
self.touched_files: set[str] = set()
|
|
1501
1713
|
|
|
1502
|
-
def
|
|
1503
|
-
"""The
|
|
1504
|
-
|
|
1505
|
-
|
|
1714
|
+
def library_for(self, path: str | None, name: str | None = None, *, old: str | None = None) -> str:
|
|
1715
|
+
"""The source_library of a new entry for `name` in `path` (the record of a MOVED file, or of an export
|
|
1716
|
+
whose change names an old_file, passes that old path as `old`, tried second). A map naming one library (or
|
|
1717
|
+
none: --skill-name) gives it; in one naming several, the library of the file, else of the nearest folder
|
|
1718
|
+
(see _library_at). With neither, or no path, the map's first library, and the entry goes to
|
|
1719
|
+
`library_unresolved` for the summary's warning."""
|
|
1720
|
+
if len(self.libraries) <= 1:
|
|
1721
|
+
return self.library
|
|
1722
|
+
for candidate in (path, old):
|
|
1723
|
+
library = self._library_at(_norm_path(candidate))
|
|
1724
|
+
if library is not None:
|
|
1725
|
+
return library
|
|
1726
|
+
self.library_unresolved.append((name, _norm_path(path) or _norm_path(old)))
|
|
1727
|
+
return self.library
|
|
1728
|
+
|
|
1729
|
+
@staticmethod
|
|
1730
|
+
def _root_library(libraries: set[str]) -> str | None:
|
|
1731
|
+
"""The one library of the set, or of several the one that is a path prefix of all the others (`storybook`
|
|
1732
|
+
beside `storybook/test`); None when no library is."""
|
|
1733
|
+
return next((library for library in sorted(libraries)
|
|
1734
|
+
if all(other == library or other.startswith(library + "/") for other in libraries)), None)
|
|
1735
|
+
|
|
1736
|
+
def _library_at(self, path: str | None) -> str | None:
|
|
1737
|
+
"""The library the map's entries of `path` settle (see _root_library), else the one the entries under the
|
|
1738
|
+
nearest folder above it that holds entries settle; None when neither does. Memoized per path."""
|
|
1739
|
+
if not path:
|
|
1740
|
+
return None
|
|
1741
|
+
if path in self._library_of:
|
|
1742
|
+
return self._library_of[path]
|
|
1743
|
+
if self._libraries_under is None:
|
|
1744
|
+
self._libraries_under = {}
|
|
1745
|
+
for file, libraries in self.libraries_by_file.items():
|
|
1746
|
+
folders = file.split("/")[:-1]
|
|
1747
|
+
for depth in range(len(folders) + 1):
|
|
1748
|
+
prefix = "/".join(folders[:depth]) + "/" if depth else ""
|
|
1749
|
+
self._libraries_under.setdefault(prefix, set()).update(libraries)
|
|
1750
|
+
library = self._root_library(self.libraries_by_file.get(path, set()))
|
|
1751
|
+
if library is None:
|
|
1752
|
+
folders = path.split("/")[:-1]
|
|
1753
|
+
for depth in range(len(folders), -1, -1):
|
|
1754
|
+
under = self._libraries_under.get("/".join(folders[:depth]) + "/" if depth else "")
|
|
1755
|
+
if under:
|
|
1756
|
+
library = self._root_library(under)
|
|
1757
|
+
break
|
|
1758
|
+
self._library_of[path] = library
|
|
1759
|
+
return library
|
|
1760
|
+
|
|
1761
|
+
def find(self, name: str, path: str | None, line: int | None = None, *, reexport: bool = False) -> int | None:
|
|
1762
|
+
"""The index of the entry with this name (and file, and line when several share both); with `reexport`,
|
|
1763
|
+
when no entry has the name, of the entry whose reexported_as is the name (a gap names a renamed re-export
|
|
1764
|
+
by its public name)."""
|
|
1765
|
+
def hits_of(key) -> list[int]:
|
|
1766
|
+
return [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and key(e) == name
|
|
1767
|
+
and (path is None or _file_of(e) == path)]
|
|
1768
|
+
|
|
1769
|
+
hits = hits_of(_name_of)
|
|
1770
|
+
if not hits and reexport:
|
|
1771
|
+
hits = hits_of(lambda e: e.get("reexported_as"))
|
|
1506
1772
|
if len(hits) > 1 and line is not None:
|
|
1507
1773
|
hits = [i for i in hits if _line_of(self.doc["entries"][i].get("source_line")) == line]
|
|
1508
1774
|
return hits[0] if len(hits) == 1 else None
|
|
@@ -1516,9 +1782,9 @@ class _Map:
|
|
|
1516
1782
|
if entry.get("source_file"):
|
|
1517
1783
|
self.touched_files.add(_norm_path(entry["source_file"]))
|
|
1518
1784
|
|
|
1519
|
-
def update(self, index: int, fields: dict) -> None:
|
|
1785
|
+
def update(self, index: int, fields: dict, drop: tuple = ()) -> None:
|
|
1520
1786
|
before = self.doc["entries"][index]
|
|
1521
|
-
after = _rewrite(before, fields)
|
|
1787
|
+
after = _rewrite(before, fields, drop)
|
|
1522
1788
|
if after != before:
|
|
1523
1789
|
self.doc["entries"][index] = after
|
|
1524
1790
|
self.updated.append(_name_of(after))
|
|
@@ -1561,7 +1827,7 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
|
|
|
1561
1827
|
not_public.append({"name": name, "file": path})
|
|
1562
1828
|
return True
|
|
1563
1829
|
|
|
1564
|
-
def refresh(index: int | None, name: str, path: str, *, add: bool) -> None:
|
|
1830
|
+
def refresh(index: int | None, name: str, path: str, *, add: bool, old: str | None = None) -> None:
|
|
1565
1831
|
record = fresh.record(name, path)
|
|
1566
1832
|
if record is None:
|
|
1567
1833
|
warnings.append(f"provenance: {name} in {path}: no extraction record, entry "
|
|
@@ -1569,7 +1835,7 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
|
|
|
1569
1835
|
return
|
|
1570
1836
|
if index is None:
|
|
1571
1837
|
if add and not off_surface(name, path):
|
|
1572
|
-
pmap.add(_new_entry(record, pmap.
|
|
1838
|
+
pmap.add(_new_entry(record, pmap.library_for(path, name, old=old)))
|
|
1573
1839
|
return
|
|
1574
1840
|
pmap.update(index, _without_none(record, keep=("ast_node_type",)))
|
|
1575
1841
|
|
|
@@ -1591,7 +1857,7 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
|
|
|
1591
1857
|
if carried is None and off_surface(name, key_path):
|
|
1592
1858
|
continue
|
|
1593
1859
|
entry = _rewrite(carried, _without_none(record, keep=("ast_node_type",))) if carried \
|
|
1594
|
-
else _new_entry(record, pmap.
|
|
1860
|
+
else _new_entry(record, pmap.library_for(key_path, name, old=old))
|
|
1595
1861
|
pmap.add(entry)
|
|
1596
1862
|
continue
|
|
1597
1863
|
if status == "ADDED":
|
|
@@ -1624,9 +1890,13 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
|
|
|
1624
1890
|
if all(record.get(part) is not None for part in ("params", "return_type")
|
|
1625
1891
|
if entry.get(part) is not None):
|
|
1626
1892
|
fields["signature_source"] = record["signature_source"]
|
|
1893
|
+
elif _tier(entry.get("signature_source")) is None:
|
|
1894
|
+
# the signature it keeps was read by eye: set that, or the backfill would derive T1
|
|
1895
|
+
# from the ast-grep label set here
|
|
1896
|
+
fields["signature_source"] = "T1-low"
|
|
1627
1897
|
pmap.update(index, fields)
|
|
1628
1898
|
elif kind in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1629
|
-
refresh(index, name, path, add=True)
|
|
1899
|
+
refresh(index, name, path, add=True, old=_norm_path(change.get("old_file")))
|
|
1630
1900
|
for rename in _get_list(categories or {}, "category_c", "renamed_exports"):
|
|
1631
1901
|
if not isinstance(rename, dict):
|
|
1632
1902
|
continue
|
|
@@ -1663,7 +1933,7 @@ def _apply_docs_only(pmap: _Map, records: dict) -> tuple[int, int]:
|
|
|
1663
1933
|
"confidence": "T3", "signature_source": "T3"}
|
|
1664
1934
|
carried = old.get(_name_of(record))
|
|
1665
1935
|
entry = _rewrite(carried, _without_none(fields)) if carried else _new_entry(
|
|
1666
|
-
_without_none(fields), pmap.
|
|
1936
|
+
_without_none(fields), pmap.library_for(url, _name_of(record)))
|
|
1667
1937
|
pmap.add(entry)
|
|
1668
1938
|
return len(changed), len(exports)
|
|
1669
1939
|
|
|
@@ -1673,8 +1943,9 @@ def _drift_note(drift: tuple | None) -> str:
|
|
|
1673
1943
|
|
|
1674
1944
|
|
|
1675
1945
|
def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, drift: tuple | None,
|
|
1676
|
-
warnings: list[str]) -> tuple[int, int] | list[dict]:
|
|
1677
|
-
"""Gap-driven mode: one write per verification record. A list return is the refusal
|
|
1946
|
+
warnings: list[str], relabeled: list[dict]) -> tuple[int, int] | list[dict]:
|
|
1947
|
+
"""Gap-driven mode: one write per verification record. A list return is the refusal; each entry whose
|
|
1948
|
+
signature_source the repair changes goes to `relabeled`."""
|
|
1678
1949
|
verification = [r for r in records.get("verification") or [] if isinstance(r, dict) and _name_of(r)]
|
|
1679
1950
|
reextracted = _Fresh(records.get("files") or [], None, None)
|
|
1680
1951
|
merged = {}
|
|
@@ -1685,18 +1956,108 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
|
|
|
1685
1956
|
def in_map(r: dict) -> bool:
|
|
1686
1957
|
return bool(r.get("in_map")) or bool(r.get("map_entry"))
|
|
1687
1958
|
|
|
1959
|
+
def signature(r: dict) -> tuple[dict, tuple]:
|
|
1960
|
+
"""(the signature parts to set, the parts to remove) of an export: with the record §4a's extraction gave at
|
|
1961
|
+
its line (its verification record has an extraction_method), every part from it, a null params or
|
|
1962
|
+
return_type removed; with none, merge's."""
|
|
1963
|
+
if r.get("extraction_method") is not None:
|
|
1964
|
+
return _without_none({part: r.get(part) for part in SIGNATURE_PARTS}), \
|
|
1965
|
+
tuple(part for part in ("params", "return_type") if r.get(part) is None)
|
|
1966
|
+
extra = merged.get(_name_of(r), {})
|
|
1967
|
+
return _without_none({part: extra.get(part) for part in SIGNATURE_PARTS}), ()
|
|
1968
|
+
|
|
1969
|
+
def resign(r: dict, index: int, fields: dict) -> None:
|
|
1970
|
+
"""Write an entry the map holds with `fields` and, when §4a found the export's record, its signature; an
|
|
1971
|
+
ast-grep entry takes the record's signature_source, and its node kind when the record's export_type is
|
|
1972
|
+
another (a source-read entry keeps T1-low and no kind); confidence and extraction_method stay."""
|
|
1973
|
+
entry = pmap.doc["entries"][index]
|
|
1974
|
+
drop: tuple = ()
|
|
1975
|
+
method = r.get("extraction_method")
|
|
1976
|
+
if r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT") and method is not None \
|
|
1977
|
+
and r.get("reachability") != "internal-unreachable":
|
|
1978
|
+
parts, gone = signature(r)
|
|
1979
|
+
fields = {**fields, **parts}
|
|
1980
|
+
drop = tuple(part for part in gone if entry.get(part) is not None)
|
|
1981
|
+
if entry.get("extraction_method") in ("ast-grep", "ast_bridge"):
|
|
1982
|
+
fields["signature_source"] = _labels(method, None)["signature_source"]
|
|
1983
|
+
if parts.get("export_type") not in (None, entry.get("export_type")):
|
|
1984
|
+
fields["ast_node_type"] = r.get("ast_node_type")
|
|
1985
|
+
old, new = entry.get("signature_source"), fields["signature_source"]
|
|
1986
|
+
if old is not None and old != new:
|
|
1987
|
+
path = _file_of(entry)
|
|
1988
|
+
relabeled.append({"name": _name_of(entry), "file": path, "from": old, "to": new})
|
|
1989
|
+
warnings.append(f"provenance: {_name_of(entry)} in {path}: signature_source {old} relabeled "
|
|
1990
|
+
f"{new}, from the {method} record of its signature")
|
|
1991
|
+
pmap.update(index, fields, drop)
|
|
1992
|
+
|
|
1993
|
+
note = _drift_note(drift)
|
|
1994
|
+
|
|
1995
|
+
def why(r: dict) -> str:
|
|
1996
|
+
"""The cause of an `unknown` outcome its record gives, `<unknown_reason>: <words>`, else ''."""
|
|
1997
|
+
reason = r.get("unknown_reason")
|
|
1998
|
+
return f"{reason}: {UNKNOWN_REASONS[reason]}" if r.get("verification") == "unknown" \
|
|
1999
|
+
and reason in UNKNOWN_REASONS else ""
|
|
2000
|
+
|
|
2001
|
+
def unchecked(r: dict) -> str | None:
|
|
2002
|
+
"""The warning for a blocking NEW_EXPORT or MODIFIED_EXPORT the map holds that its spot-check could not
|
|
2003
|
+
locate (missing, or unknown for a cause its unknown_reason names), which stays unchanged: the gap, the
|
|
2004
|
+
export, the outcome, its cause and the one fix a re-run acts on; None for any other record."""
|
|
2005
|
+
outcome = r.get("verification")
|
|
2006
|
+
if outcome not in ("missing", "unknown") or not in_map(r) or not _blocking(r.get("severity")) \
|
|
2007
|
+
or r.get("gap_category") not in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
2008
|
+
return None
|
|
2009
|
+
name, gap = _name_of(r), r.get("gap_id")
|
|
2010
|
+
path = _location(r.get("map_entry"))[0]
|
|
2011
|
+
cause = f"its map entry's file{f' {path}' if path else ''} is gone" if outcome == "missing" else why(r)
|
|
2012
|
+
detail = "; ".join(part for part in (cause, note) if part)
|
|
2013
|
+
if r.get("unknown_reason") == "ambiguous": # a Source at the definition lets the re-run's lookup pick one
|
|
2014
|
+
fix = ("its entries are left unchanged. Fix its entries by hand, or give the gap a `file:line` Source at "
|
|
2015
|
+
"its definition when that file holds only one of them, then re-run update-skill")
|
|
2016
|
+
else: # the re-run checks the same map entry again: only a hand fix changes it
|
|
2017
|
+
fix = ("its entry is left unchanged. Fix the entry by hand (set its source_file and source_line to the "
|
|
2018
|
+
"line that defines it), then re-run update-skill")
|
|
2019
|
+
return f"provenance: {name}{f' ({gap})' if gap else ''}: {outcome}{f' ({detail})' if detail else ''}; {fix}"
|
|
2020
|
+
|
|
2021
|
+
def elsewhere(r: dict) -> str | None:
|
|
2022
|
+
"""The warning for a NEW_EXPORT or MODIFIED_EXPORT whose gap cites another file than the map's entries of its
|
|
2023
|
+
name (unknown, cited-elsewhere), whatever its severity: the gap, the export, the cited file and its check's
|
|
2024
|
+
cause, the record's map_locations and the fix for each side that may be wrong; None for any other record."""
|
|
2025
|
+
if r.get("verification") != "unknown" or r.get("unknown_reason") != "cited-elsewhere":
|
|
2026
|
+
return None
|
|
2027
|
+
name, gap = _name_of(r), r.get("gap_id")
|
|
2028
|
+
cited_file, cited_line = _location(r.get("source_citation"))
|
|
2029
|
+
cited = _where(cited_file, cited_line) or cited_file
|
|
2030
|
+
check = r.get("citation_check") if isinstance(r.get("citation_check"), str) else None
|
|
2031
|
+
words = "no such file" if check == "missing" else UNKNOWN_REASONS.get(check or "")
|
|
2032
|
+
where = [w for w in r.get("map_locations") or [] if isinstance(w, str) and w]
|
|
2033
|
+
detail = "; ".join(part for part in (
|
|
2034
|
+
why(r), f"it cites {cited}" if cited else "",
|
|
2035
|
+
f"its check there: {check}{f' ({words})' if words else ''}" if check else "",
|
|
2036
|
+
f"the map holds it at {', '.join(where)}" if where else "", note) if part)
|
|
2037
|
+
several = len(where) > 1
|
|
2038
|
+
kept = "its entries are left unchanged" if several else "its entry is left unchanged"
|
|
2039
|
+
head = f"provenance: {name}{f' ({gap})' if gap else ''}: unknown ({detail}); {kept}, and nothing is " \
|
|
2040
|
+
"re-extracted for it."
|
|
2041
|
+
if check in UNREAD: # nothing was read at the cited file: make the check read it first
|
|
2042
|
+
return f"{head} Nothing was read at the cited file: {UNREAD[check]}, then re-run update-skill"
|
|
2043
|
+
moved = "when an export moved, fix by hand whichever of those entries moved" if several \
|
|
2044
|
+
else "when the export moved, fix the map entry by hand"
|
|
2045
|
+
scope = "" if check == "missing" else (", and if the gap comes back, exclude the cited file in the brief's "
|
|
2046
|
+
"scope when the skill does not cover it")
|
|
2047
|
+
return (f"{head} When the map is right, re-run Test Skill, since the test compared another file's "
|
|
2048
|
+
f"declaration{scope}; {moved} (set its source_file and source_line to the line that defines it), "
|
|
2049
|
+
f"then re-run update-skill")
|
|
2050
|
+
|
|
1688
2051
|
blocking = [{"export_name": _name_of(r), "severity": r.get("severity")} for r in verification
|
|
1689
2052
|
if r.get("verification") == "unknown" and not in_map(r)
|
|
1690
|
-
and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT")
|
|
1691
|
-
and str(r.get("severity") or "").strip().lower() not in NON_BLOCKING]
|
|
2053
|
+
and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT") and _blocking(r.get("severity"))]
|
|
1692
2054
|
if blocking:
|
|
1693
2055
|
return blocking
|
|
1694
|
-
note = _drift_note(drift)
|
|
1695
2056
|
pinned_keys: set[tuple] = set()
|
|
1696
2057
|
for r in verification:
|
|
1697
2058
|
name, outcome, category = _name_of(r), r.get("verification"), r.get("gap_category")
|
|
1698
2059
|
entry_file, entry_line = _location(r.get("map_entry"))
|
|
1699
|
-
index = pmap.find(name, entry_file, entry_line) if r.get("map_entry") else None
|
|
2060
|
+
index = pmap.find(name, entry_file, entry_line, reexport=True) if r.get("map_entry") else None
|
|
1700
2061
|
lines = r.get("pinned_definition_lines")
|
|
1701
2062
|
r5 = f"; definition lines per the test report: {lines}" if category == "MOVED_EXPORT" else ""
|
|
1702
2063
|
if outcome == "rescoped":
|
|
@@ -1705,10 +2066,10 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
|
|
|
1705
2066
|
elif outcome == "verified" and index is not None:
|
|
1706
2067
|
if drift and category == "MOVED_EXPORT":
|
|
1707
2068
|
warnings.append(f"provenance: {name}: verified at HEAD only ({note}{r5})")
|
|
2069
|
+
resign(r, index, {})
|
|
1708
2070
|
elif outcome == "moved" and index is not None:
|
|
1709
2071
|
path, line = _location(r.get("new_location"))
|
|
1710
|
-
if path and line is not None
|
|
1711
|
-
pmap.update(index, {"source_file": path, "source_line": line})
|
|
2072
|
+
resign(r, index, {"source_file": path, "source_line": line} if path and line is not None else {})
|
|
1712
2073
|
elif outcome == "re-extracted":
|
|
1713
2074
|
if r.get("reachability") == "internal-unreachable":
|
|
1714
2075
|
continue # not public API: merge queued it as a metadata update
|
|
@@ -1717,9 +2078,13 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
|
|
|
1717
2078
|
if record is None:
|
|
1718
2079
|
warnings.append(f"provenance: {name}: re-extracted, but no record of it in the run's files")
|
|
1719
2080
|
continue
|
|
1720
|
-
|
|
2081
|
+
local = (reextracted.worker.get((name, path)) or {}).get("local")
|
|
2082
|
+
if isinstance(local, str) and local and local != name:
|
|
2083
|
+
# matched by the public name a re-export gives it: the entry keeps its declared name
|
|
2084
|
+
record = {**record, "export_name": local, "reexported_as": name}
|
|
2085
|
+
existing = pmap.find(name, path, reexport=True)
|
|
1721
2086
|
if existing is None:
|
|
1722
|
-
pmap.add(_new_entry(record, pmap.
|
|
2087
|
+
pmap.add(_new_entry(record, pmap.library_for(path, name)))
|
|
1723
2088
|
else:
|
|
1724
2089
|
pmap.update(existing, _without_none(record, keep=("ast_node_type",)))
|
|
1725
2090
|
elif outcome in ("verified", "moved") and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
@@ -1727,29 +2092,28 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
|
|
|
1727
2092
|
continue
|
|
1728
2093
|
where = r.get("source_citation") if outcome == "verified" else r.get("new_location")
|
|
1729
2094
|
path, line = _location(where)
|
|
1730
|
-
if not path or line is None or (name, path) in pinned_keys
|
|
2095
|
+
if not path or line is None or (name, path) in pinned_keys \
|
|
2096
|
+
or pmap.find(name, path, reexport=True) is not None:
|
|
1731
2097
|
continue
|
|
1732
2098
|
pinned_keys.add((name, path))
|
|
1733
|
-
|
|
2099
|
+
# the signature its record gives, else merge's; the source-read labels whatever the record's tool
|
|
1734
2100
|
pmap.add(_new_entry(_without_none({
|
|
1735
|
-
"export_name": name,
|
|
1736
|
-
"export_type": extra.get("export_type") or r.get("export_type"),
|
|
1737
|
-
"params": extra.get("params"), "return_type": extra.get("return_type"),
|
|
2101
|
+
"export_name": name, **signature(r)[0],
|
|
1738
2102
|
"source_file": path, "source_line": line, **_labels("source-read", None)},
|
|
1739
|
-
keep=("ast_node_type",)), pmap.
|
|
2103
|
+
keep=("ast_node_type",)), pmap.library_for(path, name)))
|
|
1740
2104
|
elif outcome == "unknown" and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1741
|
-
extra = merged.get(name, {})
|
|
1742
2105
|
pmap.add(_new_entry(_without_none({
|
|
1743
|
-
"export_name": name,
|
|
1744
|
-
"
|
|
1745
|
-
**_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library))
|
|
2106
|
+
"export_name": name, **signature(r)[0],
|
|
2107
|
+
**_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library_for(None, name)))
|
|
1746
2108
|
else:
|
|
1747
2109
|
if r.get("unknown_reason") == "drift-override":
|
|
1748
2110
|
warnings.append(f"provenance: {name}: unknown ({note}; no line taken from HEAD{r5})")
|
|
2111
|
+
elif (warning := elsewhere(r) or unchecked(r)) is not None:
|
|
2112
|
+
warnings.append(warning)
|
|
1749
2113
|
elif drift:
|
|
1750
|
-
warnings.append(f"provenance: {name}: {outcome} ({note}{r5})")
|
|
2114
|
+
warnings.append(f"provenance: {name}: {outcome} ({why(r) + '; ' if why(r) else ''}{note}{r5})")
|
|
1751
2115
|
else:
|
|
1752
|
-
warnings.append(f"provenance: {name}: {outcome}{r5}")
|
|
2116
|
+
warnings.append(f"provenance: {name}: {outcome}{f' ({why(r)})' if why(r) else ''}{r5}")
|
|
1753
2117
|
return len(pmap.touched_files), len(verification)
|
|
1754
2118
|
|
|
1755
2119
|
|
|
@@ -1830,6 +2194,9 @@ def exports_after(metadata: object, pmap: _Map, update_type: str) -> list:
|
|
|
1830
2194
|
return []
|
|
1831
2195
|
current = metadata.get("exports") if isinstance(metadata.get("exports"), list) else []
|
|
1832
2196
|
held = {_name_of(e) for e in pmap.doc["entries"] if isinstance(e, dict)} - {None}
|
|
2197
|
+
# an entry added under its declared name, re-exported under a public one: exports[] lists the public name
|
|
2198
|
+
public = {_name_of(e): e["reexported_as"] for e in pmap.doc["entries"]
|
|
2199
|
+
if isinstance(e, dict) and isinstance(e.get("reexported_as"), str) and e["reexported_as"]}
|
|
1833
2200
|
renamed = dict(pmap.renamed)
|
|
1834
2201
|
gone = ({n for n in pmap.removed if n} | set(renamed)) - held
|
|
1835
2202
|
if update_type == "full":
|
|
@@ -1849,8 +2216,8 @@ def exports_after(metadata: object, pmap: _Map, update_type: str) -> list:
|
|
|
1849
2216
|
put(new, new if new in held else None)
|
|
1850
2217
|
else:
|
|
1851
2218
|
put(item, name)
|
|
1852
|
-
for name in pmap.added + [new for _old, new in pmap.renamed]:
|
|
1853
|
-
put(name, name if name in held else None)
|
|
2219
|
+
for name in [public.get(n, n) for n in pmap.added] + [new for _old, new in pmap.renamed]:
|
|
2220
|
+
put(name, name if name in held or name in public.values() else None)
|
|
1854
2221
|
return out
|
|
1855
2222
|
|
|
1856
2223
|
|
|
@@ -1914,6 +2281,50 @@ def file_rows_after(metadata: dict, pmap: _Map, source_root: Path | None) -> dic
|
|
|
1914
2281
|
return lists
|
|
1915
2282
|
|
|
1916
2283
|
|
|
2284
|
+
# the signature_source tiers skf-render-metadata-stats.py bins, in their spelling (it compares them case-insensitively)
|
|
2285
|
+
SIGNATURE_SOURCES = ("T1", "T1-low", "T2", "T3")
|
|
2286
|
+
WARNING_LIST_CAP = 10 # the items a warning lists before "and N more"
|
|
2287
|
+
|
|
2288
|
+
|
|
2289
|
+
def _capped(items: list[str]) -> str:
|
|
2290
|
+
"""The items joined with ", ", the first WARNING_LIST_CAP of them and "and N more" for the rest."""
|
|
2291
|
+
more = len(items) - WARNING_LIST_CAP
|
|
2292
|
+
return ", ".join(items[:WARNING_LIST_CAP] + ([f"and {more} more"] if more > 0 else []))
|
|
2293
|
+
|
|
2294
|
+
|
|
2295
|
+
def _tier(value: object) -> str | None:
|
|
2296
|
+
"""A signature_source or confidence tier in its SIGNATURE_SOURCES spelling; None for any other value."""
|
|
2297
|
+
if not isinstance(value, str):
|
|
2298
|
+
return None
|
|
2299
|
+
return next((tier for tier in SIGNATURE_SOURCES if tier.upper() == value.strip().upper()), None)
|
|
2300
|
+
|
|
2301
|
+
|
|
2302
|
+
def _backfill_signature_source(pmap: _Map) -> tuple[dict[str, int], list[tuple[object, object, str]]]:
|
|
2303
|
+
"""Give each entry whose signature_source is missing or not a tier the stats helper bins (a map written before
|
|
2304
|
+
the key existed, #802) the one its labels give: its confidence when that names T1-low, T2 or T3, else T1 for
|
|
2305
|
+
an extraction_method of ast-grep or ast_bridge (compared stripped and case-insensitively) and T1-low for any
|
|
2306
|
+
other (relabel-rule.md: T1 needs evidence an ast-grep rule matched). Returns (the fills counted by value, in
|
|
2307
|
+
tier order; (name, old value, new value) of each unrecognized value replaced, which the map no longer holds);
|
|
2308
|
+
the names are not listed as updated."""
|
|
2309
|
+
counts: dict[str, int] = {}
|
|
2310
|
+
replaced: list[tuple[object, object, str]] = []
|
|
2311
|
+
for entry in pmap.doc["entries"]:
|
|
2312
|
+
if not isinstance(entry, dict) or _tier(entry.get("signature_source")) is not None:
|
|
2313
|
+
continue
|
|
2314
|
+
confidence, method = _tier(entry.get("confidence")), entry.get("extraction_method")
|
|
2315
|
+
if confidence in ("T1-low", "T2", "T3"):
|
|
2316
|
+
value = confidence
|
|
2317
|
+
elif isinstance(method, str) and method.strip().lower() in ("ast-grep", "ast_bridge"):
|
|
2318
|
+
value = "T1"
|
|
2319
|
+
else:
|
|
2320
|
+
value = "T1-low"
|
|
2321
|
+
if entry.get("signature_source") is not None:
|
|
2322
|
+
replaced.append((entry.get("export_name"), entry["signature_source"], value))
|
|
2323
|
+
entry["signature_source"] = value
|
|
2324
|
+
counts[value] = counts.get(value, 0) + 1
|
|
2325
|
+
return {tier: counts[tier] for tier in SIGNATURE_SOURCES if tier in counts}, replaced
|
|
2326
|
+
|
|
2327
|
+
|
|
1917
2328
|
def apply_update(*, update_type: str, provenance: dict | None, skill_name: str, manifest: dict | None = None,
|
|
1918
2329
|
categories: dict | None = None, extraction: dict | None = None, details: dict | None = None,
|
|
1919
2330
|
records: dict | None = None, merge_records: dict | None = None, compare: dict | None = None,
|
|
@@ -1936,9 +2347,10 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
|
|
|
1936
2347
|
pmap = _Map(provenance, skill_name)
|
|
1937
2348
|
warnings: list[str] = []
|
|
1938
2349
|
not_public: list[dict] = []
|
|
2350
|
+
relabeled: list[dict] = []
|
|
1939
2351
|
mode = records.get("mode")
|
|
1940
2352
|
if update_type == "gap-driven":
|
|
1941
|
-
outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings)
|
|
2353
|
+
outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings, relabeled)
|
|
1942
2354
|
if isinstance(outcome, list):
|
|
1943
2355
|
return {}, {"status": "refused", "blocking_unresolved": outcome}
|
|
1944
2356
|
files, exports = outcome
|
|
@@ -1951,7 +2363,7 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
|
|
|
1951
2363
|
index = pmap.find(name, path)
|
|
1952
2364
|
record = fresh.record(name, path)
|
|
1953
2365
|
if index is None:
|
|
1954
|
-
pmap.add(_new_entry(record, pmap.
|
|
2366
|
+
pmap.add(_new_entry(record, pmap.library_for(path, name)))
|
|
1955
2367
|
else:
|
|
1956
2368
|
pmap.update(index, _without_none(record, keep=("ast_node_type",)))
|
|
1957
2369
|
files = len({path for _name, path in fresh.all_keys()})
|
|
@@ -1963,6 +2375,23 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
|
|
|
1963
2375
|
f"({fresh.surface_problem}): every new export was added")
|
|
1964
2376
|
if not_public:
|
|
1965
2377
|
warnings.append(_not_public_warning(not_public))
|
|
2378
|
+
if pmap.library_unresolved:
|
|
2379
|
+
count = len(pmap.library_unresolved)
|
|
2380
|
+
named = _capped([f"{name} in {path}" if path else f"{name} (no file)"
|
|
2381
|
+
for name, path in pmap.library_unresolved])
|
|
2382
|
+
warnings.append(f"provenance: {count} new entr{'y' if count == 1 else 'ies'} took source_library "
|
|
2383
|
+
f"{pmap.library}, the map's first, as the map names several libraries "
|
|
2384
|
+
f"({', '.join(pmap.libraries)}) and each entry's file or folder does not settle which: "
|
|
2385
|
+
f"{named}; check each entry's source_library")
|
|
2386
|
+
backfilled, replaced = _backfill_signature_source(pmap)
|
|
2387
|
+
if backfilled:
|
|
2388
|
+
count = sum(backfilled.values())
|
|
2389
|
+
listed = [f"{name}: {old if isinstance(old, str) else json.dumps(old)} -> {new}" for name, old, new in replaced]
|
|
2390
|
+
values = f"; unrecognized values replaced: {_capped(listed)}" if replaced else ""
|
|
2391
|
+
warnings.append(f"provenance: backfilled signature_source on {count} entr{'y' if count == 1 else 'ies'} "
|
|
2392
|
+
f"that had none or an unrecognized value (a map written before the key existed), from each "
|
|
2393
|
+
f"one's confidence and extraction_method: "
|
|
2394
|
+
f"{', '.join(f'{n} {value}' for value, n in backfilled.items())}{values}")
|
|
1966
2395
|
file_changes = _apply_files(pmap, compare, new_files, promoted, source_root)
|
|
1967
2396
|
doc = pmap.doc
|
|
1968
2397
|
for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
|
|
@@ -1983,6 +2412,8 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
|
|
|
1983
2412
|
"status": "written",
|
|
1984
2413
|
"entries": {"added": pmap.added, "updated": pmap.updated, "removed": pmap.removed},
|
|
1985
2414
|
"not_public": not_public,
|
|
2415
|
+
"relabeled": relabeled,
|
|
2416
|
+
"signature_source_backfilled": backfilled,
|
|
1986
2417
|
"file_entries": file_changes,
|
|
1987
2418
|
"warnings": warnings,
|
|
1988
2419
|
}
|
|
@@ -1997,11 +2428,98 @@ GENERATION_DATE_RE = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ")
|
|
|
1997
2428
|
STATS_HELPER_FIELDS = ("exports_documented", "exports_public_api", "exports_internal", "exports_total",
|
|
1998
2429
|
"public_api_coverage", "total_coverage", "scripts_count", "assets_count",
|
|
1999
2430
|
"pattern_surfaces_documented", "effective_denominator")
|
|
2431
|
+
# the other stats fields create-skill writes, by the skill's scope_type: a component library's own counts (the
|
|
2432
|
+
# Component Library stats of compile-assembly-rules.md, its stats_extra) and a reference app's notes
|
|
2433
|
+
STATS_CREATE_FIELDS = {
|
|
2434
|
+
"component-library": ("components_registered", "components_documented", "props_interfaces_extracted",
|
|
2435
|
+
"components_unique", "demo_files_excluded", "design_variants"),
|
|
2436
|
+
"reference-app": ("notes",),
|
|
2437
|
+
}
|
|
2438
|
+
# a stats key that explains effective_denominator, dropped when this run's denominator differs from the old one
|
|
2439
|
+
DENOMINATOR_NOTE_PREFIX = "effective_denominator_"
|
|
2440
|
+
SKF_NO_ROOT = ("metadata: tool_versions.skf is unknown: no --skf-root was given; pass the installed SKF's folder "
|
|
2441
|
+
"({project-root}/_bmad/skf)")
|
|
2442
|
+
AST_GREP_UNKNOWN = ("metadata: tool_versions.ast_grep is unknown: --extraction holds exports ast-grep matched but no "
|
|
2443
|
+
"ast_grep.version")
|
|
2444
|
+
_QMD_PREFIX = re.compile(r"^qmd\s+", re.IGNORECASE) # what `qmd --version` prints first
|
|
2445
|
+
|
|
2446
|
+
|
|
2447
|
+
def _tool_version(value: object) -> str | None:
|
|
2448
|
+
return value.strip() if isinstance(value, str) and value.strip() else None
|
|
2449
|
+
|
|
2450
|
+
|
|
2451
|
+
def _skf_unknown(skf_root: str | Path) -> str:
|
|
2452
|
+
return (f"metadata: tool_versions.skf is unknown: {Path(skf_root).as_posix()} holds no package.json or VERSION "
|
|
2453
|
+
f"file with a version")
|
|
2454
|
+
|
|
2455
|
+
|
|
2456
|
+
def _kept_warning(key: str) -> str:
|
|
2457
|
+
return f"metadata: stats.{key} kept from an earlier run; check it still describes this run's counts"
|
|
2458
|
+
|
|
2459
|
+
|
|
2460
|
+
def tool_versions_after(old: object, *, skf_version: str | None, skf_root: str | Path | None,
|
|
2461
|
+
ast_grep_version: str | None, qmd_version: str | None) -> tuple[dict, list[str]]:
|
|
2462
|
+
"""(the tool_versions an update leaves, its warnings) (see "Metadata" in the module docstring)."""
|
|
2463
|
+
warnings: list[str] = []
|
|
2464
|
+
if isinstance(old, dict):
|
|
2465
|
+
tools = dict(old)
|
|
2466
|
+
else:
|
|
2467
|
+
tools = {"ast_grep": None, "qmd": None}
|
|
2468
|
+
if old is not None:
|
|
2469
|
+
kind = {bool: "a boolean", str: "a string", list: "an array"}.get(type(old), "a number")
|
|
2470
|
+
warnings.append(f"metadata: tool_versions was {kind}, not an object; replaced with the versions this run "
|
|
2471
|
+
f"records")
|
|
2472
|
+
ast_grep = _tool_version(ast_grep_version)
|
|
2473
|
+
if ast_grep is not None:
|
|
2474
|
+
tools["ast_grep"] = ast_grep
|
|
2475
|
+
if ast_grep == "unknown":
|
|
2476
|
+
warnings.append(AST_GREP_UNKNOWN)
|
|
2477
|
+
qmd = _tool_version(qmd_version)
|
|
2478
|
+
if qmd is not None:
|
|
2479
|
+
qmd = _tool_version(_QMD_PREFIX.sub("", qmd, count=1))
|
|
2480
|
+
if qmd is not None:
|
|
2481
|
+
tools["qmd"] = qmd
|
|
2482
|
+
skf = _tool_version(skf_version)
|
|
2483
|
+
if skf is None or skf == "unknown":
|
|
2484
|
+
skf = "unknown"
|
|
2485
|
+
warnings.append(SKF_NO_ROOT if skf_root is None else _skf_unknown(skf_root))
|
|
2486
|
+
tools["skf"] = skf
|
|
2487
|
+
return tools, warnings
|
|
2488
|
+
|
|
2489
|
+
|
|
2490
|
+
def stats_after(old_stats: dict, new_stats: dict, scope_type: object) -> tuple[dict, list[str], list[str]]:
|
|
2491
|
+
"""(the stats an update leaves, dropped_stats, their warnings) (see "Metadata" in the module docstring)."""
|
|
2492
|
+
exempt = STATS_CREATE_FIELDS.get(scope_type, ()) if isinstance(scope_type, str) else ()
|
|
2493
|
+
denominator = new_stats.get("effective_denominator")
|
|
2494
|
+
recounted = old_stats.get("effective_denominator") != denominator
|
|
2495
|
+
kept: dict = {}
|
|
2496
|
+
dropped: list[str] = []
|
|
2497
|
+
warnings: list[str] = []
|
|
2498
|
+
for key, value in old_stats.items():
|
|
2499
|
+
if key in new_stats:
|
|
2500
|
+
continue
|
|
2501
|
+
if key in STATS_HELPER_FIELDS:
|
|
2502
|
+
dropped.append(key)
|
|
2503
|
+
warnings.append(f"metadata: stats.{key} dropped: this run's stats helper did not write it")
|
|
2504
|
+
continue
|
|
2505
|
+
if key.startswith(DENOMINATOR_NOTE_PREFIX) and recounted:
|
|
2506
|
+
dropped.append(key)
|
|
2507
|
+
warnings.append(f"metadata: stats.{key} dropped: it explained an effective_denominator this run changed "
|
|
2508
|
+
f"or removed")
|
|
2509
|
+
continue
|
|
2510
|
+
kept[key] = value
|
|
2511
|
+
# an explanation of a denominator this run left as it was still describes it
|
|
2512
|
+
if key not in exempt and not (key.startswith(DENOMINATOR_NOTE_PREFIX) and denominator is not None):
|
|
2513
|
+
warnings.append(_kept_warning(key))
|
|
2514
|
+
return {**new_stats, **kept}, dropped, warnings
|
|
2000
2515
|
|
|
2001
2516
|
|
|
2002
2517
|
def update_metadata(metadata: object, lists: object, stats: object, *, version: str, generation_date: str,
|
|
2003
|
-
source_commit: str | None = None, source_ref: str | None = None
|
|
2004
|
-
|
|
2518
|
+
source_commit: str | None = None, source_ref: str | None = None,
|
|
2519
|
+
skf_version: str | None = None, skf_root: str | Path | None = None,
|
|
2520
|
+
ast_grep_version: str | None = None, qmd_version: str | None = None) -> tuple[dict, dict]:
|
|
2521
|
+
"""(the metadata.json an update leaves, {"dropped_stats": [...], "warnings": [...]}) (see "Metadata" in the
|
|
2522
|
+
module docstring). Raises ValueError."""
|
|
2005
2523
|
if not isinstance(metadata, dict):
|
|
2006
2524
|
raise ValueError("--metadata file must hold a JSON object (the skill's metadata.json)")
|
|
2007
2525
|
if not isinstance(lists, dict) or not all(isinstance(lists.get(k), list) for k in ("exports", "scripts",
|
|
@@ -2033,11 +2551,14 @@ def update_metadata(metadata: object, lists: object, stats: object, *, version:
|
|
|
2033
2551
|
out[kind] = lists[kind]
|
|
2034
2552
|
else:
|
|
2035
2553
|
out.pop(kind, None)
|
|
2554
|
+
out["tool_versions"], warnings = tool_versions_after(
|
|
2555
|
+
metadata.get("tool_versions"), skf_version=skf_version, skf_root=skf_root,
|
|
2556
|
+
ast_grep_version=ast_grep_version, qmd_version=qmd_version)
|
|
2036
2557
|
old_stats = metadata.get("stats") if isinstance(metadata.get("stats"), dict) else {}
|
|
2037
|
-
|
|
2038
|
-
|
|
2558
|
+
out["stats"], dropped, stats_warnings = stats_after(old_stats, stats["stats"], metadata.get("scope_type"))
|
|
2559
|
+
warnings += stats_warnings
|
|
2039
2560
|
out["confidence_distribution"] = stats["confidence_distribution"]
|
|
2040
|
-
return out
|
|
2561
|
+
return out, {"dropped_stats": dropped, "warnings": warnings}
|
|
2041
2562
|
|
|
2042
2563
|
|
|
2043
2564
|
NOT_PUBLIC_NAMED = 10 # the names the not_public warning spells out; the summary lists them all
|
|
@@ -2177,6 +2698,20 @@ def build_records(extraction: dict | None, details: dict | None, files: list | N
|
|
|
2177
2698
|
if patch.get(field) is not None:
|
|
2178
2699
|
seeded[key][field] = patch[field]
|
|
2179
2700
|
fill_if_null(key, patch)
|
|
2701
|
+
diff = extraction.get("entry_point_diff") if isinstance(extraction, dict) else None
|
|
2702
|
+
aliases: dict[tuple, set] = {}
|
|
2703
|
+
rows = [*(diff.get("public") or []), *(diff.get("other_definitions") or [])] if isinstance(diff, dict) else []
|
|
2704
|
+
for item in rows:
|
|
2705
|
+
members = item.get("members") if isinstance(item, dict) and item.get("via") == "namespace" else None
|
|
2706
|
+
for named in [item, *(members if isinstance(members, list) else [])]:
|
|
2707
|
+
if not isinstance(named, dict):
|
|
2708
|
+
continue
|
|
2709
|
+
name, local = named.get("name"), named.get("local")
|
|
2710
|
+
key = (local, _norm_path(named.get("file")))
|
|
2711
|
+
if isinstance(name, str) and name and isinstance(local, str) and local != name and key in seeded:
|
|
2712
|
+
aliases.setdefault(key, set()).add(name) # the public name the runner gives `local` in that file
|
|
2713
|
+
for key, names in aliases.items():
|
|
2714
|
+
seeded[key]["aliases"] = sorted(names)
|
|
2180
2715
|
surface, _problem = public_surface(extraction) # apply warns when a public-api run gives no surface
|
|
2181
2716
|
for key in seeded if surface is not None else ():
|
|
2182
2717
|
mark = surface.public(key[0], key[1], languages.get(key))
|
|
@@ -2217,6 +2752,26 @@ DRIFT_IN_MAP = "a signature from the tree"
|
|
|
2217
2752
|
REACHABILITY = ("public", "internal-unreachable")
|
|
2218
2753
|
VERIFIER_MISSING = "provenance: spot-checks not run: skf-verify-provenance-completeness.py is missing; re-install SKF"
|
|
2219
2754
|
OUTCOMES = ("verified", "moved", "missing", "re-extracted", "rescoped", "unknown")
|
|
2755
|
+
# Why an entry ends `unknown` (its unknown_reason, see "Gap records"), with the words apply's warning gives it.
|
|
2756
|
+
UNKNOWN_REASONS = {
|
|
2757
|
+
"ambiguous": "several map entries of its name",
|
|
2758
|
+
"no-definition": "no line of its file defines it, by the definition-line rules",
|
|
2759
|
+
"several-definitions": "several lines of its file define it, none the recorded one",
|
|
2760
|
+
"unchecked-language": "the definition-line rules cover no language of its file, and no line was answered as "
|
|
2761
|
+
"declaring it",
|
|
2762
|
+
"no-line": "its map entry gives no file or line to check",
|
|
2763
|
+
"spot-check-failed": "the spot-check failed",
|
|
2764
|
+
"verifier-missing": "skf-verify-provenance-completeness.py is missing",
|
|
2765
|
+
"no-source-root": "no source tree to check",
|
|
2766
|
+
"drift-override": "the drift override kept its line from being moved",
|
|
2767
|
+
"cited-elsewhere": "the gap cites another file than the provenance map's entries of its name",
|
|
2768
|
+
}
|
|
2769
|
+
# A cited-elsewhere entry's citation_check that read nothing at the cited file, with the fix that lets a re-run read it.
|
|
2770
|
+
UNREAD = {
|
|
2771
|
+
"no-source-root": "give update-skill the source tree",
|
|
2772
|
+
"verifier-missing": "re-install SKF",
|
|
2773
|
+
"spot-check-failed": "fix what its spot-check failed on, which the `spot-check failed` warning names",
|
|
2774
|
+
}
|
|
2220
2775
|
_DEFINITION_LINES = re.compile(r"definition lines?\s*\(([\d,\s]+)\)", re.IGNORECASE)
|
|
2221
2776
|
_VERIFIER = None
|
|
2222
2777
|
|
|
@@ -2268,40 +2823,48 @@ def _drift_reason(entry: dict) -> str | None:
|
|
|
2268
2823
|
return None
|
|
2269
2824
|
if entry.get("provenance_completeness"):
|
|
2270
2825
|
return DRIFT_COMPLETENESS
|
|
2271
|
-
|
|
2272
|
-
|
|
2826
|
+
match = entry.get("map_match") if isinstance(entry.get("map_match"), dict) else {}
|
|
2827
|
+
status = match.get("status")
|
|
2828
|
+
# a not-found with a candidate that has a file is a citation of another file than the map's entries: held (#824)
|
|
2829
|
+
held = status in ("found", "ambiguous") or status == "not-found" and any(
|
|
2830
|
+
isinstance(c, dict) and _norm_path(c.get("source_file")) for c in match.get("candidates") or [])
|
|
2831
|
+
return DRIFT_IN_MAP if held else DRIFT_NOT_IN_MAP
|
|
2273
2832
|
|
|
2274
2833
|
|
|
2275
2834
|
def _spot_check(root: Path | None, path: str | None, name: str, line: int | None, export_type: object,
|
|
2276
|
-
answers: dict) -> tuple[str, int | None, bool, str | None]:
|
|
2835
|
+
answers: dict) -> tuple[str, int | None, bool, str | None, str | None]:
|
|
2277
2836
|
"""(outcome, the line a `moved` export moves to, whether it waits on a `declaring_line` answer, why the
|
|
2278
|
-
check failed)."""
|
|
2837
|
+
check failed, why it is `unknown`: an UNKNOWN_REASONS key)."""
|
|
2838
|
+
if root is None:
|
|
2839
|
+
return "unknown", None, False, None, "no-source-root"
|
|
2279
2840
|
verifier = _verifier()
|
|
2280
|
-
if verifier is None
|
|
2281
|
-
return "unknown", None, False, None
|
|
2841
|
+
if verifier is None:
|
|
2842
|
+
return "unknown", None, False, None, "verifier-missing"
|
|
2843
|
+
if not path or line is None:
|
|
2844
|
+
return "unknown", None, False, None, "no-line"
|
|
2282
2845
|
try:
|
|
2283
2846
|
report = verifier.definition_lines_report(path, name, root, line,
|
|
2284
2847
|
export_type if isinstance(export_type, str) else None)
|
|
2285
2848
|
except Exception as exc: # noqa: BLE001 - a file it cannot read, or any verifier failure: this entry only
|
|
2286
|
-
return "unknown", None, False, f"{type(exc).__name__}: {exc}"
|
|
2849
|
+
return "unknown", None, False, f"{type(exc).__name__}: {exc}", "spot-check-failed"
|
|
2287
2850
|
check = report.get("line_check")
|
|
2288
2851
|
if check == "file-missing":
|
|
2289
|
-
return "missing", None, False, None
|
|
2852
|
+
return "missing", None, False, None, None
|
|
2290
2853
|
if check == "skipped-export-type":
|
|
2291
|
-
return "verified", None, False, None
|
|
2854
|
+
return "verified", None, False, None, None
|
|
2292
2855
|
if check == "skipped-language":
|
|
2293
2856
|
if "declaring_line" not in answers:
|
|
2294
|
-
return "unknown", None, True, None
|
|
2857
|
+
return "unknown", None, True, None, None
|
|
2295
2858
|
declared = answers["declaring_line"]
|
|
2296
2859
|
if declared is None:
|
|
2297
|
-
return "unknown", None, False, None
|
|
2298
|
-
return ("verified", None, False, None) if declared == line else ("moved", declared, False, None)
|
|
2860
|
+
return "unknown", None, False, None, "unchecked-language"
|
|
2861
|
+
return ("verified", None, False, None, None) if declared == line else ("moved", declared, False, None, None)
|
|
2299
2862
|
if report.get("line_is_definition"):
|
|
2300
|
-
return "verified", None, False, None
|
|
2863
|
+
return "verified", None, False, None, None
|
|
2301
2864
|
defs = report.get("definition_lines") or []
|
|
2302
2865
|
if len(defs) == 1 and defs[0] != line:
|
|
2303
|
-
return "moved", defs[0], False, None
|
|
2304
|
-
return "unknown", None, False, None
|
|
2866
|
+
return "moved", defs[0], False, None, None
|
|
2867
|
+
return "unknown", None, False, None, "several-definitions" if defs else "no-definition"
|
|
2305
2868
|
|
|
2306
2869
|
|
|
2307
2870
|
def _pinned_lines(remediation: object) -> list[int] | None:
|
|
@@ -2311,6 +2874,18 @@ def _pinned_lines(remediation: object) -> list[int] | None:
|
|
|
2311
2874
|
return lines or None
|
|
2312
2875
|
|
|
2313
2876
|
|
|
2877
|
+
def _inside(root: Path | None, path: str | None) -> bool:
|
|
2878
|
+
"""True for a relative path that stays under the source root, links resolved."""
|
|
2879
|
+
rel = Path(path) if path else None
|
|
2880
|
+
if root is None or rel is None or rel.is_absolute() or ".." in rel.parts:
|
|
2881
|
+
return False
|
|
2882
|
+
try:
|
|
2883
|
+
(root / rel).resolve().relative_to(root.resolve())
|
|
2884
|
+
except (OSError, ValueError):
|
|
2885
|
+
return False
|
|
2886
|
+
return True
|
|
2887
|
+
|
|
2888
|
+
|
|
2314
2889
|
def _where(path: object, line: object) -> str | None:
|
|
2315
2890
|
file, number = _norm_path(path), _line_of(line)
|
|
2316
2891
|
return f"{file}:{number}" if file and number is not None else None
|
|
@@ -2330,7 +2905,14 @@ class _GapEntry:
|
|
|
2330
2905
|
self.map_entry = {"source_file": found.get("source_file"), "source_line": found.get("source_line")} \
|
|
2331
2906
|
if found else None
|
|
2332
2907
|
self.map_export_type = found.get("export_type") if found else None
|
|
2908
|
+
# the map entry's own name when skf-parse-gaps matched it by its reexported_as (the gap names the public one)
|
|
2909
|
+
self.declared = _name_of(found) if found and _name_of(found) else self.name
|
|
2333
2910
|
self.in_map = bool(match.get("candidates"))
|
|
2911
|
+
# where map_match's candidates sit, `file:line` (or the file); a placeholder with no file names none
|
|
2912
|
+
self.map_locations = list(dict.fromkeys(
|
|
2913
|
+
_where(c.get("source_file"), c.get("source_line")) or _norm_path(c.get("source_file"))
|
|
2914
|
+
for c in match.get("candidates") or [] if isinstance(c, dict) and _norm_path(c.get("source_file"))))
|
|
2915
|
+
self.citation_check: str | None = None # a cited-elsewhere entry's citation check: missing, or its reason
|
|
2334
2916
|
citation = entry.get("source_citation")
|
|
2335
2917
|
self.citation = citation if isinstance(citation, dict) else None
|
|
2336
2918
|
self.recorded: tuple[str | None, int | None] = (None, None) # the file and line a check read
|
|
@@ -2338,8 +2920,9 @@ class _GapEntry:
|
|
|
2338
2920
|
self.new_line: int | None = None
|
|
2339
2921
|
self.unknown_reason: str | None = None
|
|
2340
2922
|
self.reextract = False
|
|
2341
|
-
self.record: dict | None = None # the §4a record it matched
|
|
2923
|
+
self.record: dict | None = None # the §4a record it matched, or took its signature from
|
|
2342
2924
|
self.record_path: str | None = None
|
|
2925
|
+
self.signature_at: tuple[str, int] | None = None # where a check located it: its signature's record
|
|
2343
2926
|
self.waits = False # on a declaring_line answer
|
|
2344
2927
|
self.failure: str | None = None # why its spot-check failed
|
|
2345
2928
|
|
|
@@ -2353,8 +2936,25 @@ class _GapEntry:
|
|
|
2353
2936
|
|
|
2354
2937
|
def check(self, root: Path | None, path: object, line: object, export_type: object) -> None:
|
|
2355
2938
|
self.recorded = (_norm_path(path), _line_of(line))
|
|
2356
|
-
self.outcome, self.new_line, self.waits, self.failure = _spot_check(
|
|
2357
|
-
root, self.recorded[0], self.
|
|
2939
|
+
self.outcome, self.new_line, self.waits, self.failure, self.unknown_reason = _spot_check(
|
|
2940
|
+
root, self.recorded[0], self.declared, self.recorded[1], export_type, self.answers)
|
|
2941
|
+
|
|
2942
|
+
def locate(self, root: Path | None) -> None:
|
|
2943
|
+
"""Record where a check located the export (the checked file, at the recorded line or the one it moved
|
|
2944
|
+
to), so its signature comes from §4a's record there; never a file outside the source root."""
|
|
2945
|
+
line = self.new_line if self.outcome == "moved" else self.recorded[1]
|
|
2946
|
+
if self.outcome in ("verified", "moved") and line is not None and _inside(root, self.recorded[0]):
|
|
2947
|
+
self.signature_at = (self.recorded[0], line)
|
|
2948
|
+
|
|
2949
|
+
def take_signature(self, blocks: dict[str, list[dict]]) -> None:
|
|
2950
|
+
"""Take the one record of the entry's name in the block of the file it was located in that is at its line,
|
|
2951
|
+
else the one with no line; none otherwise."""
|
|
2952
|
+
path, line = self.signature_at
|
|
2953
|
+
named = [r for r in blocks.get(path, []) if _name_of(r) in (self.name, self.declared)]
|
|
2954
|
+
at = [r for r in named if _line_of(r.get("location")) == line] \
|
|
2955
|
+
or [r for r in named if _line_of(r.get("location")) is None]
|
|
2956
|
+
if len(at) == 1:
|
|
2957
|
+
self.record, self.record_path = at[0], path
|
|
2358
2958
|
|
|
2359
2959
|
def route(self, root: Path | None, drift: bool) -> None:
|
|
2360
2960
|
"""Spot-check the entry, or send it to targeted re-extraction (see "Gap records" in the module docstring)."""
|
|
@@ -2373,14 +2973,23 @@ class _GapEntry:
|
|
|
2373
2973
|
return
|
|
2374
2974
|
if self.match_status == "found":
|
|
2375
2975
|
self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
|
|
2976
|
+
self.locate(root)
|
|
2376
2977
|
return
|
|
2377
|
-
if self.match_status == "ambiguous":
|
|
2378
|
-
self.outcome = "unknown"
|
|
2978
|
+
if self.match_status == "ambiguous": # several map entries of its name: no check, apply warns
|
|
2979
|
+
self.outcome, self.unknown_reason = "unknown", "ambiguous"
|
|
2379
2980
|
return
|
|
2380
2981
|
if self.citation:
|
|
2381
2982
|
self.check(root, self.citation.get("file"), self.citation.get("line"), None)
|
|
2382
2983
|
if self.waits or self.outcome in ("verified", "moved"):
|
|
2984
|
+
if not self.in_map: # a citation of another file than the map's entries of its name takes none
|
|
2985
|
+
self.locate(root)
|
|
2383
2986
|
return
|
|
2987
|
+
if self.match_status == "not-found" and self.map_locations:
|
|
2988
|
+
# the map holds its name at other files: the cited file's declaration may be another package's, so it is
|
|
2989
|
+
# never re-extracted there (#824); the record keeps the check's cause, and apply warns with what to fix
|
|
2990
|
+
self.citation_check = "missing" if self.outcome == "missing" else self.unknown_reason
|
|
2991
|
+
self.outcome, self.unknown_reason = "unknown", "cited-elsewhere"
|
|
2992
|
+
return
|
|
2384
2993
|
if self.entry.get("provenance_completeness") and root is not None:
|
|
2385
2994
|
self.reextract = True
|
|
2386
2995
|
elif self.blocking:
|
|
@@ -2391,14 +3000,20 @@ class _GapEntry:
|
|
|
2391
3000
|
self.outcome = "unknown"
|
|
2392
3001
|
|
|
2393
3002
|
def match(self, blocks: dict[str, list[dict]]) -> tuple[int, int]:
|
|
2394
|
-
"""Take the first export of the entry's name in the blocks of its resolved files
|
|
2395
|
-
|
|
3003
|
+
"""Take the first export of the entry's name in the blocks of its resolved files, else the first whose
|
|
3004
|
+
`aliases` hold its name (the public name the runner gives its local name in that file); (files scanned,
|
|
3005
|
+
exports found in them)."""
|
|
2396
3006
|
exports = 0
|
|
3007
|
+
aliased: tuple[dict, str] | None = None
|
|
2397
3008
|
for path in self.paths:
|
|
2398
3009
|
for record in blocks.get(path, []):
|
|
2399
3010
|
exports += 1
|
|
2400
3011
|
if self.record is None and _name_of(record) == self.name:
|
|
2401
3012
|
self.record, self.record_path = record, path
|
|
3013
|
+
elif aliased is None and isinstance(record.get("aliases"), list) and self.name in record["aliases"]:
|
|
3014
|
+
aliased = (record, path)
|
|
3015
|
+
if self.record is None and aliased is not None:
|
|
3016
|
+
self.record, self.record_path = aliased
|
|
2402
3017
|
if self.record is not None:
|
|
2403
3018
|
self.outcome = "re-extracted"
|
|
2404
3019
|
return len(self.paths), exports
|
|
@@ -2427,8 +3042,14 @@ class _GapEntry:
|
|
|
2427
3042
|
return (self.record or {}).get("location")
|
|
2428
3043
|
return None
|
|
2429
3044
|
|
|
3045
|
+
def signature(self) -> dict | None:
|
|
3046
|
+
"""The record a located export takes its signature from, or None."""
|
|
3047
|
+
return self.record if self.signature_at and not self.internal else None
|
|
3048
|
+
|
|
2430
3049
|
def verification(self) -> dict:
|
|
3050
|
+
signed = self.signature() or {}
|
|
2431
3051
|
recorded = _where(*self.recorded)
|
|
3052
|
+
elsewhere = self.outcome == "unknown" and self.unknown_reason == "cited-elsewhere"
|
|
2432
3053
|
if self.outcome == "re-extracted":
|
|
2433
3054
|
citation = self.new_location()
|
|
2434
3055
|
elif self.outcome in ("verified", "moved", "missing", "rescoped") and recorded:
|
|
@@ -2445,13 +3066,19 @@ class _GapEntry:
|
|
|
2445
3066
|
"provenance_citation": citation,
|
|
2446
3067
|
"source_citation": self.citation,
|
|
2447
3068
|
"new_location": self.new_location(),
|
|
2448
|
-
"unknown_reason": self.unknown_reason,
|
|
3069
|
+
"unknown_reason": self.unknown_reason if self.outcome == "unknown" else None,
|
|
3070
|
+
"citation_check": self.citation_check if elsewhere else None,
|
|
3071
|
+
"map_locations": self.map_locations if elsewhere else None,
|
|
2449
3072
|
"pinned_definition_lines": _pinned_lines(self.entry.get("remediation"))
|
|
2450
3073
|
if self.category == "MOVED_EXPORT" else None,
|
|
2451
3074
|
"resolution_source": "remediation-paths" if self.outcome == "re-extracted" else None,
|
|
2452
3075
|
"reachability": self.answers.get("reachability") if self.category == "NEW_EXPORT" and self.located
|
|
2453
3076
|
else None,
|
|
2454
|
-
"export_type":
|
|
3077
|
+
"export_type": signed.get("type", signed.get("export_type")),
|
|
3078
|
+
"params": signed.get("params", signed.get("parameters")),
|
|
3079
|
+
"return_type": signed.get("return_type"),
|
|
3080
|
+
"extraction_method": signed.get("extraction_method"),
|
|
3081
|
+
"ast_node_type": signed.get("ast_node_type"),
|
|
2455
3082
|
}
|
|
2456
3083
|
|
|
2457
3084
|
def label(self, provenance: list[dict]) -> str:
|
|
@@ -2464,7 +3091,7 @@ class _GapEntry:
|
|
|
2464
3091
|
elif self.map_entry is not None:
|
|
2465
3092
|
file, line = _norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"])
|
|
2466
3093
|
for entry in provenance:
|
|
2467
|
-
if (_name_of(entry) == self.
|
|
3094
|
+
if (_name_of(entry) == self.declared and _file_of(entry) == file
|
|
2468
3095
|
and _line_of(entry.get("source_line")) == line):
|
|
2469
3096
|
method = entry.get("extraction_method")
|
|
2470
3097
|
break
|
|
@@ -2513,17 +3140,22 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
|
|
|
2513
3140
|
if waiting:
|
|
2514
3141
|
return None, {"status": "needs-judgment", "needs_judgment": waiting, "warnings": warnings}
|
|
2515
3142
|
to_scan = [g for g in routed if g.reextract]
|
|
2516
|
-
|
|
3143
|
+
located = [g for g in routed if g.signature_at and not g.internal] # an internal export takes no signature
|
|
3144
|
+
files = list(dict.fromkeys([p for g in to_scan for p in g.paths] + [g.signature_at[0] for g in located]))
|
|
2517
3145
|
if plan:
|
|
2518
3146
|
return None, {"status": "planned", "reextract": {
|
|
2519
3147
|
"entries": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
|
|
2520
|
-
"resolved_paths": g.paths} for g in to_scan], "files": files
|
|
3148
|
+
"resolved_paths": g.paths} for g in to_scan], "files": files,
|
|
3149
|
+
"signatures": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "file": g.signature_at[0],
|
|
3150
|
+
"line": g.signature_at[1]} for g in located]}, "warnings": warnings}
|
|
2521
3151
|
blocks: dict[str, list[dict]] = {}
|
|
2522
3152
|
scanned = 0
|
|
3153
|
+
targeted_files = {p for g in to_scan for p in g.paths}
|
|
3154
|
+
signature_files = {g.signature_at[0] for g in located}
|
|
2523
3155
|
if files:
|
|
2524
3156
|
if remediation is None:
|
|
2525
|
-
raise ValueError("
|
|
2526
|
-
"of `records` over them")
|
|
3157
|
+
raise ValueError("gap-records has files to match (targeted re-extraction, or a located export's "
|
|
3158
|
+
"signature): pass --remediation-records, the output of `records` over them")
|
|
2527
3159
|
if not isinstance(remediation, dict) or not isinstance(remediation.get("files"), list):
|
|
2528
3160
|
raise ValueError("--remediation-records has no `files` array: is it the output of `records`?")
|
|
2529
3161
|
for block in remediation["files"]:
|
|
@@ -2545,21 +3177,36 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
|
|
|
2545
3177
|
"files_scanned": files_scanned, "exports_found_in_scan": found})
|
|
2546
3178
|
if unresolved:
|
|
2547
3179
|
return None, {"status": "unresolved", "unresolved": unresolved, "warnings": warnings}
|
|
3180
|
+
for g in located:
|
|
3181
|
+
g.take_signature(blocks)
|
|
2548
3182
|
needs = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": g.needs(),
|
|
2549
3183
|
"file": _location(g.new_location())[0] if g.outcome != "verified" else g.recorded[0],
|
|
2550
3184
|
"line": _location(g.new_location())[1] if g.outcome != "verified" else g.recorded[1]}
|
|
2551
3185
|
for g in routed if g.needs()]
|
|
2552
3186
|
if needs:
|
|
2553
3187
|
return None, {"status": "needs-judgment", "needs_judgment": needs, "warnings": warnings}
|
|
3188
|
+
warnings += [f"provenance: {g.name}: located at {g.signature_at[0]}:{g.signature_at[1]}, but no record of it "
|
|
3189
|
+
f"there; signature not repaired" for g in located if g.record is None]
|
|
2554
3190
|
kept: dict[str, list[dict]] = {}
|
|
2555
3191
|
for g in routed:
|
|
2556
|
-
if g.record is
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
3192
|
+
if g.record is None or g.internal:
|
|
3193
|
+
continue
|
|
3194
|
+
record = dict(g.record)
|
|
3195
|
+
record.pop("public", None) # a repair documents and maps every record: the mark is a normal update's
|
|
3196
|
+
record.pop("aliases", None) # the match is made: the copy names the one it took
|
|
3197
|
+
if _name_of(record) != g.name: # matched by its public name: documented under it, its own name as `local`
|
|
3198
|
+
record = {"name": g.name, "local": _name_of(record),
|
|
3199
|
+
**{k: v for k, v in record.items() if k not in ("name", "export_name", "local")}}
|
|
3200
|
+
if g.outcome == "re-extracted" and g.answers.get("docstring") is not None:
|
|
3201
|
+
record["docstring"] = g.answers["docstring"] # a located export's signature record asks none
|
|
3202
|
+
block = kept.setdefault(g.record_path, [])
|
|
3203
|
+
location = record.get("location")
|
|
3204
|
+
same = next((r for r in block if (_name_of(r) == _name_of(record) and r.get("location") == location
|
|
3205
|
+
if location is not None else r == record)), None)
|
|
3206
|
+
if same is None:
|
|
3207
|
+
block.append(record)
|
|
3208
|
+
elif same.get("docstring") is None and record.get("docstring") is not None:
|
|
3209
|
+
same["docstring"] = record["docstring"]
|
|
2563
3210
|
breakdown = {"T1": 0, "T1-low": 0, "unlabeled": 0, "T2": 0}
|
|
2564
3211
|
provenance_entries = _entries_of(provenance)
|
|
2565
3212
|
for g in routed:
|
|
@@ -2570,13 +3217,15 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
|
|
|
2570
3217
|
"files_extracted": scanned,
|
|
2571
3218
|
"exports_extracted": len(entries),
|
|
2572
3219
|
"confidence_breakdown": breakdown,
|
|
2573
|
-
"verification": [g.verification() for g in routed],
|
|
3220
|
+
"verification": [{"gap_id": g.entry.get("gap_id"), **g.verification()} for g in routed],
|
|
2574
3221
|
"files": [{"file_path": path, "exports": exports} for path, exports in kept.items()],
|
|
2575
3222
|
}
|
|
2576
3223
|
matched = [g for g in to_scan if g.record is not None]
|
|
2577
|
-
|
|
3224
|
+
scanned_paths = set(blocks)
|
|
3225
|
+
targeted = {"resolved_count": len(matched), "files_scanned": len(scanned_paths & targeted_files),
|
|
3226
|
+
"signature_files_scanned": len(scanned_paths & signature_files),
|
|
2578
3227
|
"exports_matched": len({(g.record_path, _name_of(g.record)) for g in matched}),
|
|
2579
|
-
"tier": tier} if to_scan else None
|
|
3228
|
+
"tier": tier} if to_scan or located else None
|
|
2580
3229
|
summary = {
|
|
2581
3230
|
"status": "written",
|
|
2582
3231
|
"gap_count": len(entries),
|
|
@@ -2766,13 +3415,62 @@ def _metadata_in(path: str | None) -> dict | None:
|
|
|
2766
3415
|
return metadata
|
|
2767
3416
|
|
|
2768
3417
|
|
|
3418
|
+
_QUICK_METADATA = None
|
|
3419
|
+
|
|
3420
|
+
|
|
3421
|
+
def _quick_metadata():
|
|
3422
|
+
"""skf-render-quick-metadata.py, loaded once from this folder as skf-extraction-inventory.py loads it: its
|
|
3423
|
+
probe_skf_version reads the installed SKF's version, so create-skill and update-skill read it one way. Raises
|
|
3424
|
+
ValueError, one line, when it cannot be loaded."""
|
|
3425
|
+
global _QUICK_METADATA
|
|
3426
|
+
if _QUICK_METADATA is None:
|
|
3427
|
+
path = Path(__file__).resolve().parent / "skf-render-quick-metadata.py"
|
|
3428
|
+
cannot = f"cannot load {path.name} beside {Path(__file__).name}"
|
|
3429
|
+
if not path.is_file():
|
|
3430
|
+
raise ValueError(f"{cannot}: it is missing; re-install SKF")
|
|
3431
|
+
spec = importlib.util.spec_from_file_location("skf_render_quick_metadata", path)
|
|
3432
|
+
if spec is None or spec.loader is None:
|
|
3433
|
+
raise ValueError(f"{cannot}: re-install SKF")
|
|
3434
|
+
module = importlib.util.module_from_spec(spec)
|
|
3435
|
+
try:
|
|
3436
|
+
spec.loader.exec_module(module)
|
|
3437
|
+
except (ImportError, SyntaxError, OSError) as exc:
|
|
3438
|
+
raise ValueError(f"{cannot} ({type(exc).__name__}: {' '.join(str(exc).split())}): re-install "
|
|
3439
|
+
f"SKF") from exc
|
|
3440
|
+
if not callable(getattr(module, "probe_skf_version", None)):
|
|
3441
|
+
raise ValueError(f"{cannot}: it has no probe_skf_version; re-install SKF")
|
|
3442
|
+
_QUICK_METADATA = module
|
|
3443
|
+
return _QUICK_METADATA
|
|
3444
|
+
|
|
3445
|
+
|
|
3446
|
+
def _runner_ast_grep(path: str | None) -> str | None:
|
|
3447
|
+
"""The ast-grep version the recipe runner's output (--extraction) records: "unknown" when it records none but
|
|
3448
|
+
holds an export ast-grep matched, and None when ast-grep did not run (no export it matched, as in the
|
|
3449
|
+
{"exports": []} step 2 writes when the runner cannot run)."""
|
|
3450
|
+
if not path:
|
|
3451
|
+
return None
|
|
3452
|
+
extraction = _load_json_file(Path(path), "--extraction file")
|
|
3453
|
+
if not isinstance(extraction, dict):
|
|
3454
|
+
raise ValueError("--extraction file must hold a JSON object (the recipe runner's output)")
|
|
3455
|
+
ast_grep = extraction.get("ast_grep")
|
|
3456
|
+
version = _tool_version(ast_grep.get("version")) if isinstance(ast_grep, dict) else None
|
|
3457
|
+
if version is not None:
|
|
3458
|
+
return version
|
|
3459
|
+
exports = extraction.get("exports") if isinstance(extraction.get("exports"), list) else []
|
|
3460
|
+
matched = any(isinstance(e, dict) and e.get("extraction_method") == "ast-grep" for e in exports)
|
|
3461
|
+
return "unknown" if matched else None
|
|
3462
|
+
|
|
3463
|
+
|
|
2769
3464
|
def _cmd_metadata(args: argparse.Namespace) -> int:
|
|
2770
3465
|
target = Path(args.metadata)
|
|
2771
3466
|
try:
|
|
2772
|
-
|
|
3467
|
+
skf_version = _quick_metadata().probe_skf_version(args.skf_root) if args.skf_root else None
|
|
3468
|
+
metadata, summary = update_metadata(
|
|
2773
3469
|
_load_json_file(target, "--metadata file"), _load_json_file(Path(args.lists), "--lists file"),
|
|
2774
3470
|
_load_json_file(Path(args.stats), "--stats file"), version=args.version,
|
|
2775
|
-
generation_date=args.generation_date, source_commit=args.source_commit, source_ref=args.source_ref
|
|
3471
|
+
generation_date=args.generation_date, source_commit=args.source_commit, source_ref=args.source_ref,
|
|
3472
|
+
skf_version=skf_version, skf_root=args.skf_root, ast_grep_version=_runner_ast_grep(args.extraction),
|
|
3473
|
+
qmd_version=args.qmd_version)
|
|
2776
3474
|
_write_json_atomic(target, metadata)
|
|
2777
3475
|
except ValueError as exc:
|
|
2778
3476
|
print(f"error: {exc}", file=sys.stderr)
|
|
@@ -2782,7 +3480,8 @@ def _cmd_metadata(args: argparse.Namespace) -> int:
|
|
|
2782
3480
|
return 1
|
|
2783
3481
|
print(json.dumps({"status": "written", "metadata": target.as_posix(), "version": metadata["version"],
|
|
2784
3482
|
"exports": len(metadata["exports"]), "scripts": len(metadata.get("scripts") or []),
|
|
2785
|
-
"assets": len(metadata.get("assets") or [])
|
|
3483
|
+
"assets": len(metadata.get("assets") or []), "tool_versions": metadata["tool_versions"],
|
|
3484
|
+
**summary}))
|
|
2786
3485
|
return 0
|
|
2787
3486
|
|
|
2788
3487
|
|
|
@@ -3061,9 +3760,15 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
3061
3760
|
description="Rewrite the skill's metadata.json in place. It sets version and generation_date, "
|
|
3062
3761
|
"source_commit and source_ref when given, exports, scripts and assets from --lists (a script "
|
|
3063
3762
|
"or asset list left out when empty), confidence_distribution from --stats and the stats "
|
|
3064
|
-
"helper's stats fields, dropping one an earlier run wrote that --stats lacks;
|
|
3065
|
-
"
|
|
3066
|
-
"
|
|
3763
|
+
"helper's stats fields, dropping one an earlier run wrote that --stats lacks; tool_versions.skf "
|
|
3764
|
+
"from --skf-root (unknown, with a warning, when it gives no version), and tool_versions.ast_grep "
|
|
3765
|
+
"and tool_versions.qmd from --extraction and --qmd-version when this run used them, else as "
|
|
3766
|
+
"they stand. When --stats' effective_denominator differs from the old one (or one is absent), it "
|
|
3767
|
+
"drops each old stats key named effective_denominator_*; each dropped key is in dropped_stats, "
|
|
3768
|
+
"with a warning. It keeps every other old stats key, with a warning for each one create-skill "
|
|
3769
|
+
"does not write for the skill's scope_type. It removes last_update and "
|
|
3770
|
+
"update_type and keeps every other field, a patched one included. It refuses a --stats whose "
|
|
3771
|
+
"scripts_count or assets_count is not the length of --lists' scripts or assets.")
|
|
3067
3772
|
p_meta.add_argument("--metadata", required=True, metavar="FILE",
|
|
3068
3773
|
help="the skill's metadata.json, rewritten in place")
|
|
3069
3774
|
p_meta.add_argument("--lists", required=True, metavar="FILE", help="the lists apply wrote (--lists-out)")
|
|
@@ -3073,6 +3778,14 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
3073
3778
|
p_meta.add_argument("--generation-date", required=True, help="the clock's UTC time, YYYY-MM-DDTHH:MM:SSZ")
|
|
3074
3779
|
p_meta.add_argument("--source-commit", default=None, help="the commit this update read (an empty one: null)")
|
|
3075
3780
|
p_meta.add_argument("--source-ref", default=None, help="the ref this update read (an empty one: null)")
|
|
3781
|
+
p_meta.add_argument("--skf-root", metavar="DIR",
|
|
3782
|
+
help="the installed SKF ({project-root}/_bmad/skf): tool_versions.skf, from its package.json "
|
|
3783
|
+
"or VERSION file")
|
|
3784
|
+
p_meta.add_argument("--extraction", metavar="FILE",
|
|
3785
|
+
help="the recipe runner's output this run wrote: tool_versions.ast_grep, its ast_grep.version")
|
|
3786
|
+
p_meta.add_argument("--qmd-version", default=None,
|
|
3787
|
+
help="what qmd --version printed, when this run queried QMD: tool_versions.qmd (a leading "
|
|
3788
|
+
"\"qmd \" removed)")
|
|
3076
3789
|
p_meta.set_defaults(func=_cmd_metadata)
|
|
3077
3790
|
|
|
3078
3791
|
p_records = sub.add_parser("records", help="write step 3's re-extraction records from the runner's and the "
|
|
@@ -3094,9 +3807,9 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
3094
3807
|
help="the workspace drift check's status; overridden runs the drift gate")
|
|
3095
3808
|
p_gap.add_argument("--judgments", metavar="FILE", help="the answers to its needs_judgment[], keyed by gap id")
|
|
3096
3809
|
p_gap.add_argument("--remediation-records", metavar="FILE",
|
|
3097
|
-
help="`records` output over the files
|
|
3810
|
+
help="`records` output over the files the plan listed")
|
|
3098
3811
|
p_gap.add_argument("--plan", action="store_true",
|
|
3099
|
-
help="print the answers it needs and the files
|
|
3812
|
+
help="print the answers it needs and the files §4a's extraction scans; write no records")
|
|
3100
3813
|
p_gap.add_argument("--files-out", metavar="FILE", help="with --plan: write those files here, a JSON array")
|
|
3101
3814
|
p_gap.add_argument("--evidence", metavar="FILE",
|
|
3102
3815
|
help="append the targeted_reextraction record to this JSON-lines file when it ran")
|