bmad-module-skill-forge 4.0.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +3 -3
  3. package/docs/_data/pinned.yaml +19 -13
  4. package/docs/architecture.md +2 -2
  5. package/docs/concepts.md +4 -4
  6. package/docs/examples.md +32 -25
  7. package/docs/getting-started.md +3 -3
  8. package/docs/how-it-works.md +2 -2
  9. package/docs/index.md +1 -1
  10. package/docs/skill-model.md +41 -39
  11. package/docs/verifying-a-skill.md +41 -20
  12. package/docs/why-skf.md +1 -1
  13. package/docs/workflows.md +2 -2
  14. package/package.json +1 -1
  15. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +11 -3
  16. package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
  17. package/src/shared/scripts/skf-build-change-manifest.py +881 -168
  18. package/src/shared/scripts/skf-count-tokens.py +54 -10
  19. package/src/shared/scripts/skf-detect-workspaces.py +67 -0
  20. package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
  21. package/src/shared/scripts/skf-extract-public-api.py +439 -107
  22. package/src/shared/scripts/skf-extraction-inventory.py +10 -6
  23. package/src/shared/scripts/skf-extraction-snapshot.py +6 -6
  24. package/src/shared/scripts/skf-parse-gaps.py +135 -26
  25. package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
  26. package/src/shared/scripts/skf-resolve-customization.py +197 -0
  27. package/src/shared/scripts/skf-shard-body.py +239 -41
  28. package/src/shared/scripts/skf-validate-output.py +12 -3
  29. package/src/shared/scripts/skf-verify-provenance-completeness.py +470 -38
  30. package/src/shared/tool-requirements.yaml +8 -2
  31. package/src/skf-analyze-source/SKILL.md +2 -2
  32. package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
  33. package/src/skf-audit-skill/SKILL.md +2 -2
  34. package/src/skf-audit-skill/references/re-index.md +2 -2
  35. package/src/skf-brief-skill/SKILL.md +2 -2
  36. package/src/skf-brief-skill/assets/skill-brief-schema.md +2 -2
  37. package/src/skf-brief-skill/references/scope-definition.md +4 -4
  38. package/src/skf-campaign/SKILL.md +2 -2
  39. package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
  40. package/src/skf-create-skill/SKILL.md +2 -2
  41. package/src/skf-create-skill/assets/skill-sections.md +1 -1
  42. package/src/skf-create-skill/references/extract.md +2 -1
  43. package/src/skf-create-skill/references/extraction-patterns.md +3 -3
  44. package/src/skf-create-skill/references/report.md +6 -1
  45. package/src/skf-create-stack-skill/SKILL.md +2 -2
  46. package/src/skf-create-stack-skill/references/validate.md +1 -1
  47. package/src/skf-drop-skill/SKILL.md +2 -2
  48. package/src/skf-export-skill/SKILL.md +2 -2
  49. package/src/skf-export-skill/assets/snippet-format.md +5 -5
  50. package/src/skf-export-skill/references/generate-snippet.md +22 -15
  51. package/src/skf-export-skill/references/load-skill.md +1 -1
  52. package/src/skf-export-skill/references/token-report.md +29 -9
  53. package/src/skf-forger/SKILL.md +2 -2
  54. package/src/skf-quick-skill/SKILL.md +2 -2
  55. package/src/skf-refine-architecture/SKILL.md +6 -8
  56. package/src/skf-rename-skill/SKILL.md +2 -2
  57. package/src/skf-setup/SKILL.md +2 -2
  58. package/src/skf-test-skill/SKILL.md +2 -2
  59. package/src/skf-test-skill/assets/output-section-formats.md +6 -4
  60. package/src/skf-test-skill/references/coherence-check.md +4 -3
  61. package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
  62. package/src/skf-test-skill/references/coverage-check-tiers.md +1 -1
  63. package/src/skf-test-skill/references/coverage-check.md +45 -69
  64. package/src/skf-test-skill/references/init.md +2 -1
  65. package/src/skf-test-skill/references/migration-section-rules.md +26 -14
  66. package/src/skf-test-skill/references/report.md +2 -2
  67. package/src/skf-test-skill/references/scoring-rules.md +5 -4
  68. package/src/skf-test-skill/references/source-access-protocol.md +6 -6
  69. package/src/skf-test-skill/scripts/gap-ledger.py +378 -163
  70. package/src/skf-test-skill/scripts/load-coverage-inputs.py +171 -70
  71. package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
  72. package/src/skf-test-skill/scripts/reconcile-coverage.py +192 -39
  73. package/src/skf-test-skill/scripts/score-signatures.py +329 -54
  74. package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
  75. package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
  76. package/src/skf-test-skill/templates/test-report-template.md +1 -0
  77. package/src/skf-update-skill/SKILL.md +2 -2
  78. package/src/skf-update-skill/references/detect-changes.md +1 -1
  79. package/src/skf-update-skill/references/gap-driven.md +36 -21
  80. package/src/skf-update-skill/references/merge.md +15 -10
  81. package/src/skf-update-skill/references/re-extract.md +3 -1
  82. package/src/skf-update-skill/references/report.md +5 -4
  83. package/src/skf-update-skill/references/write.md +39 -18
  84. package/src/skf-verify-stack/SKILL.md +2 -2
  85. package/tools/cli/lib/tool-check.js +158 -2
@@ -64,11 +64,13 @@ Subcommands:
64
64
 
65
65
  metadata --metadata <metadata.json> --lists <file> --stats <file>
66
66
  --version <version> --generation-date <iso>
67
- [--source-commit <sha>] [--source-ref <ref>]
67
+ [--source-commit <sha>] [--source-ref <ref>] [--skf-root <dir>]
68
+ [--extraction <file>] [--qmd-version <version>]
68
69
  The skill's metadata.json an update leaves (see "Metadata" below),
69
70
  written over --metadata through a temporary file and a rename.
70
71
  Prints {"status": "written", "metadata": <path>, "version",
71
- "exports": N, "scripts": N, "assets": N}.
72
+ "exports": N, "scripts": N, "assets": N, "tool_versions": {...},
73
+ "dropped_stats": [...], "warnings": [...]}.
72
74
 
73
75
  records [--extraction <file>] [--export-details <file>]
74
76
  [--files-from <file>] [--patches <dir>] -o <file>
@@ -84,8 +86,9 @@ Subcommands:
84
86
  (--plan [--files-out <file>] | -o <file>)
85
87
  A gap-driven repair's spot-checks, routing and verification records
86
88
  (see "Gap records" below): --plan prints what to answer and which
87
- files targeted re-extraction scans; without it, the records are
88
- written to -o through a temporary file and a rename.
89
+ files §4a's extraction scans (targeted re-extraction, and the
90
+ signatures of the exports a spot-check located); without it, the
91
+ records are written to -o through a temporary file and a rename.
89
92
 
90
93
  baseline-gaps --provenance-map <file> --extraction <file> --files <file>
91
94
  --source-root <dir> -o <file>
@@ -241,14 +244,32 @@ Apply (update-skill write.md §3):
241
244
 
242
245
  Reads the provenance map the update started from (--provenance-map;
243
246
  without it, a degraded run's new map) and prints nothing but a summary:
244
- the new map goes to -o. Every entry the run leaves alone keeps its exact
245
- value; every entry it adds or rewrites carries signature_source, set
246
- from the tool that produced the signature: ast-grep gives T1 (with
247
- confidence T1 and the recipe's ast_node_type), a signature read by eye
248
- (source-read) gives T1-low (confidence T1-low, ast_node_type null).
249
- A rewritten entry keeps any key the update does not set (notes,
250
- deprecated). New entries take source_library from the map's entries,
251
- else --skill-name.
247
+ the new map goes to -o. Every entry it adds or rewrites carries
248
+ signature_source, set from the tool that produced the signature:
249
+ ast-grep gives T1 (with confidence T1 and the recipe's ast_node_type),
250
+ a signature read by eye (source-read) gives T1-low (confidence T1-low,
251
+ ast_node_type null). Every entry the run leaves alone keeps its exact
252
+ value, but for a signature_source it lacks or holds as a value other
253
+ than T1, T1-low, T2 or T3 (compared case-insensitively), as a map
254
+ written before the key existed does: once the mode's changes are made,
255
+ each such entry takes its confidence when that names T1-low, T2 or T3,
256
+ else T1 for an extraction_method of ast-grep or ast_bridge (compared
257
+ stripped and case-insensitively) and T1-low for any other (T1 needs an
258
+ ast-grep match). The summary's signature_source_backfilled counts these
259
+ fills by value ({"T1": 1, "T1-low": 2}, {} when none), with one warning
260
+ that lists each unrecognized value replaced (name: old -> new, the
261
+ first 10, then "and N more"); their names are not in its `updated`. A
262
+ rewritten entry keeps any key the update does not set (notes,
263
+ deprecated). A new entry's source_library is the map's library when its
264
+ entries name one (--skill-name when they name none). When they name
265
+ several, it is the library the entries of its file settle, else the one
266
+ the entries under the nearest folder above the file that holds entries
267
+ settle: the one library they name, or of several, the one that is a
268
+ path prefix of all the others (storybook beside storybook/test). The
269
+ record of a MOVED file, or of an export whose change names an old_file,
270
+ tries its new path, then its old. When neither settles it, or the entry
271
+ has no file, it is the map's first library, and one warning counts and
272
+ names those entries (the first 10, then "and N more").
252
273
 
253
274
  incremental (normal mode) reads --manifest (build's output), the renamed
254
275
  exports of --category-c and --ccc-pairs (or of --input, the category
@@ -274,9 +295,10 @@ Apply (update-skill write.md §3):
274
295
  MOVED_EXPORT whose fresh record is the runner's (ast-grep) also takes
275
296
  its confidence, extraction_method and ast_node_type (the entry's when
276
297
  the record's is null), and its signature_source when the record holds
277
- each of params and return_type the entry holds (else the entry's),
278
- never its export_type, params or return_type; one read by eye moves
279
- the line and file only.
298
+ each of params and return_type the entry holds (else the entry's, or
299
+ T1-low when the entry has none it recognizes: the signature it keeps
300
+ was not the runner's), never its export_type, params or return_type;
301
+ one read by eye moves the line and file only.
280
302
 
281
303
  Public-api skills (incremental only): a NEW_EXPORT or MODIFIED_EXPORT
282
304
  the map does not hold, an ADDED file's record and a MOVED file's record
@@ -299,31 +321,91 @@ Apply (update-skill write.md §3):
299
321
  gap-driven reads --reextract-records ({"mode": "gap-driven",
300
322
  "verification": [...], "files": [...]}) and --merge-records ({"exports":
301
323
  [{export_name, export_type, params, return_type}]}, merge's output for
302
- an export no fresh record holds). Each verification record names
324
+ an export no record covers). Each verification record names gap_id,
303
325
  export_name, gap_category, severity, verification, in_map (the map holds
304
326
  an entry of that name), map_entry ({source_file, source_line} of the one
305
327
  entry the spot-check took, else null), source_citation, new_location,
306
- unknown_reason, pinned_definition_lines and reachability:
307
- verified (map_entry) unchanged
308
- moved (map_entry) source_line (and source_file) from new_location
328
+ unknown_reason, citation_check, map_locations (see "Gap records"),
329
+ pinned_definition_lines, reachability, and the located
330
+ export's export_type, params, return_type, extraction_method and
331
+ ast_node_type (from §4a's record at its line, see "Gap records"; null
332
+ when none holds it). It finds an export's entry by its name, else by the
333
+ entry's reexported_as (a renamed re-export the map holds under its
334
+ declared name, which the gap names by its public one):
335
+ verified (map_entry) unchanged, but a NEW_EXPORT's or
336
+ MODIFIED_EXPORT's signature (below) when its
337
+ record was found
338
+ moved (map_entry) source_line (and source_file) from new_location,
339
+ and a NEW_EXPORT's or MODIFIED_EXPORT's
340
+ signature (below) when its record was found
309
341
  re-extracted the entry from its `files` record (gap-driven.md §4a);
310
- none when its reachability is internal-unreachable
342
+ none when its reachability is internal-unreachable;
343
+ a record with a `local` name (matched by the
344
+ public name a re-export gives it) writes
345
+ export_name the declared `local` name, at its
346
+ line, and reexported_as the public one
311
347
  verified/moved, not in_map, a NEW_EXPORT or MODIFIED_EXPORT whose
312
348
  reachability is not internal-unreachable
313
349
  one source-read entry at the citation's line
314
350
  (verified) or new_location (moved), at most one
315
- per name and file, export_type and the
316
- signature from --merge-records
351
+ per name and file, with the signature (below)
352
+ and the source-read labels whatever its record's
353
+ tool
317
354
  unknown, not in_map, a NEW_EXPORT or MODIFIED_EXPORT, a Medium, Low or
318
355
  Info severity an entry from --merge-records with no
319
356
  source_file or source_line
320
357
  unknown, not in_map, any other severity (a missing one included)
321
358
  refused: exit 3, nothing written
322
359
  missing, unknown in_map, unknown on a MOVED_EXPORT
323
- unchanged, with a warning for a person
360
+ unchanged, with a warning for a person that
361
+ names an unknown's unknown_reason and its
362
+ words (UNKNOWN_REASONS); for a NEW_EXPORT or
363
+ MODIFIED_EXPORT whose severity is not Medium,
364
+ Low or Info (a missing one included), the
365
+ warning also names its gap_id and the one
366
+ fix a re-run acts on: for an ambiguous one,
367
+ fix its entries by hand, or give the gap a
368
+ `file:line` Source at its definition when
369
+ that file holds only one of them (the
370
+ re-run's map_match then finds that one); for
371
+ any other, fix the entry by hand (its
372
+ source_file and source_line), since the
373
+ re-run checks the same entry again. A
374
+ cited-elsewhere one, whatever its severity,
375
+ takes in their place a warning that names
376
+ its gap_id, its source_citation, its
377
+ citation_check (with its words), its
378
+ map_locations and the fix for each side
379
+ that may be wrong: when the map is right,
380
+ re-run Test Skill (the test compared
381
+ another file's declaration), and if the gap
382
+ comes back (not for a missing one), exclude
383
+ the cited file in the brief's scope when the
384
+ skill does not cover it; when the export
385
+ moved, fix the entry by hand (with several
386
+ map_locations, whichever of those entries
387
+ moved), then re-run update-skill. A
388
+ citation_check that read nothing (UNREAD:
389
+ no-source-root, verifier-missing,
390
+ spot-check-failed) takes its own fix in
391
+ their place: give update-skill the source
392
+ tree, re-install SKF, or fix what the check
393
+ failed on, then re-run update-skill
324
394
  rescoped the entry removed
395
+ The signature: when §4a found the export's record (its verification
396
+ record has an extraction_method), every part from it, a params or
397
+ return_type it gives as null removed from the entry; with none, an
398
+ entry the map holds keeps its signature, and an entry this run adds
399
+ takes --merge-records'. An entry the map holds keeps its confidence and
400
+ extraction_method; when its own extraction_method is ast-grep or
401
+ ast_bridge, it takes the signature_source of its record's labels (T1
402
+ for an ast-grep record, T1-low for a source-read one) and, when the
403
+ record's export_type is another, the record's ast_node_type, while a
404
+ source-read entry keeps T1-low and no node kind. Each signature_source
405
+ it changes is listed in `relabeled`, {name, file, from, to}, each with a
406
+ warning. An internal-unreachable NEW_EXPORT takes none.
325
407
  Under the drift override (--drift-head and --drift-pinned) each warning
326
- names the drift.
408
+ names the drift, after the outcome's cause.
327
409
 
328
410
  full (degraded mode) makes every fresh record an entry.
329
411
 
@@ -351,8 +433,9 @@ Apply (update-skill write.md §3):
351
433
  --metadata-in exports[] less each name whose entry this run removed (a
352
434
  DELETED_EXPORT, a DELETED file's or a MOVED file's old path's, a
353
435
  rescoped one, a changed URL's) or renamed, unless an entry of the new
354
- map still holds it, then each name it added an entry for or renamed an
355
- entry to, each once; a renamed name takes its old name's place. A name
436
+ map still holds it, then each name it added an entry for (the entry's
437
+ reexported_as when it has one, the public name) or renamed an entry to,
438
+ each once; a renamed name takes its old name's place. A name
356
439
  left out as off the public surface (the summary's not_public) has no
357
440
  entry, so it is not added. A full update, which rebuilds the map, also
358
441
  drops each name the new map does not hold, and a reference app
@@ -379,15 +462,45 @@ Metadata (update-skill write.md §2):
379
462
  component library's own counts, a reference app's notes, a field a
380
463
  gap-driven patch added), the stats helper's fields an earlier run wrote
381
464
  that this run's does not (effective_denominator,
382
- pattern_surfaces_documented) dropped. It removes last_update and
383
- update_type, which the provenance map's update block records, and keeps
384
- every other field. It refuses (exit 1, nothing written) a --stats whose
385
- scripts_count or assets_count is not the length of --lists' scripts or
386
- assets, a --generation-date of another form, and a file it cannot read.
465
+ pattern_surfaces_documented) dropped and listed in `dropped_stats`,
466
+ each with a warning. When this run's effective_denominator differs
467
+ from the old one (one of them absent included), it also drops each old
468
+ stats key named effective_denominator_* (an explanation of the old
469
+ count), listed in `dropped_stats` with the warning "metadata:
470
+ stats.<key> dropped: it explained an effective_denominator this run
471
+ changed or removed"; when the two are the same, such a key stays with
472
+ no warning, since it still describes the denominator. Each other old
473
+ stats key it keeps adds the warning "metadata: stats.<key> kept from an
474
+ earlier run; check it still describes this run's counts", unless
475
+ create-skill writes it for the skill's scope_type (STATS_CREATE_FIELDS:
476
+ a component-library's Component Library stats of
477
+ compile-assembly-rules.md, a reference-app's notes).
478
+
479
+ tool_versions names the tools this run used: the old tool_versions
480
+ ({"ast_grep": null, "qmd": null} when absent, or when it is not an
481
+ object, which adds a warning), with skf set to the version
482
+ skf-render-quick-metadata.py's probe_skf_version reads under
483
+ --skf-root (package.json, else VERSION), never the old value, or to
484
+ "unknown" with a warning, one when no --skf-root is given and another
485
+ naming the folder when it holds no version; ast_grep set to the
486
+ ast_grep.version of --extraction (the recipe runner's output), or to
487
+ "unknown" with a warning when that is null but an export of it has
488
+ extraction_method ast-grep; qmd set to --qmd-version, a leading "qmd "
489
+ (as `qmd --version` prints it) removed. A tool this run did not use
490
+ (no --qmd-version, no --extraction, or one with no export ast-grep
491
+ matched) keeps its old value. It removes last_update and update_type,
492
+ which the provenance map's update block records, and keeps every
493
+ other field. It refuses (exit 1, nothing written, one stderr line) a
494
+ --stats whose scripts_count or assets_count is not the length of
495
+ --lists' scripts or assets, a --generation-date of another form, an
496
+ --extraction that is not a JSON object, a file it cannot read, and a
497
+ skf-render-quick-metadata.py beside it that cannot be loaded.
387
498
 
388
499
  Summary on stdout: {"status": "written", "map": <-o>, "entries":
389
500
  {"added", "updated", "removed"}, "not_public": [{name, file}],
390
- "file_entries": {"added", "updated", "removed"}, "warnings": [...]},
501
+ "relabeled": [{name, file, from, to}] (gap-driven),
502
+ "signature_source_backfilled": {tier: count}, "file_entries":
503
+ {"added", "updated", "removed"}, "warnings": [...]},
391
504
  or {"status": "refused",
392
505
  "blocking_unresolved": [{export_name, severity}]} with exit 3.
393
506
 
@@ -415,13 +528,26 @@ it scans):
415
528
  no seed holds is not added, with a warning. confidence_breakdown counts
416
529
  the exports by confidence, and T2 the exports with a qmd_evidence.
417
530
 
531
+ Aliases. Each --extraction entry_point_diff `public` and
532
+ `other_definitions` item (an other in-scope definition of a public
533
+ name), and each member of a namespace item, whose `local` differs from
534
+ its `name` gives its name to the record of its `local` name and file,
535
+ when one is seeded:
536
+ that record's `aliases` lists each such public name once, sorted (an
537
+ export re-exported under another name, `export { MockUniversalStore as
538
+ experimental_MockUniversalStore }`, keeps its declared name and carries
539
+ the public one). A record no item renames has no `aliases`, and with
540
+ no entry_point_diff (Quick tier, a runner exit 3, an export read by
541
+ eye) no record has any.
542
+
418
543
  Public surface. When --extraction's scope.type is public-api (in any
419
544
  case) and it has an entry_point_diff, each export of a language family
420
545
  whose entry_points.by_language status is `barrel` is marked `public`:
421
546
  true when its name and file are a pair of the runner's entry_point_diff
422
- `public` (its name, or the name `local` gives it in that file) or
423
- `extraction_gaps`, or when a `public` item of its name and family has
424
- no file; false otherwise. A `public` item with `via` namespace and a
547
+ `public` or `other_definitions` (its name, or the name `local` gives it
548
+ in that file) or `extraction_gaps`, or when a `public` item of its name
549
+ and family has no file; false otherwise. A `public` item with `via`
550
+ namespace and a
425
551
  `members` list (what the submodule passes on, which users reach as
426
552
  `pkg.sub.name`, and which counts.exports_public_api counts in its place)
427
553
  gives its members' pairs (each member's name, or its `local` one, and
@@ -450,8 +576,11 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
450
576
  but a MOVED_EXPORT needs the tree, with the first reason that fits:
451
577
  a DELETED_EXPORT "a public API recount from the tree (rule R1)", a
452
578
  provenance_completeness entry "a line from the tree (rule R3)", one
453
- whose map_match is not-found "a line and a signature from the tree",
454
- any other "a signature from the tree". Any such entry: {"status":
579
+ whose map_match is not-found with no candidate that has a
580
+ source_file "a line and a signature from the tree", any other (a
581
+ not-found with such a candidate, a citation of another file than
582
+ the map's entries, among them) "a signature from the tree". Any
583
+ such entry: {"status":
455
584
  "drift-blocked", "drift_blocked": [{gap_id, name, change_category,
456
585
  severity, reason}]}, and no spot-check runs.
457
586
  2. A DELETED_EXPORT without a `rescope` that holds its `amendment` and
@@ -467,10 +596,28 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
467
596
  its remediation's "definition line(s) (...)"
468
597
  NEW_EXPORT, MODIFIED_EXPORT
469
598
  map_match found checked at the map entry (its export_type)
470
- ambiguous unknown
599
+ ambiguous unknown, unknown_reason ambiguous (several
600
+ map entries of its name: no check runs)
471
601
  not-found checked at its source_citation (no export
472
- type) when it has one; when that pins no
473
- line (unknown or missing), or with no
602
+ type) when it has one. When the map holds
603
+ entries of its name (in_map, which a
604
+ not-found has only for a citation of
605
+ another file than theirs), one of them
606
+ with a source_file, and no check pins a
607
+ line (unknown or missing): unknown,
608
+ unknown_reason cited-elsewhere, whatever
609
+ its severity (a provenance_completeness
610
+ one too), and never re-extracted, so no
611
+ other file's declaration is read for it;
612
+ its record's citation_check keeps the
613
+ check's cause (missing, or its
614
+ unknown_reason) and its map_locations the
615
+ `file:line` (or file) of each candidate
616
+ with a source_file, both null on any other
617
+ record. Candidates with no source_file
618
+ (placeholders) do not count. Otherwise,
619
+ when that pins no line
620
+ (unknown or missing), or with no
474
621
  citation: targeted re-extraction (§4a) for a
475
622
  provenance_completeness entry with a source
476
623
  root, for a blocking one whatever its
@@ -480,37 +627,78 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
480
627
  A check gives `verified` (the line is a definition line, or a module
481
628
  or package entry whose file exists), `moved` (one definition line,
482
629
  not the recorded one: new_location), `missing` (no such file) or
483
- `unknown` (several definition lines, none the recorded one, or
484
- none). A file the rules cover no language of
485
- takes the answered `declaring_line`: the recorded line is verified,
486
- another is moved, null is unknown. With no verifier every check is
487
- unknown, with a `provenance: spot-checks not run` warning; a check
488
- the verifier fails on (a file it cannot read, or any error) is
489
- unknown, with a `provenance: spot-check failed for {name}: {error}`
490
- warning.
630
+ `unknown`, its unknown_reason `several-definitions` (several
631
+ definition lines, none the recorded one) or `no-definition` (none).
632
+ A file the rules cover no language of takes the answered
633
+ `declaring_line`: the recorded line is verified, another is moved,
634
+ null is unknown (`unchecked-language`). With no --source-root every
635
+ check is unknown (`no-source-root`); with no verifier every check is
636
+ unknown (`verifier-missing`), with a `provenance: spot-checks not
637
+ run` warning; a check the verifier fails on (a file it cannot read,
638
+ or any error) is unknown (`spot-check-failed`), with a `provenance:
639
+ spot-check failed for {name}: {error}` warning; a map entry with no
640
+ file or line is unknown (`no-line`). UNKNOWN_REASONS gives each its
641
+ words. Every other outcome has a null unknown_reason. A map_match
642
+ whose entry carries its own export_name (skf-parse-gaps matched it
643
+ by its reexported_as) is checked under that name.
644
+ A NEW_EXPORT or MODIFIED_EXPORT the map holds entries of its name
645
+ for (its map_match found or ambiguous, or not-found with a
646
+ candidate that has a source_file) never goes to targeted
647
+ re-extraction, whatever its severity: one that ends missing or
648
+ unknown keeps its map entries unchanged and adds none, and apply
649
+ warns (see "Apply").
650
+ A NEW_EXPORT or MODIFIED_EXPORT a check gives verified or moved, at
651
+ its map entry or at a citation the map holds no entry of its name
652
+ for, is located: its signature_at is the checked file and the
653
+ recorded line (verified) or the new one (moved). One whose citation
654
+ names another file than the map's entries of its name is not (apply
655
+ warns, as before), nor one whose file is absolute or outside
656
+ --source-root, links resolved, nor a NEW_EXPORT --judgments answers
657
+ internal-unreachable (one not yet answered is located, so its scan
658
+ may go unused).
491
659
  --plan stops here: {"status": "planned", "reextract": {"entries":
492
660
  [{gap_id, name, severity, resolved_paths}], "files": [the union of
493
- their resolved_paths]}, "warnings"}, the files also written to
494
- --files-out as a JSON array.
661
+ their resolved_paths, then of the located exports' files],
662
+ "signatures": [{gap_id, name, file, line} of each located export]},
663
+ "warnings"}, the files also written to --files-out as a JSON array.
495
664
  4. Targeted re-extraction (--remediation-records, `records`' output
496
665
  over those files): each routed entry with resolved_paths takes the
497
666
  first export of its name in the blocks of those files, in their
498
- order: `re-extracted`, new_location its record's location,
667
+ order, else the first whose `aliases` hold it (matched by name, or
668
+ by the public name the runner gives its local name in that file,
669
+ see "Records"): `re-extracted`, new_location its record's location,
499
670
  resolution_source remediation-paths, and the record copied into
500
671
  `files` as `records` wrote it, less its `public` mark (a normal
501
- update's), with the answered `docstring` added.
672
+ update's) and its `aliases`, with the answered `docstring` added. A
673
+ record matched by an alias is copied with `name` the public name and
674
+ `local` its own name. The aliases come from the runner's
675
+ entry_point_diff, so at Forge tier and above only: at Quick tier, or
676
+ when the runner exits 3 and §4a reads the files by eye, a gap naming
677
+ a renamed re-export by its public name matches no record.
502
678
  An entry with no match, or no path to scan, is unresolved, except a
503
679
  missing-export or missing-type one that is not blocking, which is
504
680
  unknown. Any unresolved entry: {"status": "unresolved",
505
681
  "unresolved": [{gap_id, name, severity, remediation_paths,
506
682
  rejected_paths, files_scanned, exports_found_in_scan}]}.
683
+ Each located export takes the one record of its name (or of its map
684
+ entry's own export_name) in the block of its file at its line, else
685
+ the one with no line: the record copied into `files` as above, under
686
+ the gap's name, less its `public` mark and `aliases`, with no
687
+ docstring asked
688
+ (one copy per name and location, or per record with no location),
689
+ and its type, params, return_type, extraction_method and
690
+ ast_node_type on the verification record. One with no such record
691
+ keeps them null, with a `provenance: {name}: located at
692
+ {file}:{line}, but no record of it there; signature not repaired`
693
+ warning (§4a's by-eye pass may add it to --export-details). Files to
694
+ match, of either kind, with no --remediation-records are an error.
507
695
  5. The public-reachability gate: every NEW_EXPORT that is verified,
508
696
  moved or re-extracted takes the answered `reachability`, `public` or
509
697
  `internal-unreachable`. An internal one keeps its outcome in its
510
- verification record, its record is not in `files`, `counts` tallies
511
- it under `reclassified` instead of its outcome, and it is listed in
512
- `reclassified` (merge queues it as a metadata update; apply writes no
513
- entry for it).
698
+ verification record (with no signature), its record is not in
699
+ `files`, `counts` tallies it under `reclassified` instead of its
700
+ outcome, and it is listed in `reclassified` (merge queues it as a
701
+ metadata update; apply writes no entry for it).
514
702
 
515
703
  Any answer missing in step 3 or 5 (a `docstring` too, for each
516
704
  re-extracted entry): {"status": "needs-judgment", "needs_judgment":
@@ -522,26 +710,32 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
522
710
  "unlabeled", "T2": 0} (each manifest entry by the extraction_method of
523
711
  its re-extracted record or, in the map, of its map entry; a pinned cited
524
712
  export the map lacks and the gate passed is T1-low), "verification":
525
- [one record per export-bearing entry: export_name, gap_category,
526
- severity, verification, in_map, map_entry, provenance_citation,
527
- source_citation, new_location, unknown_reason, pinned_definition_lines,
528
- resolution_source, reachability, export_type (always null: for a cited
529
- export the map lacks, apply takes it from merge.md's
530
- merge-records.json)], "files": [...]}, and prints {"status": "written",
713
+ [one record per export-bearing entry: gap_id, export_name,
714
+ gap_category, severity, verification, in_map, map_entry,
715
+ provenance_citation, source_citation, new_location, unknown_reason,
716
+ citation_check, map_locations, pinned_definition_lines,
717
+ resolution_source, reachability, and
718
+ export_type, params, return_type, extraction_method and ast_node_type
719
+ (a located export's record's, step 4; else null, and apply takes the
720
+ signature of an entry it adds from merge.md's merge-records.json)],
721
+ "files": [...]}, and prints {"status": "written",
531
722
  "output", "gap_count", "files_extracted", "exports_extracted",
532
723
  "confidence_breakdown", "counts" (per verification outcome, and
533
724
  `reclassified`), "reclassified", "forwarded", "targeted_reextraction",
534
- "warnings"}. When targeted re-extraction ran, `targeted_reextraction`
535
- is {resolved_count, files_scanned, exports_matched, tier}, also appended
536
- to --evidence as one JSON line keyed `targeted_reextraction`.
725
+ "warnings"}. When §4a ran (targeted re-extraction, or a located
726
+ export's signature), `targeted_reextraction` is {resolved_count,
727
+ files_scanned (the targeted re-extraction's files),
728
+ signature_files_scanned (the located exports' files), exports_matched,
729
+ tier}, also appended to --evidence as one JSON line keyed
730
+ `targeted_reextraction`.
537
731
 
538
732
  Baseline gaps (update-skill detect-changes Category B, step 1):
539
733
 
540
734
  The recipe runner never reports a module, an alias that renames, a
541
- dunder such as `__version__` or another underscore name, so the diff
542
- would read each such map entry as a deleted export. baseline-gaps lists,
543
- for step 2 to read by eye, each entry of a file --files lists (the
544
- modified files, Category A's modified-files.json) whose name and file
735
+ dunder such as `__version__` or another Python or Rust `_` name, so the
736
+ diff would read each such map entry as a deleted export. baseline-gaps
737
+ lists, for step 2 to read by eye, each entry of a file --files lists
738
+ (the modified files, Category A's modified-files.json) whose name and file
545
739
  no export of --extraction (the runner's JSON, or {"exports": []} when
546
740
  it could not run) holds, under its name or its re-export target, and
547
741
  whose file still declares it. The selection and the declaration rule
@@ -563,9 +757,10 @@ Exit codes:
563
757
  0 operation succeeded (for gap-records, whatever its `status`)
564
758
  1 user error (malformed JSON, bad path, malformed provenance file, a
565
759
  helper file that is not the output it names, a gap-records answer of
566
- the wrong type, targeted re-extraction to match with no
567
- --remediation-records, or, for baseline-gaps, a source root that is
568
- no folder or a verifier that cannot be loaded)
760
+ the wrong type, files to match (targeted re-extraction or a located
761
+ export's signature) with no --remediation-records, or, for
762
+ baseline-gaps, a source root that is no folder or a verifier that
763
+ cannot be loaded)
569
764
  3 apply refused: a blocking gap reached the provenance write with no
570
765
  source line (nothing written)
571
766
  """
@@ -1330,6 +1525,8 @@ REPLACE_WAIT_SECONDS = 5.0 # how long a Windows rename onto a file held open is
1330
1525
  # The update block apply sets at the top level of the map, in this order.
1331
1526
  UPDATE_BLOCK_KEYS = ("last_update", "update_type", "test_report_run_id", "files_changed", "exports_affected",
1332
1527
  "confidence_tier", "manual_sections_preserved")
1528
+ # The parts of a signature a gap-driven repair writes from a verification record, else from merge's records.
1529
+ SIGNATURE_PARTS = ("export_type", "params", "return_type")
1333
1530
 
1334
1531
 
1335
1532
  def _labels(method: object, node: object) -> dict:
@@ -1457,16 +1654,17 @@ def _without_none(fields: dict, keep: tuple = ()) -> dict:
1457
1654
 
1458
1655
  def _new_entry(fields: dict, library: str) -> dict:
1459
1656
  """An entry in create-skill's key order, from the fields a record gives."""
1460
- order = ("export_name", "export_type", "source_library", "params", "return_type", "source_file",
1657
+ order = ("export_name", "reexported_as", "export_type", "source_library", "params", "return_type", "source_file",
1461
1658
  "source_line", "confidence", "extraction_method", "ast_node_type", "signature_source")
1462
1659
  merged = {**fields, "source_library": fields.get("source_library") or library}
1463
1660
  out = {k: merged[k] for k in order if k in merged and (merged[k] is not None or k == "ast_node_type")}
1464
1661
  return out
1465
1662
 
1466
1663
 
1467
- def _rewrite(entry: dict, fields: dict) -> dict:
1468
- """The entry with `fields` set in place; keys the update does not set keep their values."""
1469
- out = dict(entry)
1664
+ def _rewrite(entry: dict, fields: dict, drop: tuple = ()) -> dict:
1665
+ """The entry with `fields` set in place and the keys `drop` names removed; keys the update does not set keep
1666
+ their values."""
1667
+ out = {k: v for k, v in entry.items() if k not in drop}
1470
1668
  for key, value in fields.items():
1471
1669
  if value is None and key not in ("ast_node_type",):
1472
1670
  continue
@@ -1490,19 +1688,87 @@ class _Map:
1490
1688
  "provenance_version": "2.0", "skill_name": skill_name, "skill_type": "single", "entries": []}
1491
1689
  if not isinstance(self.doc.get("entries"), list):
1492
1690
  raise ValueError("provenance `entries` must be an array")
1493
- libraries = [e.get("source_library") for e in self.doc["entries"]
1494
- if isinstance(e, dict) and isinstance(e.get("source_library"), str) and e["source_library"]]
1495
- self.library = libraries[0] if libraries else skill_name
1691
+ # the map's libraries in entry order, and each file's, as the map stood before the run (#808)
1692
+ self.libraries: list[str] = []
1693
+ self.libraries_by_file: dict[str, set[str]] = {}
1694
+ for entry in self.doc["entries"]:
1695
+ library = entry.get("source_library") if isinstance(entry, dict) else None
1696
+ if not (isinstance(library, str) and library):
1697
+ continue
1698
+ if library not in self.libraries:
1699
+ self.libraries.append(library)
1700
+ if _file_of(entry):
1701
+ self.libraries_by_file.setdefault(_file_of(entry), set()).add(library)
1702
+ self.library = self.libraries[0] if self.libraries else skill_name
1703
+ # each folder's libraries (the entries under it, "" the root) and _library_at per path, built on first use
1704
+ self._libraries_under: dict[str, set[str]] | None = None
1705
+ self._library_of: dict[str, str | None] = {}
1706
+ # (name, file) of each new entry library_for gave the first library for want of a better one
1707
+ self.library_unresolved: list[tuple[str | None, str | None]] = []
1496
1708
  self.added: list[str] = []
1497
1709
  self.updated: list[str] = []
1498
1710
  self.removed: list[str] = []
1499
1711
  self.renamed: list[tuple[str, str]] = [] # (old name, new name)
1500
1712
  self.touched_files: set[str] = set()
1501
1713
 
1502
- def find(self, name: str, path: str | None, line: int | None = None) -> int | None:
1503
- """The index of the entry with this name (and file, and line when several share both)."""
1504
- hits = [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and _name_of(e) == name
1505
- and (path is None or _file_of(e) == path)]
1714
+ def library_for(self, path: str | None, name: str | None = None, *, old: str | None = None) -> str:
1715
+ """The source_library of a new entry for `name` in `path` (the record of a MOVED file, or of an export
1716
+ whose change names an old_file, passes that old path as `old`, tried second). A map naming one library (or
1717
+ none: --skill-name) gives it; in one naming several, the library of the file, else of the nearest folder
1718
+ (see _library_at). With neither, or no path, the map's first library, and the entry goes to
1719
+ `library_unresolved` for the summary's warning."""
1720
+ if len(self.libraries) <= 1:
1721
+ return self.library
1722
+ for candidate in (path, old):
1723
+ library = self._library_at(_norm_path(candidate))
1724
+ if library is not None:
1725
+ return library
1726
+ self.library_unresolved.append((name, _norm_path(path) or _norm_path(old)))
1727
+ return self.library
1728
+
1729
+ @staticmethod
1730
+ def _root_library(libraries: set[str]) -> str | None:
1731
+ """The one library of the set, or of several the one that is a path prefix of all the others (`storybook`
1732
+ beside `storybook/test`); None when no library is."""
1733
+ return next((library for library in sorted(libraries)
1734
+ if all(other == library or other.startswith(library + "/") for other in libraries)), None)
1735
+
1736
+ def _library_at(self, path: str | None) -> str | None:
1737
+ """The library the map's entries of `path` settle (see _root_library), else the one the entries under the
1738
+ nearest folder above it that holds entries settle; None when neither does. Memoized per path."""
1739
+ if not path:
1740
+ return None
1741
+ if path in self._library_of:
1742
+ return self._library_of[path]
1743
+ if self._libraries_under is None:
1744
+ self._libraries_under = {}
1745
+ for file, libraries in self.libraries_by_file.items():
1746
+ folders = file.split("/")[:-1]
1747
+ for depth in range(len(folders) + 1):
1748
+ prefix = "/".join(folders[:depth]) + "/" if depth else ""
1749
+ self._libraries_under.setdefault(prefix, set()).update(libraries)
1750
+ library = self._root_library(self.libraries_by_file.get(path, set()))
1751
+ if library is None:
1752
+ folders = path.split("/")[:-1]
1753
+ for depth in range(len(folders), -1, -1):
1754
+ under = self._libraries_under.get("/".join(folders[:depth]) + "/" if depth else "")
1755
+ if under:
1756
+ library = self._root_library(under)
1757
+ break
1758
+ self._library_of[path] = library
1759
+ return library
1760
+
1761
+ def find(self, name: str, path: str | None, line: int | None = None, *, reexport: bool = False) -> int | None:
1762
+ """The index of the entry with this name (and file, and line when several share both); with `reexport`,
1763
+ when no entry has the name, of the entry whose reexported_as is the name (a gap names a renamed re-export
1764
+ by its public name)."""
1765
+ def hits_of(key) -> list[int]:
1766
+ return [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and key(e) == name
1767
+ and (path is None or _file_of(e) == path)]
1768
+
1769
+ hits = hits_of(_name_of)
1770
+ if not hits and reexport:
1771
+ hits = hits_of(lambda e: e.get("reexported_as"))
1506
1772
  if len(hits) > 1 and line is not None:
1507
1773
  hits = [i for i in hits if _line_of(self.doc["entries"][i].get("source_line")) == line]
1508
1774
  return hits[0] if len(hits) == 1 else None
@@ -1516,9 +1782,9 @@ class _Map:
1516
1782
  if entry.get("source_file"):
1517
1783
  self.touched_files.add(_norm_path(entry["source_file"]))
1518
1784
 
1519
- def update(self, index: int, fields: dict) -> None:
1785
+ def update(self, index: int, fields: dict, drop: tuple = ()) -> None:
1520
1786
  before = self.doc["entries"][index]
1521
- after = _rewrite(before, fields)
1787
+ after = _rewrite(before, fields, drop)
1522
1788
  if after != before:
1523
1789
  self.doc["entries"][index] = after
1524
1790
  self.updated.append(_name_of(after))
@@ -1561,7 +1827,7 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
1561
1827
  not_public.append({"name": name, "file": path})
1562
1828
  return True
1563
1829
 
1564
- def refresh(index: int | None, name: str, path: str, *, add: bool) -> None:
1830
+ def refresh(index: int | None, name: str, path: str, *, add: bool, old: str | None = None) -> None:
1565
1831
  record = fresh.record(name, path)
1566
1832
  if record is None:
1567
1833
  warnings.append(f"provenance: {name} in {path}: no extraction record, entry "
@@ -1569,7 +1835,7 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
1569
1835
  return
1570
1836
  if index is None:
1571
1837
  if add and not off_surface(name, path):
1572
- pmap.add(_new_entry(record, pmap.library))
1838
+ pmap.add(_new_entry(record, pmap.library_for(path, name, old=old)))
1573
1839
  return
1574
1840
  pmap.update(index, _without_none(record, keep=("ast_node_type",)))
1575
1841
 
@@ -1591,7 +1857,7 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
1591
1857
  if carried is None and off_surface(name, key_path):
1592
1858
  continue
1593
1859
  entry = _rewrite(carried, _without_none(record, keep=("ast_node_type",))) if carried \
1594
- else _new_entry(record, pmap.library)
1860
+ else _new_entry(record, pmap.library_for(key_path, name, old=old))
1595
1861
  pmap.add(entry)
1596
1862
  continue
1597
1863
  if status == "ADDED":
@@ -1624,9 +1890,13 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
1624
1890
  if all(record.get(part) is not None for part in ("params", "return_type")
1625
1891
  if entry.get(part) is not None):
1626
1892
  fields["signature_source"] = record["signature_source"]
1893
+ elif _tier(entry.get("signature_source")) is None:
1894
+ # the signature it keeps was read by eye: set that, or the backfill would derive T1
1895
+ # from the ast-grep label set here
1896
+ fields["signature_source"] = "T1-low"
1627
1897
  pmap.update(index, fields)
1628
1898
  elif kind in ("NEW_EXPORT", "MODIFIED_EXPORT"):
1629
- refresh(index, name, path, add=True)
1899
+ refresh(index, name, path, add=True, old=_norm_path(change.get("old_file")))
1630
1900
  for rename in _get_list(categories or {}, "category_c", "renamed_exports"):
1631
1901
  if not isinstance(rename, dict):
1632
1902
  continue
@@ -1663,7 +1933,7 @@ def _apply_docs_only(pmap: _Map, records: dict) -> tuple[int, int]:
1663
1933
  "confidence": "T3", "signature_source": "T3"}
1664
1934
  carried = old.get(_name_of(record))
1665
1935
  entry = _rewrite(carried, _without_none(fields)) if carried else _new_entry(
1666
- _without_none(fields), pmap.library)
1936
+ _without_none(fields), pmap.library_for(url, _name_of(record)))
1667
1937
  pmap.add(entry)
1668
1938
  return len(changed), len(exports)
1669
1939
 
@@ -1673,8 +1943,9 @@ def _drift_note(drift: tuple | None) -> str:
1673
1943
 
1674
1944
 
1675
1945
  def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, drift: tuple | None,
1676
- warnings: list[str]) -> tuple[int, int] | list[dict]:
1677
- """Gap-driven mode: one write per verification record. A list return is the refusal."""
1946
+ warnings: list[str], relabeled: list[dict]) -> tuple[int, int] | list[dict]:
1947
+ """Gap-driven mode: one write per verification record. A list return is the refusal; each entry whose
1948
+ signature_source the repair changes goes to `relabeled`."""
1678
1949
  verification = [r for r in records.get("verification") or [] if isinstance(r, dict) and _name_of(r)]
1679
1950
  reextracted = _Fresh(records.get("files") or [], None, None)
1680
1951
  merged = {}
@@ -1685,18 +1956,108 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
1685
1956
  def in_map(r: dict) -> bool:
1686
1957
  return bool(r.get("in_map")) or bool(r.get("map_entry"))
1687
1958
 
1959
+ def signature(r: dict) -> tuple[dict, tuple]:
1960
+ """(the signature parts to set, the parts to remove) of an export: with the record §4a's extraction gave at
1961
+ its line (its verification record has an extraction_method), every part from it, a null params or
1962
+ return_type removed; with none, merge's."""
1963
+ if r.get("extraction_method") is not None:
1964
+ return _without_none({part: r.get(part) for part in SIGNATURE_PARTS}), \
1965
+ tuple(part for part in ("params", "return_type") if r.get(part) is None)
1966
+ extra = merged.get(_name_of(r), {})
1967
+ return _without_none({part: extra.get(part) for part in SIGNATURE_PARTS}), ()
1968
+
1969
+ def resign(r: dict, index: int, fields: dict) -> None:
1970
+ """Write an entry the map holds with `fields` and, when §4a found the export's record, its signature; an
1971
+ ast-grep entry takes the record's signature_source, and its node kind when the record's export_type is
1972
+ another (a source-read entry keeps T1-low and no kind); confidence and extraction_method stay."""
1973
+ entry = pmap.doc["entries"][index]
1974
+ drop: tuple = ()
1975
+ method = r.get("extraction_method")
1976
+ if r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT") and method is not None \
1977
+ and r.get("reachability") != "internal-unreachable":
1978
+ parts, gone = signature(r)
1979
+ fields = {**fields, **parts}
1980
+ drop = tuple(part for part in gone if entry.get(part) is not None)
1981
+ if entry.get("extraction_method") in ("ast-grep", "ast_bridge"):
1982
+ fields["signature_source"] = _labels(method, None)["signature_source"]
1983
+ if parts.get("export_type") not in (None, entry.get("export_type")):
1984
+ fields["ast_node_type"] = r.get("ast_node_type")
1985
+ old, new = entry.get("signature_source"), fields["signature_source"]
1986
+ if old is not None and old != new:
1987
+ path = _file_of(entry)
1988
+ relabeled.append({"name": _name_of(entry), "file": path, "from": old, "to": new})
1989
+ warnings.append(f"provenance: {_name_of(entry)} in {path}: signature_source {old} relabeled "
1990
+ f"{new}, from the {method} record of its signature")
1991
+ pmap.update(index, fields, drop)
1992
+
1993
+ note = _drift_note(drift)
1994
+
1995
+ def why(r: dict) -> str:
1996
+ """The cause of an `unknown` outcome its record gives, `<unknown_reason>: <words>`, else ''."""
1997
+ reason = r.get("unknown_reason")
1998
+ return f"{reason}: {UNKNOWN_REASONS[reason]}" if r.get("verification") == "unknown" \
1999
+ and reason in UNKNOWN_REASONS else ""
2000
+
2001
+ def unchecked(r: dict) -> str | None:
2002
+ """The warning for a blocking NEW_EXPORT or MODIFIED_EXPORT the map holds that its spot-check could not
2003
+ locate (missing, or unknown for a cause its unknown_reason names), which stays unchanged: the gap, the
2004
+ export, the outcome, its cause and the one fix a re-run acts on; None for any other record."""
2005
+ outcome = r.get("verification")
2006
+ if outcome not in ("missing", "unknown") or not in_map(r) or not _blocking(r.get("severity")) \
2007
+ or r.get("gap_category") not in ("NEW_EXPORT", "MODIFIED_EXPORT"):
2008
+ return None
2009
+ name, gap = _name_of(r), r.get("gap_id")
2010
+ path = _location(r.get("map_entry"))[0]
2011
+ cause = f"its map entry's file{f' {path}' if path else ''} is gone" if outcome == "missing" else why(r)
2012
+ detail = "; ".join(part for part in (cause, note) if part)
2013
+ if r.get("unknown_reason") == "ambiguous": # a Source at the definition lets the re-run's lookup pick one
2014
+ fix = ("its entries are left unchanged. Fix its entries by hand, or give the gap a `file:line` Source at "
2015
+ "its definition when that file holds only one of them, then re-run update-skill")
2016
+ else: # the re-run checks the same map entry again: only a hand fix changes it
2017
+ fix = ("its entry is left unchanged. Fix the entry by hand (set its source_file and source_line to the "
2018
+ "line that defines it), then re-run update-skill")
2019
+ return f"provenance: {name}{f' ({gap})' if gap else ''}: {outcome}{f' ({detail})' if detail else ''}; {fix}"
2020
+
2021
+ def elsewhere(r: dict) -> str | None:
2022
+ """The warning for a NEW_EXPORT or MODIFIED_EXPORT whose gap cites another file than the map's entries of its
2023
+ name (unknown, cited-elsewhere), whatever its severity: the gap, the export, the cited file and its check's
2024
+ cause, the record's map_locations and the fix for each side that may be wrong; None for any other record."""
2025
+ if r.get("verification") != "unknown" or r.get("unknown_reason") != "cited-elsewhere":
2026
+ return None
2027
+ name, gap = _name_of(r), r.get("gap_id")
2028
+ cited_file, cited_line = _location(r.get("source_citation"))
2029
+ cited = _where(cited_file, cited_line) or cited_file
2030
+ check = r.get("citation_check") if isinstance(r.get("citation_check"), str) else None
2031
+ words = "no such file" if check == "missing" else UNKNOWN_REASONS.get(check or "")
2032
+ where = [w for w in r.get("map_locations") or [] if isinstance(w, str) and w]
2033
+ detail = "; ".join(part for part in (
2034
+ why(r), f"it cites {cited}" if cited else "",
2035
+ f"its check there: {check}{f' ({words})' if words else ''}" if check else "",
2036
+ f"the map holds it at {', '.join(where)}" if where else "", note) if part)
2037
+ several = len(where) > 1
2038
+ kept = "its entries are left unchanged" if several else "its entry is left unchanged"
2039
+ head = f"provenance: {name}{f' ({gap})' if gap else ''}: unknown ({detail}); {kept}, and nothing is " \
2040
+ "re-extracted for it."
2041
+ if check in UNREAD: # nothing was read at the cited file: make the check read it first
2042
+ return f"{head} Nothing was read at the cited file: {UNREAD[check]}, then re-run update-skill"
2043
+ moved = "when an export moved, fix by hand whichever of those entries moved" if several \
2044
+ else "when the export moved, fix the map entry by hand"
2045
+ scope = "" if check == "missing" else (", and if the gap comes back, exclude the cited file in the brief's "
2046
+ "scope when the skill does not cover it")
2047
+ return (f"{head} When the map is right, re-run Test Skill, since the test compared another file's "
2048
+ f"declaration{scope}; {moved} (set its source_file and source_line to the line that defines it), "
2049
+ f"then re-run update-skill")
2050
+
1688
2051
  blocking = [{"export_name": _name_of(r), "severity": r.get("severity")} for r in verification
1689
2052
  if r.get("verification") == "unknown" and not in_map(r)
1690
- and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT")
1691
- and str(r.get("severity") or "").strip().lower() not in NON_BLOCKING]
2053
+ and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT") and _blocking(r.get("severity"))]
1692
2054
  if blocking:
1693
2055
  return blocking
1694
- note = _drift_note(drift)
1695
2056
  pinned_keys: set[tuple] = set()
1696
2057
  for r in verification:
1697
2058
  name, outcome, category = _name_of(r), r.get("verification"), r.get("gap_category")
1698
2059
  entry_file, entry_line = _location(r.get("map_entry"))
1699
- index = pmap.find(name, entry_file, entry_line) if r.get("map_entry") else None
2060
+ index = pmap.find(name, entry_file, entry_line, reexport=True) if r.get("map_entry") else None
1700
2061
  lines = r.get("pinned_definition_lines")
1701
2062
  r5 = f"; definition lines per the test report: {lines}" if category == "MOVED_EXPORT" else ""
1702
2063
  if outcome == "rescoped":
@@ -1705,10 +2066,10 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
1705
2066
  elif outcome == "verified" and index is not None:
1706
2067
  if drift and category == "MOVED_EXPORT":
1707
2068
  warnings.append(f"provenance: {name}: verified at HEAD only ({note}{r5})")
2069
+ resign(r, index, {})
1708
2070
  elif outcome == "moved" and index is not None:
1709
2071
  path, line = _location(r.get("new_location"))
1710
- if path and line is not None:
1711
- pmap.update(index, {"source_file": path, "source_line": line})
2072
+ resign(r, index, {"source_file": path, "source_line": line} if path and line is not None else {})
1712
2073
  elif outcome == "re-extracted":
1713
2074
  if r.get("reachability") == "internal-unreachable":
1714
2075
  continue # not public API: merge queued it as a metadata update
@@ -1717,9 +2078,13 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
1717
2078
  if record is None:
1718
2079
  warnings.append(f"provenance: {name}: re-extracted, but no record of it in the run's files")
1719
2080
  continue
1720
- existing = pmap.find(name, path)
2081
+ local = (reextracted.worker.get((name, path)) or {}).get("local")
2082
+ if isinstance(local, str) and local and local != name:
2083
+ # matched by the public name a re-export gives it: the entry keeps its declared name
2084
+ record = {**record, "export_name": local, "reexported_as": name}
2085
+ existing = pmap.find(name, path, reexport=True)
1721
2086
  if existing is None:
1722
- pmap.add(_new_entry(record, pmap.library))
2087
+ pmap.add(_new_entry(record, pmap.library_for(path, name)))
1723
2088
  else:
1724
2089
  pmap.update(existing, _without_none(record, keep=("ast_node_type",)))
1725
2090
  elif outcome in ("verified", "moved") and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
@@ -1727,29 +2092,28 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
1727
2092
  continue
1728
2093
  where = r.get("source_citation") if outcome == "verified" else r.get("new_location")
1729
2094
  path, line = _location(where)
1730
- if not path or line is None or (name, path) in pinned_keys or pmap.find(name, path) is not None:
2095
+ if not path or line is None or (name, path) in pinned_keys \
2096
+ or pmap.find(name, path, reexport=True) is not None:
1731
2097
  continue
1732
2098
  pinned_keys.add((name, path))
1733
- extra = merged.get(name, {})
2099
+ # the signature its record gives, else merge's; the source-read labels whatever the record's tool
1734
2100
  pmap.add(_new_entry(_without_none({
1735
- "export_name": name,
1736
- "export_type": extra.get("export_type") or r.get("export_type"),
1737
- "params": extra.get("params"), "return_type": extra.get("return_type"),
2101
+ "export_name": name, **signature(r)[0],
1738
2102
  "source_file": path, "source_line": line, **_labels("source-read", None)},
1739
- keep=("ast_node_type",)), pmap.library))
2103
+ keep=("ast_node_type",)), pmap.library_for(path, name)))
1740
2104
  elif outcome == "unknown" and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
1741
- extra = merged.get(name, {})
1742
2105
  pmap.add(_new_entry(_without_none({
1743
- "export_name": name, "export_type": extra.get("export_type") or r.get("export_type"),
1744
- "params": extra.get("params"), "return_type": extra.get("return_type"),
1745
- **_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library))
2106
+ "export_name": name, **signature(r)[0],
2107
+ **_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library_for(None, name)))
1746
2108
  else:
1747
2109
  if r.get("unknown_reason") == "drift-override":
1748
2110
  warnings.append(f"provenance: {name}: unknown ({note}; no line taken from HEAD{r5})")
2111
+ elif (warning := elsewhere(r) or unchecked(r)) is not None:
2112
+ warnings.append(warning)
1749
2113
  elif drift:
1750
- warnings.append(f"provenance: {name}: {outcome} ({note}{r5})")
2114
+ warnings.append(f"provenance: {name}: {outcome} ({why(r) + '; ' if why(r) else ''}{note}{r5})")
1751
2115
  else:
1752
- warnings.append(f"provenance: {name}: {outcome}{r5}")
2116
+ warnings.append(f"provenance: {name}: {outcome}{f' ({why(r)})' if why(r) else ''}{r5}")
1753
2117
  return len(pmap.touched_files), len(verification)
1754
2118
 
1755
2119
 
@@ -1830,6 +2194,9 @@ def exports_after(metadata: object, pmap: _Map, update_type: str) -> list:
1830
2194
  return []
1831
2195
  current = metadata.get("exports") if isinstance(metadata.get("exports"), list) else []
1832
2196
  held = {_name_of(e) for e in pmap.doc["entries"] if isinstance(e, dict)} - {None}
2197
+ # an entry added under its declared name, re-exported under a public one: exports[] lists the public name
2198
+ public = {_name_of(e): e["reexported_as"] for e in pmap.doc["entries"]
2199
+ if isinstance(e, dict) and isinstance(e.get("reexported_as"), str) and e["reexported_as"]}
1833
2200
  renamed = dict(pmap.renamed)
1834
2201
  gone = ({n for n in pmap.removed if n} | set(renamed)) - held
1835
2202
  if update_type == "full":
@@ -1849,8 +2216,8 @@ def exports_after(metadata: object, pmap: _Map, update_type: str) -> list:
1849
2216
  put(new, new if new in held else None)
1850
2217
  else:
1851
2218
  put(item, name)
1852
- for name in pmap.added + [new for _old, new in pmap.renamed]:
1853
- put(name, name if name in held else None)
2219
+ for name in [public.get(n, n) for n in pmap.added] + [new for _old, new in pmap.renamed]:
2220
+ put(name, name if name in held or name in public.values() else None)
1854
2221
  return out
1855
2222
 
1856
2223
 
@@ -1914,6 +2281,50 @@ def file_rows_after(metadata: dict, pmap: _Map, source_root: Path | None) -> dic
1914
2281
  return lists
1915
2282
 
1916
2283
 
2284
+ # the signature_source tiers skf-render-metadata-stats.py bins, in their spelling (it compares them case-insensitively)
2285
+ SIGNATURE_SOURCES = ("T1", "T1-low", "T2", "T3")
2286
+ WARNING_LIST_CAP = 10 # the items a warning lists before "and N more"
2287
+
2288
+
2289
+ def _capped(items: list[str]) -> str:
2290
+ """The items joined with ", ", the first WARNING_LIST_CAP of them and "and N more" for the rest."""
2291
+ more = len(items) - WARNING_LIST_CAP
2292
+ return ", ".join(items[:WARNING_LIST_CAP] + ([f"and {more} more"] if more > 0 else []))
2293
+
2294
+
2295
+ def _tier(value: object) -> str | None:
2296
+ """A signature_source or confidence tier in its SIGNATURE_SOURCES spelling; None for any other value."""
2297
+ if not isinstance(value, str):
2298
+ return None
2299
+ return next((tier for tier in SIGNATURE_SOURCES if tier.upper() == value.strip().upper()), None)
2300
+
2301
+
2302
+ def _backfill_signature_source(pmap: _Map) -> tuple[dict[str, int], list[tuple[object, object, str]]]:
2303
+ """Give each entry whose signature_source is missing or not a tier the stats helper bins (a map written before
2304
+ the key existed, #802) the one its labels give: its confidence when that names T1-low, T2 or T3, else T1 for
2305
+ an extraction_method of ast-grep or ast_bridge (compared stripped and case-insensitively) and T1-low for any
2306
+ other (relabel-rule.md: T1 needs evidence an ast-grep rule matched). Returns (the fills counted by value, in
2307
+ tier order; (name, old value, new value) of each unrecognized value replaced, which the map no longer holds);
2308
+ the names are not listed as updated."""
2309
+ counts: dict[str, int] = {}
2310
+ replaced: list[tuple[object, object, str]] = []
2311
+ for entry in pmap.doc["entries"]:
2312
+ if not isinstance(entry, dict) or _tier(entry.get("signature_source")) is not None:
2313
+ continue
2314
+ confidence, method = _tier(entry.get("confidence")), entry.get("extraction_method")
2315
+ if confidence in ("T1-low", "T2", "T3"):
2316
+ value = confidence
2317
+ elif isinstance(method, str) and method.strip().lower() in ("ast-grep", "ast_bridge"):
2318
+ value = "T1"
2319
+ else:
2320
+ value = "T1-low"
2321
+ if entry.get("signature_source") is not None:
2322
+ replaced.append((entry.get("export_name"), entry["signature_source"], value))
2323
+ entry["signature_source"] = value
2324
+ counts[value] = counts.get(value, 0) + 1
2325
+ return {tier: counts[tier] for tier in SIGNATURE_SOURCES if tier in counts}, replaced
2326
+
2327
+
1917
2328
  def apply_update(*, update_type: str, provenance: dict | None, skill_name: str, manifest: dict | None = None,
1918
2329
  categories: dict | None = None, extraction: dict | None = None, details: dict | None = None,
1919
2330
  records: dict | None = None, merge_records: dict | None = None, compare: dict | None = None,
@@ -1936,9 +2347,10 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
1936
2347
  pmap = _Map(provenance, skill_name)
1937
2348
  warnings: list[str] = []
1938
2349
  not_public: list[dict] = []
2350
+ relabeled: list[dict] = []
1939
2351
  mode = records.get("mode")
1940
2352
  if update_type == "gap-driven":
1941
- outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings)
2353
+ outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings, relabeled)
1942
2354
  if isinstance(outcome, list):
1943
2355
  return {}, {"status": "refused", "blocking_unresolved": outcome}
1944
2356
  files, exports = outcome
@@ -1951,7 +2363,7 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
1951
2363
  index = pmap.find(name, path)
1952
2364
  record = fresh.record(name, path)
1953
2365
  if index is None:
1954
- pmap.add(_new_entry(record, pmap.library))
2366
+ pmap.add(_new_entry(record, pmap.library_for(path, name)))
1955
2367
  else:
1956
2368
  pmap.update(index, _without_none(record, keep=("ast_node_type",)))
1957
2369
  files = len({path for _name, path in fresh.all_keys()})
@@ -1963,6 +2375,23 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
1963
2375
  f"({fresh.surface_problem}): every new export was added")
1964
2376
  if not_public:
1965
2377
  warnings.append(_not_public_warning(not_public))
2378
+ if pmap.library_unresolved:
2379
+ count = len(pmap.library_unresolved)
2380
+ named = _capped([f"{name} in {path}" if path else f"{name} (no file)"
2381
+ for name, path in pmap.library_unresolved])
2382
+ warnings.append(f"provenance: {count} new entr{'y' if count == 1 else 'ies'} took source_library "
2383
+ f"{pmap.library}, the map's first, as the map names several libraries "
2384
+ f"({', '.join(pmap.libraries)}) and each entry's file or folder does not settle which: "
2385
+ f"{named}; check each entry's source_library")
2386
+ backfilled, replaced = _backfill_signature_source(pmap)
2387
+ if backfilled:
2388
+ count = sum(backfilled.values())
2389
+ listed = [f"{name}: {old if isinstance(old, str) else json.dumps(old)} -> {new}" for name, old, new in replaced]
2390
+ values = f"; unrecognized values replaced: {_capped(listed)}" if replaced else ""
2391
+ warnings.append(f"provenance: backfilled signature_source on {count} entr{'y' if count == 1 else 'ies'} "
2392
+ f"that had none or an unrecognized value (a map written before the key existed), from each "
2393
+ f"one's confidence and extraction_method: "
2394
+ f"{', '.join(f'{n} {value}' for value, n in backfilled.items())}{values}")
1966
2395
  file_changes = _apply_files(pmap, compare, new_files, promoted, source_root)
1967
2396
  doc = pmap.doc
1968
2397
  for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
@@ -1983,6 +2412,8 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
1983
2412
  "status": "written",
1984
2413
  "entries": {"added": pmap.added, "updated": pmap.updated, "removed": pmap.removed},
1985
2414
  "not_public": not_public,
2415
+ "relabeled": relabeled,
2416
+ "signature_source_backfilled": backfilled,
1986
2417
  "file_entries": file_changes,
1987
2418
  "warnings": warnings,
1988
2419
  }
@@ -1997,11 +2428,98 @@ GENERATION_DATE_RE = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ")
1997
2428
  STATS_HELPER_FIELDS = ("exports_documented", "exports_public_api", "exports_internal", "exports_total",
1998
2429
  "public_api_coverage", "total_coverage", "scripts_count", "assets_count",
1999
2430
  "pattern_surfaces_documented", "effective_denominator")
2431
+ # the other stats fields create-skill writes, by the skill's scope_type: a component library's own counts (the
2432
+ # Component Library stats of compile-assembly-rules.md, its stats_extra) and a reference app's notes
2433
+ STATS_CREATE_FIELDS = {
2434
+ "component-library": ("components_registered", "components_documented", "props_interfaces_extracted",
2435
+ "components_unique", "demo_files_excluded", "design_variants"),
2436
+ "reference-app": ("notes",),
2437
+ }
2438
+ # a stats key that explains effective_denominator, dropped when this run's denominator differs from the old one
2439
+ DENOMINATOR_NOTE_PREFIX = "effective_denominator_"
2440
+ SKF_NO_ROOT = ("metadata: tool_versions.skf is unknown: no --skf-root was given; pass the installed SKF's folder "
2441
+ "({project-root}/_bmad/skf)")
2442
+ AST_GREP_UNKNOWN = ("metadata: tool_versions.ast_grep is unknown: --extraction holds exports ast-grep matched but no "
2443
+ "ast_grep.version")
2444
+ _QMD_PREFIX = re.compile(r"^qmd\s+", re.IGNORECASE) # what `qmd --version` prints first
2445
+
2446
+
2447
+ def _tool_version(value: object) -> str | None:
2448
+ return value.strip() if isinstance(value, str) and value.strip() else None
2449
+
2450
+
2451
+ def _skf_unknown(skf_root: str | Path) -> str:
2452
+ return (f"metadata: tool_versions.skf is unknown: {Path(skf_root).as_posix()} holds no package.json or VERSION "
2453
+ f"file with a version")
2454
+
2455
+
2456
+ def _kept_warning(key: str) -> str:
2457
+ return f"metadata: stats.{key} kept from an earlier run; check it still describes this run's counts"
2458
+
2459
+
2460
+ def tool_versions_after(old: object, *, skf_version: str | None, skf_root: str | Path | None,
2461
+ ast_grep_version: str | None, qmd_version: str | None) -> tuple[dict, list[str]]:
2462
+ """(the tool_versions an update leaves, its warnings) (see "Metadata" in the module docstring)."""
2463
+ warnings: list[str] = []
2464
+ if isinstance(old, dict):
2465
+ tools = dict(old)
2466
+ else:
2467
+ tools = {"ast_grep": None, "qmd": None}
2468
+ if old is not None:
2469
+ kind = {bool: "a boolean", str: "a string", list: "an array"}.get(type(old), "a number")
2470
+ warnings.append(f"metadata: tool_versions was {kind}, not an object; replaced with the versions this run "
2471
+ f"records")
2472
+ ast_grep = _tool_version(ast_grep_version)
2473
+ if ast_grep is not None:
2474
+ tools["ast_grep"] = ast_grep
2475
+ if ast_grep == "unknown":
2476
+ warnings.append(AST_GREP_UNKNOWN)
2477
+ qmd = _tool_version(qmd_version)
2478
+ if qmd is not None:
2479
+ qmd = _tool_version(_QMD_PREFIX.sub("", qmd, count=1))
2480
+ if qmd is not None:
2481
+ tools["qmd"] = qmd
2482
+ skf = _tool_version(skf_version)
2483
+ if skf is None or skf == "unknown":
2484
+ skf = "unknown"
2485
+ warnings.append(SKF_NO_ROOT if skf_root is None else _skf_unknown(skf_root))
2486
+ tools["skf"] = skf
2487
+ return tools, warnings
2488
+
2489
+
2490
+ def stats_after(old_stats: dict, new_stats: dict, scope_type: object) -> tuple[dict, list[str], list[str]]:
2491
+ """(the stats an update leaves, dropped_stats, their warnings) (see "Metadata" in the module docstring)."""
2492
+ exempt = STATS_CREATE_FIELDS.get(scope_type, ()) if isinstance(scope_type, str) else ()
2493
+ denominator = new_stats.get("effective_denominator")
2494
+ recounted = old_stats.get("effective_denominator") != denominator
2495
+ kept: dict = {}
2496
+ dropped: list[str] = []
2497
+ warnings: list[str] = []
2498
+ for key, value in old_stats.items():
2499
+ if key in new_stats:
2500
+ continue
2501
+ if key in STATS_HELPER_FIELDS:
2502
+ dropped.append(key)
2503
+ warnings.append(f"metadata: stats.{key} dropped: this run's stats helper did not write it")
2504
+ continue
2505
+ if key.startswith(DENOMINATOR_NOTE_PREFIX) and recounted:
2506
+ dropped.append(key)
2507
+ warnings.append(f"metadata: stats.{key} dropped: it explained an effective_denominator this run changed "
2508
+ f"or removed")
2509
+ continue
2510
+ kept[key] = value
2511
+ # an explanation of a denominator this run left as it was still describes it
2512
+ if key not in exempt and not (key.startswith(DENOMINATOR_NOTE_PREFIX) and denominator is not None):
2513
+ warnings.append(_kept_warning(key))
2514
+ return {**new_stats, **kept}, dropped, warnings
2000
2515
 
2001
2516
 
2002
2517
  def update_metadata(metadata: object, lists: object, stats: object, *, version: str, generation_date: str,
2003
- source_commit: str | None = None, source_ref: str | None = None) -> dict:
2004
- """The metadata.json an update leaves (see "Metadata" in the module docstring). Raises ValueError."""
2518
+ source_commit: str | None = None, source_ref: str | None = None,
2519
+ skf_version: str | None = None, skf_root: str | Path | None = None,
2520
+ ast_grep_version: str | None = None, qmd_version: str | None = None) -> tuple[dict, dict]:
2521
+ """(the metadata.json an update leaves, {"dropped_stats": [...], "warnings": [...]}) (see "Metadata" in the
2522
+ module docstring). Raises ValueError."""
2005
2523
  if not isinstance(metadata, dict):
2006
2524
  raise ValueError("--metadata file must hold a JSON object (the skill's metadata.json)")
2007
2525
  if not isinstance(lists, dict) or not all(isinstance(lists.get(k), list) for k in ("exports", "scripts",
@@ -2033,11 +2551,14 @@ def update_metadata(metadata: object, lists: object, stats: object, *, version:
2033
2551
  out[kind] = lists[kind]
2034
2552
  else:
2035
2553
  out.pop(kind, None)
2554
+ out["tool_versions"], warnings = tool_versions_after(
2555
+ metadata.get("tool_versions"), skf_version=skf_version, skf_root=skf_root,
2556
+ ast_grep_version=ast_grep_version, qmd_version=qmd_version)
2036
2557
  old_stats = metadata.get("stats") if isinstance(metadata.get("stats"), dict) else {}
2037
- kept = {k: v for k, v in old_stats.items() if k not in STATS_HELPER_FIELDS and k not in stats["stats"]}
2038
- out["stats"] = {**stats["stats"], **kept}
2558
+ out["stats"], dropped, stats_warnings = stats_after(old_stats, stats["stats"], metadata.get("scope_type"))
2559
+ warnings += stats_warnings
2039
2560
  out["confidence_distribution"] = stats["confidence_distribution"]
2040
- return out
2561
+ return out, {"dropped_stats": dropped, "warnings": warnings}
2041
2562
 
2042
2563
 
2043
2564
  NOT_PUBLIC_NAMED = 10 # the names the not_public warning spells out; the summary lists them all
@@ -2177,6 +2698,20 @@ def build_records(extraction: dict | None, details: dict | None, files: list | N
2177
2698
  if patch.get(field) is not None:
2178
2699
  seeded[key][field] = patch[field]
2179
2700
  fill_if_null(key, patch)
2701
+ diff = extraction.get("entry_point_diff") if isinstance(extraction, dict) else None
2702
+ aliases: dict[tuple, set] = {}
2703
+ rows = [*(diff.get("public") or []), *(diff.get("other_definitions") or [])] if isinstance(diff, dict) else []
2704
+ for item in rows:
2705
+ members = item.get("members") if isinstance(item, dict) and item.get("via") == "namespace" else None
2706
+ for named in [item, *(members if isinstance(members, list) else [])]:
2707
+ if not isinstance(named, dict):
2708
+ continue
2709
+ name, local = named.get("name"), named.get("local")
2710
+ key = (local, _norm_path(named.get("file")))
2711
+ if isinstance(name, str) and name and isinstance(local, str) and local != name and key in seeded:
2712
+ aliases.setdefault(key, set()).add(name) # the public name the runner gives `local` in that file
2713
+ for key, names in aliases.items():
2714
+ seeded[key]["aliases"] = sorted(names)
2180
2715
  surface, _problem = public_surface(extraction) # apply warns when a public-api run gives no surface
2181
2716
  for key in seeded if surface is not None else ():
2182
2717
  mark = surface.public(key[0], key[1], languages.get(key))
@@ -2217,6 +2752,26 @@ DRIFT_IN_MAP = "a signature from the tree"
2217
2752
  REACHABILITY = ("public", "internal-unreachable")
2218
2753
  VERIFIER_MISSING = "provenance: spot-checks not run: skf-verify-provenance-completeness.py is missing; re-install SKF"
2219
2754
  OUTCOMES = ("verified", "moved", "missing", "re-extracted", "rescoped", "unknown")
2755
+ # Why an entry ends `unknown` (its unknown_reason, see "Gap records"), with the words apply's warning gives it.
2756
+ UNKNOWN_REASONS = {
2757
+ "ambiguous": "several map entries of its name",
2758
+ "no-definition": "no line of its file defines it, by the definition-line rules",
2759
+ "several-definitions": "several lines of its file define it, none the recorded one",
2760
+ "unchecked-language": "the definition-line rules cover no language of its file, and no line was answered as "
2761
+ "declaring it",
2762
+ "no-line": "its map entry gives no file or line to check",
2763
+ "spot-check-failed": "the spot-check failed",
2764
+ "verifier-missing": "skf-verify-provenance-completeness.py is missing",
2765
+ "no-source-root": "no source tree to check",
2766
+ "drift-override": "the drift override kept its line from being moved",
2767
+ "cited-elsewhere": "the gap cites another file than the provenance map's entries of its name",
2768
+ }
2769
+ # A cited-elsewhere entry's citation_check that read nothing at the cited file, with the fix that lets a re-run read it.
2770
+ UNREAD = {
2771
+ "no-source-root": "give update-skill the source tree",
2772
+ "verifier-missing": "re-install SKF",
2773
+ "spot-check-failed": "fix what its spot-check failed on, which the `spot-check failed` warning names",
2774
+ }
2220
2775
  _DEFINITION_LINES = re.compile(r"definition lines?\s*\(([\d,\s]+)\)", re.IGNORECASE)
2221
2776
  _VERIFIER = None
2222
2777
 
@@ -2268,40 +2823,48 @@ def _drift_reason(entry: dict) -> str | None:
2268
2823
  return None
2269
2824
  if entry.get("provenance_completeness"):
2270
2825
  return DRIFT_COMPLETENESS
2271
- status = (entry.get("map_match") or {}).get("status")
2272
- return DRIFT_IN_MAP if status in ("found", "ambiguous") else DRIFT_NOT_IN_MAP
2826
+ match = entry.get("map_match") if isinstance(entry.get("map_match"), dict) else {}
2827
+ status = match.get("status")
2828
+ # a not-found with a candidate that has a file is a citation of another file than the map's entries: held (#824)
2829
+ held = status in ("found", "ambiguous") or status == "not-found" and any(
2830
+ isinstance(c, dict) and _norm_path(c.get("source_file")) for c in match.get("candidates") or [])
2831
+ return DRIFT_IN_MAP if held else DRIFT_NOT_IN_MAP
2273
2832
 
2274
2833
 
2275
2834
  def _spot_check(root: Path | None, path: str | None, name: str, line: int | None, export_type: object,
2276
- answers: dict) -> tuple[str, int | None, bool, str | None]:
2835
+ answers: dict) -> tuple[str, int | None, bool, str | None, str | None]:
2277
2836
  """(outcome, the line a `moved` export moves to, whether it waits on a `declaring_line` answer, why the
2278
- check failed)."""
2837
+ check failed, why it is `unknown`: an UNKNOWN_REASONS key)."""
2838
+ if root is None:
2839
+ return "unknown", None, False, None, "no-source-root"
2279
2840
  verifier = _verifier()
2280
- if verifier is None or root is None or not path or line is None:
2281
- return "unknown", None, False, None
2841
+ if verifier is None:
2842
+ return "unknown", None, False, None, "verifier-missing"
2843
+ if not path or line is None:
2844
+ return "unknown", None, False, None, "no-line"
2282
2845
  try:
2283
2846
  report = verifier.definition_lines_report(path, name, root, line,
2284
2847
  export_type if isinstance(export_type, str) else None)
2285
2848
  except Exception as exc: # noqa: BLE001 - a file it cannot read, or any verifier failure: this entry only
2286
- return "unknown", None, False, f"{type(exc).__name__}: {exc}"
2849
+ return "unknown", None, False, f"{type(exc).__name__}: {exc}", "spot-check-failed"
2287
2850
  check = report.get("line_check")
2288
2851
  if check == "file-missing":
2289
- return "missing", None, False, None
2852
+ return "missing", None, False, None, None
2290
2853
  if check == "skipped-export-type":
2291
- return "verified", None, False, None
2854
+ return "verified", None, False, None, None
2292
2855
  if check == "skipped-language":
2293
2856
  if "declaring_line" not in answers:
2294
- return "unknown", None, True, None
2857
+ return "unknown", None, True, None, None
2295
2858
  declared = answers["declaring_line"]
2296
2859
  if declared is None:
2297
- return "unknown", None, False, None
2298
- return ("verified", None, False, None) if declared == line else ("moved", declared, False, None)
2860
+ return "unknown", None, False, None, "unchecked-language"
2861
+ return ("verified", None, False, None, None) if declared == line else ("moved", declared, False, None, None)
2299
2862
  if report.get("line_is_definition"):
2300
- return "verified", None, False, None
2863
+ return "verified", None, False, None, None
2301
2864
  defs = report.get("definition_lines") or []
2302
2865
  if len(defs) == 1 and defs[0] != line:
2303
- return "moved", defs[0], False, None
2304
- return "unknown", None, False, None
2866
+ return "moved", defs[0], False, None, None
2867
+ return "unknown", None, False, None, "several-definitions" if defs else "no-definition"
2305
2868
 
2306
2869
 
2307
2870
  def _pinned_lines(remediation: object) -> list[int] | None:
@@ -2311,6 +2874,18 @@ def _pinned_lines(remediation: object) -> list[int] | None:
2311
2874
  return lines or None
2312
2875
 
2313
2876
 
2877
+ def _inside(root: Path | None, path: str | None) -> bool:
2878
+ """True for a relative path that stays under the source root, links resolved."""
2879
+ rel = Path(path) if path else None
2880
+ if root is None or rel is None or rel.is_absolute() or ".." in rel.parts:
2881
+ return False
2882
+ try:
2883
+ (root / rel).resolve().relative_to(root.resolve())
2884
+ except (OSError, ValueError):
2885
+ return False
2886
+ return True
2887
+
2888
+
2314
2889
  def _where(path: object, line: object) -> str | None:
2315
2890
  file, number = _norm_path(path), _line_of(line)
2316
2891
  return f"{file}:{number}" if file and number is not None else None
@@ -2330,7 +2905,14 @@ class _GapEntry:
2330
2905
  self.map_entry = {"source_file": found.get("source_file"), "source_line": found.get("source_line")} \
2331
2906
  if found else None
2332
2907
  self.map_export_type = found.get("export_type") if found else None
2908
+ # the map entry's own name when skf-parse-gaps matched it by its reexported_as (the gap names the public one)
2909
+ self.declared = _name_of(found) if found and _name_of(found) else self.name
2333
2910
  self.in_map = bool(match.get("candidates"))
2911
+ # where map_match's candidates sit, `file:line` (or the file); a placeholder with no file names none
2912
+ self.map_locations = list(dict.fromkeys(
2913
+ _where(c.get("source_file"), c.get("source_line")) or _norm_path(c.get("source_file"))
2914
+ for c in match.get("candidates") or [] if isinstance(c, dict) and _norm_path(c.get("source_file"))))
2915
+ self.citation_check: str | None = None # a cited-elsewhere entry's citation check: missing, or its reason
2334
2916
  citation = entry.get("source_citation")
2335
2917
  self.citation = citation if isinstance(citation, dict) else None
2336
2918
  self.recorded: tuple[str | None, int | None] = (None, None) # the file and line a check read
@@ -2338,8 +2920,9 @@ class _GapEntry:
2338
2920
  self.new_line: int | None = None
2339
2921
  self.unknown_reason: str | None = None
2340
2922
  self.reextract = False
2341
- self.record: dict | None = None # the §4a record it matched
2923
+ self.record: dict | None = None # the §4a record it matched, or took its signature from
2342
2924
  self.record_path: str | None = None
2925
+ self.signature_at: tuple[str, int] | None = None # where a check located it: its signature's record
2343
2926
  self.waits = False # on a declaring_line answer
2344
2927
  self.failure: str | None = None # why its spot-check failed
2345
2928
 
@@ -2353,8 +2936,25 @@ class _GapEntry:
2353
2936
 
2354
2937
  def check(self, root: Path | None, path: object, line: object, export_type: object) -> None:
2355
2938
  self.recorded = (_norm_path(path), _line_of(line))
2356
- self.outcome, self.new_line, self.waits, self.failure = _spot_check(
2357
- root, self.recorded[0], self.name, self.recorded[1], export_type, self.answers)
2939
+ self.outcome, self.new_line, self.waits, self.failure, self.unknown_reason = _spot_check(
2940
+ root, self.recorded[0], self.declared, self.recorded[1], export_type, self.answers)
2941
+
2942
+ def locate(self, root: Path | None) -> None:
2943
+ """Record where a check located the export (the checked file, at the recorded line or the one it moved
2944
+ to), so its signature comes from §4a's record there; never a file outside the source root."""
2945
+ line = self.new_line if self.outcome == "moved" else self.recorded[1]
2946
+ if self.outcome in ("verified", "moved") and line is not None and _inside(root, self.recorded[0]):
2947
+ self.signature_at = (self.recorded[0], line)
2948
+
2949
+ def take_signature(self, blocks: dict[str, list[dict]]) -> None:
2950
+ """Take the one record of the entry's name in the block of the file it was located in that is at its line,
2951
+ else the one with no line; none otherwise."""
2952
+ path, line = self.signature_at
2953
+ named = [r for r in blocks.get(path, []) if _name_of(r) in (self.name, self.declared)]
2954
+ at = [r for r in named if _line_of(r.get("location")) == line] \
2955
+ or [r for r in named if _line_of(r.get("location")) is None]
2956
+ if len(at) == 1:
2957
+ self.record, self.record_path = at[0], path
2358
2958
 
2359
2959
  def route(self, root: Path | None, drift: bool) -> None:
2360
2960
  """Spot-check the entry, or send it to targeted re-extraction (see "Gap records" in the module docstring)."""
@@ -2373,14 +2973,23 @@ class _GapEntry:
2373
2973
  return
2374
2974
  if self.match_status == "found":
2375
2975
  self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
2976
+ self.locate(root)
2376
2977
  return
2377
- if self.match_status == "ambiguous":
2378
- self.outcome = "unknown"
2978
+ if self.match_status == "ambiguous": # several map entries of its name: no check, apply warns
2979
+ self.outcome, self.unknown_reason = "unknown", "ambiguous"
2379
2980
  return
2380
2981
  if self.citation:
2381
2982
  self.check(root, self.citation.get("file"), self.citation.get("line"), None)
2382
2983
  if self.waits or self.outcome in ("verified", "moved"):
2984
+ if not self.in_map: # a citation of another file than the map's entries of its name takes none
2985
+ self.locate(root)
2383
2986
  return
2987
+ if self.match_status == "not-found" and self.map_locations:
2988
+ # the map holds its name at other files: the cited file's declaration may be another package's, so it is
2989
+ # never re-extracted there (#824); the record keeps the check's cause, and apply warns with what to fix
2990
+ self.citation_check = "missing" if self.outcome == "missing" else self.unknown_reason
2991
+ self.outcome, self.unknown_reason = "unknown", "cited-elsewhere"
2992
+ return
2384
2993
  if self.entry.get("provenance_completeness") and root is not None:
2385
2994
  self.reextract = True
2386
2995
  elif self.blocking:
@@ -2391,14 +3000,20 @@ class _GapEntry:
2391
3000
  self.outcome = "unknown"
2392
3001
 
2393
3002
  def match(self, blocks: dict[str, list[dict]]) -> tuple[int, int]:
2394
- """Take the first export of the entry's name in the blocks of its resolved files; (files scanned, exports
2395
- found in them)."""
3003
+ """Take the first export of the entry's name in the blocks of its resolved files, else the first whose
3004
+ `aliases` hold its name (the public name the runner gives its local name in that file); (files scanned,
3005
+ exports found in them)."""
2396
3006
  exports = 0
3007
+ aliased: tuple[dict, str] | None = None
2397
3008
  for path in self.paths:
2398
3009
  for record in blocks.get(path, []):
2399
3010
  exports += 1
2400
3011
  if self.record is None and _name_of(record) == self.name:
2401
3012
  self.record, self.record_path = record, path
3013
+ elif aliased is None and isinstance(record.get("aliases"), list) and self.name in record["aliases"]:
3014
+ aliased = (record, path)
3015
+ if self.record is None and aliased is not None:
3016
+ self.record, self.record_path = aliased
2402
3017
  if self.record is not None:
2403
3018
  self.outcome = "re-extracted"
2404
3019
  return len(self.paths), exports
@@ -2427,8 +3042,14 @@ class _GapEntry:
2427
3042
  return (self.record or {}).get("location")
2428
3043
  return None
2429
3044
 
3045
+ def signature(self) -> dict | None:
3046
+ """The record a located export takes its signature from, or None."""
3047
+ return self.record if self.signature_at and not self.internal else None
3048
+
2430
3049
  def verification(self) -> dict:
3050
+ signed = self.signature() or {}
2431
3051
  recorded = _where(*self.recorded)
3052
+ elsewhere = self.outcome == "unknown" and self.unknown_reason == "cited-elsewhere"
2432
3053
  if self.outcome == "re-extracted":
2433
3054
  citation = self.new_location()
2434
3055
  elif self.outcome in ("verified", "moved", "missing", "rescoped") and recorded:
@@ -2445,13 +3066,19 @@ class _GapEntry:
2445
3066
  "provenance_citation": citation,
2446
3067
  "source_citation": self.citation,
2447
3068
  "new_location": self.new_location(),
2448
- "unknown_reason": self.unknown_reason,
3069
+ "unknown_reason": self.unknown_reason if self.outcome == "unknown" else None,
3070
+ "citation_check": self.citation_check if elsewhere else None,
3071
+ "map_locations": self.map_locations if elsewhere else None,
2449
3072
  "pinned_definition_lines": _pinned_lines(self.entry.get("remediation"))
2450
3073
  if self.category == "MOVED_EXPORT" else None,
2451
3074
  "resolution_source": "remediation-paths" if self.outcome == "re-extracted" else None,
2452
3075
  "reachability": self.answers.get("reachability") if self.category == "NEW_EXPORT" and self.located
2453
3076
  else None,
2454
- "export_type": None,
3077
+ "export_type": signed.get("type", signed.get("export_type")),
3078
+ "params": signed.get("params", signed.get("parameters")),
3079
+ "return_type": signed.get("return_type"),
3080
+ "extraction_method": signed.get("extraction_method"),
3081
+ "ast_node_type": signed.get("ast_node_type"),
2455
3082
  }
2456
3083
 
2457
3084
  def label(self, provenance: list[dict]) -> str:
@@ -2464,7 +3091,7 @@ class _GapEntry:
2464
3091
  elif self.map_entry is not None:
2465
3092
  file, line = _norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"])
2466
3093
  for entry in provenance:
2467
- if (_name_of(entry) == self.name and _file_of(entry) == file
3094
+ if (_name_of(entry) == self.declared and _file_of(entry) == file
2468
3095
  and _line_of(entry.get("source_line")) == line):
2469
3096
  method = entry.get("extraction_method")
2470
3097
  break
@@ -2513,17 +3140,22 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
2513
3140
  if waiting:
2514
3141
  return None, {"status": "needs-judgment", "needs_judgment": waiting, "warnings": warnings}
2515
3142
  to_scan = [g for g in routed if g.reextract]
2516
- files = list(dict.fromkeys(p for g in to_scan for p in g.paths))
3143
+ located = [g for g in routed if g.signature_at and not g.internal] # an internal export takes no signature
3144
+ files = list(dict.fromkeys([p for g in to_scan for p in g.paths] + [g.signature_at[0] for g in located]))
2517
3145
  if plan:
2518
3146
  return None, {"status": "planned", "reextract": {
2519
3147
  "entries": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
2520
- "resolved_paths": g.paths} for g in to_scan], "files": files}, "warnings": warnings}
3148
+ "resolved_paths": g.paths} for g in to_scan], "files": files,
3149
+ "signatures": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "file": g.signature_at[0],
3150
+ "line": g.signature_at[1]} for g in located]}, "warnings": warnings}
2521
3151
  blocks: dict[str, list[dict]] = {}
2522
3152
  scanned = 0
3153
+ targeted_files = {p for g in to_scan for p in g.paths}
3154
+ signature_files = {g.signature_at[0] for g in located}
2523
3155
  if files:
2524
3156
  if remediation is None:
2525
- raise ValueError("targeted re-extraction has files to match: pass --remediation-records, the output "
2526
- "of `records` over them")
3157
+ raise ValueError("gap-records has files to match (targeted re-extraction, or a located export's "
3158
+ "signature): pass --remediation-records, the output of `records` over them")
2527
3159
  if not isinstance(remediation, dict) or not isinstance(remediation.get("files"), list):
2528
3160
  raise ValueError("--remediation-records has no `files` array: is it the output of `records`?")
2529
3161
  for block in remediation["files"]:
@@ -2545,21 +3177,36 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
2545
3177
  "files_scanned": files_scanned, "exports_found_in_scan": found})
2546
3178
  if unresolved:
2547
3179
  return None, {"status": "unresolved", "unresolved": unresolved, "warnings": warnings}
3180
+ for g in located:
3181
+ g.take_signature(blocks)
2548
3182
  needs = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": g.needs(),
2549
3183
  "file": _location(g.new_location())[0] if g.outcome != "verified" else g.recorded[0],
2550
3184
  "line": _location(g.new_location())[1] if g.outcome != "verified" else g.recorded[1]}
2551
3185
  for g in routed if g.needs()]
2552
3186
  if needs:
2553
3187
  return None, {"status": "needs-judgment", "needs_judgment": needs, "warnings": warnings}
3188
+ warnings += [f"provenance: {g.name}: located at {g.signature_at[0]}:{g.signature_at[1]}, but no record of it "
3189
+ f"there; signature not repaired" for g in located if g.record is None]
2554
3190
  kept: dict[str, list[dict]] = {}
2555
3191
  for g in routed:
2556
- if g.record is not None and not g.internal:
2557
- record = dict(g.record)
2558
- record.pop("public", None) # a repair documents and maps every record: the mark is a normal update's
2559
- if g.answers.get("docstring") is not None:
2560
- record["docstring"] = g.answers["docstring"]
2561
- if record not in kept.setdefault(g.record_path, []):
2562
- kept[g.record_path].append(record)
3192
+ if g.record is None or g.internal:
3193
+ continue
3194
+ record = dict(g.record)
3195
+ record.pop("public", None) # a repair documents and maps every record: the mark is a normal update's
3196
+ record.pop("aliases", None) # the match is made: the copy names the one it took
3197
+ if _name_of(record) != g.name: # matched by its public name: documented under it, its own name as `local`
3198
+ record = {"name": g.name, "local": _name_of(record),
3199
+ **{k: v for k, v in record.items() if k not in ("name", "export_name", "local")}}
3200
+ if g.outcome == "re-extracted" and g.answers.get("docstring") is not None:
3201
+ record["docstring"] = g.answers["docstring"] # a located export's signature record asks none
3202
+ block = kept.setdefault(g.record_path, [])
3203
+ location = record.get("location")
3204
+ same = next((r for r in block if (_name_of(r) == _name_of(record) and r.get("location") == location
3205
+ if location is not None else r == record)), None)
3206
+ if same is None:
3207
+ block.append(record)
3208
+ elif same.get("docstring") is None and record.get("docstring") is not None:
3209
+ same["docstring"] = record["docstring"]
2563
3210
  breakdown = {"T1": 0, "T1-low": 0, "unlabeled": 0, "T2": 0}
2564
3211
  provenance_entries = _entries_of(provenance)
2565
3212
  for g in routed:
@@ -2570,13 +3217,15 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
2570
3217
  "files_extracted": scanned,
2571
3218
  "exports_extracted": len(entries),
2572
3219
  "confidence_breakdown": breakdown,
2573
- "verification": [g.verification() for g in routed],
3220
+ "verification": [{"gap_id": g.entry.get("gap_id"), **g.verification()} for g in routed],
2574
3221
  "files": [{"file_path": path, "exports": exports} for path, exports in kept.items()],
2575
3222
  }
2576
3223
  matched = [g for g in to_scan if g.record is not None]
2577
- targeted = {"resolved_count": len(matched), "files_scanned": scanned,
3224
+ scanned_paths = set(blocks)
3225
+ targeted = {"resolved_count": len(matched), "files_scanned": len(scanned_paths & targeted_files),
3226
+ "signature_files_scanned": len(scanned_paths & signature_files),
2578
3227
  "exports_matched": len({(g.record_path, _name_of(g.record)) for g in matched}),
2579
- "tier": tier} if to_scan else None
3228
+ "tier": tier} if to_scan or located else None
2580
3229
  summary = {
2581
3230
  "status": "written",
2582
3231
  "gap_count": len(entries),
@@ -2766,13 +3415,62 @@ def _metadata_in(path: str | None) -> dict | None:
2766
3415
  return metadata
2767
3416
 
2768
3417
 
3418
+ _QUICK_METADATA = None
3419
+
3420
+
3421
+ def _quick_metadata():
3422
+ """skf-render-quick-metadata.py, loaded once from this folder as skf-extraction-inventory.py loads it: its
3423
+ probe_skf_version reads the installed SKF's version, so create-skill and update-skill read it one way. Raises
3424
+ ValueError, one line, when it cannot be loaded."""
3425
+ global _QUICK_METADATA
3426
+ if _QUICK_METADATA is None:
3427
+ path = Path(__file__).resolve().parent / "skf-render-quick-metadata.py"
3428
+ cannot = f"cannot load {path.name} beside {Path(__file__).name}"
3429
+ if not path.is_file():
3430
+ raise ValueError(f"{cannot}: it is missing; re-install SKF")
3431
+ spec = importlib.util.spec_from_file_location("skf_render_quick_metadata", path)
3432
+ if spec is None or spec.loader is None:
3433
+ raise ValueError(f"{cannot}: re-install SKF")
3434
+ module = importlib.util.module_from_spec(spec)
3435
+ try:
3436
+ spec.loader.exec_module(module)
3437
+ except (ImportError, SyntaxError, OSError) as exc:
3438
+ raise ValueError(f"{cannot} ({type(exc).__name__}: {' '.join(str(exc).split())}): re-install "
3439
+ f"SKF") from exc
3440
+ if not callable(getattr(module, "probe_skf_version", None)):
3441
+ raise ValueError(f"{cannot}: it has no probe_skf_version; re-install SKF")
3442
+ _QUICK_METADATA = module
3443
+ return _QUICK_METADATA
3444
+
3445
+
3446
+ def _runner_ast_grep(path: str | None) -> str | None:
3447
+ """The ast-grep version the recipe runner's output (--extraction) records: "unknown" when it records none but
3448
+ holds an export ast-grep matched, and None when ast-grep did not run (no export it matched, as in the
3449
+ {"exports": []} step 2 writes when the runner cannot run)."""
3450
+ if not path:
3451
+ return None
3452
+ extraction = _load_json_file(Path(path), "--extraction file")
3453
+ if not isinstance(extraction, dict):
3454
+ raise ValueError("--extraction file must hold a JSON object (the recipe runner's output)")
3455
+ ast_grep = extraction.get("ast_grep")
3456
+ version = _tool_version(ast_grep.get("version")) if isinstance(ast_grep, dict) else None
3457
+ if version is not None:
3458
+ return version
3459
+ exports = extraction.get("exports") if isinstance(extraction.get("exports"), list) else []
3460
+ matched = any(isinstance(e, dict) and e.get("extraction_method") == "ast-grep" for e in exports)
3461
+ return "unknown" if matched else None
3462
+
3463
+
2769
3464
  def _cmd_metadata(args: argparse.Namespace) -> int:
2770
3465
  target = Path(args.metadata)
2771
3466
  try:
2772
- metadata = update_metadata(
3467
+ skf_version = _quick_metadata().probe_skf_version(args.skf_root) if args.skf_root else None
3468
+ metadata, summary = update_metadata(
2773
3469
  _load_json_file(target, "--metadata file"), _load_json_file(Path(args.lists), "--lists file"),
2774
3470
  _load_json_file(Path(args.stats), "--stats file"), version=args.version,
2775
- generation_date=args.generation_date, source_commit=args.source_commit, source_ref=args.source_ref)
3471
+ generation_date=args.generation_date, source_commit=args.source_commit, source_ref=args.source_ref,
3472
+ skf_version=skf_version, skf_root=args.skf_root, ast_grep_version=_runner_ast_grep(args.extraction),
3473
+ qmd_version=args.qmd_version)
2776
3474
  _write_json_atomic(target, metadata)
2777
3475
  except ValueError as exc:
2778
3476
  print(f"error: {exc}", file=sys.stderr)
@@ -2782,7 +3480,8 @@ def _cmd_metadata(args: argparse.Namespace) -> int:
2782
3480
  return 1
2783
3481
  print(json.dumps({"status": "written", "metadata": target.as_posix(), "version": metadata["version"],
2784
3482
  "exports": len(metadata["exports"]), "scripts": len(metadata.get("scripts") or []),
2785
- "assets": len(metadata.get("assets") or [])}))
3483
+ "assets": len(metadata.get("assets") or []), "tool_versions": metadata["tool_versions"],
3484
+ **summary}))
2786
3485
  return 0
2787
3486
 
2788
3487
 
@@ -3061,9 +3760,15 @@ def _build_parser() -> argparse.ArgumentParser:
3061
3760
  description="Rewrite the skill's metadata.json in place. It sets version and generation_date, "
3062
3761
  "source_commit and source_ref when given, exports, scripts and assets from --lists (a script "
3063
3762
  "or asset list left out when empty), confidence_distribution from --stats and the stats "
3064
- "helper's stats fields, dropping one an earlier run wrote that --stats lacks; it removes "
3065
- "last_update and update_type and keeps every other field, a patched one included. It refuses "
3066
- "a --stats whose scripts_count or assets_count is not the length of --lists' scripts or assets.")
3763
+ "helper's stats fields, dropping one an earlier run wrote that --stats lacks; tool_versions.skf "
3764
+ "from --skf-root (unknown, with a warning, when it gives no version), and tool_versions.ast_grep "
3765
+ "and tool_versions.qmd from --extraction and --qmd-version when this run used them, else as "
3766
+ "they stand. When --stats' effective_denominator differs from the old one (or one is absent), it "
3767
+ "drops each old stats key named effective_denominator_*; each dropped key is in dropped_stats, "
3768
+ "with a warning. It keeps every other old stats key, with a warning for each one create-skill "
3769
+ "does not write for the skill's scope_type. It removes last_update and "
3770
+ "update_type and keeps every other field, a patched one included. It refuses a --stats whose "
3771
+ "scripts_count or assets_count is not the length of --lists' scripts or assets.")
3067
3772
  p_meta.add_argument("--metadata", required=True, metavar="FILE",
3068
3773
  help="the skill's metadata.json, rewritten in place")
3069
3774
  p_meta.add_argument("--lists", required=True, metavar="FILE", help="the lists apply wrote (--lists-out)")
@@ -3073,6 +3778,14 @@ def _build_parser() -> argparse.ArgumentParser:
3073
3778
  p_meta.add_argument("--generation-date", required=True, help="the clock's UTC time, YYYY-MM-DDTHH:MM:SSZ")
3074
3779
  p_meta.add_argument("--source-commit", default=None, help="the commit this update read (an empty one: null)")
3075
3780
  p_meta.add_argument("--source-ref", default=None, help="the ref this update read (an empty one: null)")
3781
+ p_meta.add_argument("--skf-root", metavar="DIR",
3782
+ help="the installed SKF ({project-root}/_bmad/skf): tool_versions.skf, from its package.json "
3783
+ "or VERSION file")
3784
+ p_meta.add_argument("--extraction", metavar="FILE",
3785
+ help="the recipe runner's output this run wrote: tool_versions.ast_grep, its ast_grep.version")
3786
+ p_meta.add_argument("--qmd-version", default=None,
3787
+ help="what qmd --version printed, when this run queried QMD: tool_versions.qmd (a leading "
3788
+ "\"qmd \" removed)")
3076
3789
  p_meta.set_defaults(func=_cmd_metadata)
3077
3790
 
3078
3791
  p_records = sub.add_parser("records", help="write step 3's re-extraction records from the runner's and the "
@@ -3094,9 +3807,9 @@ def _build_parser() -> argparse.ArgumentParser:
3094
3807
  help="the workspace drift check's status; overridden runs the drift gate")
3095
3808
  p_gap.add_argument("--judgments", metavar="FILE", help="the answers to its needs_judgment[], keyed by gap id")
3096
3809
  p_gap.add_argument("--remediation-records", metavar="FILE",
3097
- help="`records` output over the files targeted re-extraction scanned")
3810
+ help="`records` output over the files the plan listed")
3098
3811
  p_gap.add_argument("--plan", action="store_true",
3099
- help="print the answers it needs and the files targeted re-extraction scans; write no records")
3812
+ help="print the answers it needs and the files §4a's extraction scans; write no records")
3100
3813
  p_gap.add_argument("--files-out", metavar="FILE", help="with --plan: write those files here, a JSON array")
3101
3814
  p_gap.add_argument("--evidence", metavar="FILE",
3102
3815
  help="append the targeted_reextraction record to this JSON-lines file when it ran")