bmad-module-skill-forge 3.0.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +3 -3
  3. package/docs/_data/pinned.yaml +19 -13
  4. package/docs/architecture.md +2 -2
  5. package/docs/concepts.md +4 -4
  6. package/docs/examples.md +33 -26
  7. package/docs/getting-started.md +3 -3
  8. package/docs/how-it-works.md +2 -2
  9. package/docs/index.md +1 -1
  10. package/docs/skill-model.md +41 -39
  11. package/docs/troubleshooting.md +8 -0
  12. package/docs/verifying-a-skill.md +46 -21
  13. package/docs/why-skf.md +1 -1
  14. package/docs/workflows.md +5 -5
  15. package/package.json +1 -1
  16. package/src/knowledge/qmd-registry.md +1 -1
  17. package/src/knowledge/version-paths.md +1 -1
  18. package/src/shared/health-check.md +2 -50
  19. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
  20. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +12 -4
  21. package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
  22. package/src/shared/scripts/skf-atomic-write.py +212 -59
  23. package/src/shared/scripts/skf-build-change-manifest.py +1437 -178
  24. package/src/shared/scripts/skf-classify-changed-files.py +78 -0
  25. package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
  26. package/src/shared/scripts/skf-count-tokens.py +54 -10
  27. package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
  28. package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
  29. package/src/shared/scripts/skf-detect-workspaces.py +67 -0
  30. package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
  31. package/src/shared/scripts/skf-extract-public-api.py +1673 -189
  32. package/src/shared/scripts/skf-extraction-inventory.py +584 -46
  33. package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
  34. package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
  35. package/src/shared/scripts/skf-github-probe.py +47 -8
  36. package/src/shared/scripts/skf-hash-content.py +14 -8
  37. package/src/shared/scripts/skf-load-provenance.py +3 -1
  38. package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
  39. package/src/shared/scripts/skf-parse-gaps.py +135 -26
  40. package/src/shared/scripts/skf-preflight.py +15 -1
  41. package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
  42. package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
  43. package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
  44. package/src/shared/scripts/skf-resolve-customization.py +197 -0
  45. package/src/shared/scripts/skf-shard-body.py +239 -41
  46. package/src/shared/scripts/skf-structural-diff.py +133 -18
  47. package/src/shared/scripts/skf-update-active-symlink.py +115 -62
  48. package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
  49. package/src/shared/scripts/skf-validate-output.py +12 -3
  50. package/src/shared/scripts/skf-verify-provenance-completeness.py +1439 -79
  51. package/src/shared/scripts/skf-write-skill-brief.py +41 -8
  52. package/src/shared/tool-requirements.yaml +8 -2
  53. package/src/skf-analyze-source/SKILL.md +2 -2
  54. package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
  55. package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
  56. package/src/skf-audit-skill/SKILL.md +2 -2
  57. package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
  58. package/src/skf-audit-skill/references/headless-contract.md +1 -1
  59. package/src/skf-audit-skill/references/health-check.md +2 -0
  60. package/src/skf-audit-skill/references/init.md +2 -1
  61. package/src/skf-audit-skill/references/re-index.md +14 -9
  62. package/src/skf-audit-skill/references/report.md +2 -2
  63. package/src/skf-audit-skill/references/semantic-diff.md +15 -15
  64. package/src/skf-audit-skill/references/structural-diff.md +10 -5
  65. package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
  66. package/src/skf-brief-skill/SKILL.md +3 -3
  67. package/src/skf-brief-skill/assets/scope-templates.md +1 -1
  68. package/src/skf-brief-skill/assets/skill-brief-schema.md +4 -11
  69. package/src/skf-brief-skill/references/analyze-target.md +6 -6
  70. package/src/skf-brief-skill/references/confirm-brief.md +1 -1
  71. package/src/skf-brief-skill/references/scope-definition.md +4 -4
  72. package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
  73. package/src/skf-brief-skill/references/version-resolution.md +5 -9
  74. package/src/skf-campaign/SKILL.md +2 -2
  75. package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
  76. package/src/skf-create-skill/SKILL.md +2 -2
  77. package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
  78. package/src/skf-create-skill/assets/skill-sections.md +7 -7
  79. package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
  80. package/src/skf-create-skill/references/compile.md +31 -60
  81. package/src/skf-create-skill/references/component-extraction.md +3 -3
  82. package/src/skf-create-skill/references/enrich.md +2 -2
  83. package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
  84. package/src/skf-create-skill/references/extract.md +23 -19
  85. package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
  86. package/src/skf-create-skill/references/extraction-patterns.md +10 -11
  87. package/src/skf-create-skill/references/generate-artifacts.md +1 -1
  88. package/src/skf-create-skill/references/report.md +21 -5
  89. package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
  90. package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
  91. package/src/skf-create-stack-skill/SKILL.md +2 -2
  92. package/src/skf-create-stack-skill/references/validate.md +1 -1
  93. package/src/skf-drop-skill/SKILL.md +2 -2
  94. package/src/skf-drop-skill/references/execute.md +5 -1
  95. package/src/skf-drop-skill/references/health-check.md +2 -0
  96. package/src/skf-drop-skill/references/invocation-contract.md +1 -1
  97. package/src/skf-drop-skill/references/report.md +2 -2
  98. package/src/skf-drop-skill/references/select.md +7 -3
  99. package/src/skf-export-skill/SKILL.md +2 -2
  100. package/src/skf-export-skill/assets/snippet-format.md +5 -5
  101. package/src/skf-export-skill/references/generate-snippet.md +22 -15
  102. package/src/skf-export-skill/references/load-skill.md +1 -1
  103. package/src/skf-export-skill/references/token-report.md +29 -9
  104. package/src/skf-forger/SKILL.md +2 -2
  105. package/src/skf-quick-skill/SKILL.md +2 -2
  106. package/src/skf-refine-architecture/SKILL.md +6 -8
  107. package/src/skf-rename-skill/SKILL.md +2 -2
  108. package/src/skf-setup/SKILL.md +2 -2
  109. package/src/skf-test-skill/SKILL.md +2 -2
  110. package/src/skf-test-skill/assets/output-section-formats.md +6 -4
  111. package/src/skf-test-skill/references/coherence-check.md +5 -4
  112. package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
  113. package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
  114. package/src/skf-test-skill/references/coverage-check.md +69 -111
  115. package/src/skf-test-skill/references/init.md +5 -6
  116. package/src/skf-test-skill/references/migration-section-rules.md +26 -14
  117. package/src/skf-test-skill/references/report.md +11 -9
  118. package/src/skf-test-skill/references/scoring-rules.md +8 -6
  119. package/src/skf-test-skill/references/source-access-protocol.md +15 -18
  120. package/src/skf-test-skill/scripts/build-result-context.py +114 -21
  121. package/src/skf-test-skill/scripts/gap-ledger.py +580 -80
  122. package/src/skf-test-skill/scripts/load-coverage-inputs.py +646 -107
  123. package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
  124. package/src/skf-test-skill/scripts/reconcile-coverage.py +450 -23
  125. package/src/skf-test-skill/scripts/score-signatures.py +356 -61
  126. package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
  127. package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
  128. package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
  129. package/src/skf-test-skill/templates/test-report-template.md +1 -0
  130. package/src/skf-update-skill/SKILL.md +2 -2
  131. package/src/skf-update-skill/references/detect-changes.md +32 -13
  132. package/src/skf-update-skill/references/gap-driven.md +37 -22
  133. package/src/skf-update-skill/references/init.md +6 -2
  134. package/src/skf-update-skill/references/invocation-contract.md +1 -1
  135. package/src/skf-update-skill/references/merge.md +17 -12
  136. package/src/skf-update-skill/references/re-extract.md +4 -2
  137. package/src/skf-update-skill/references/report.md +7 -4
  138. package/src/skf-update-skill/references/write.md +77 -36
  139. package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
  140. package/src/skf-verify-stack/SKILL.md +2 -2
  141. package/tools/cli/lib/tool-check.js +158 -2
@@ -55,16 +55,30 @@ Subcommands:
55
55
  apply --update-type incremental|gap-driven|full --skill-name <name>
56
56
  --generation-date <iso> --confidence-tier <tier>
57
57
  --manual-sections-preserved <n> -o <file> [inputs...]
58
+ [--not-public-out <file>] [--metadata-in <file> --lists-out <file>]
58
59
  The provenance map an update writes (see "Apply" below): the old
59
60
  map with this run's changes, written to -o through a temporary file
60
- and a rename. Prints a summary with the warnings for a person.
61
+ and a rename, the summary's not_public list to --not-public-out and
62
+ metadata.json's exports[], scripts[] and assets[] to --lists-out
63
+ when given. Prints a summary with the warnings for a person.
64
+
65
+ metadata --metadata <metadata.json> --lists <file> --stats <file>
66
+ --version <version> --generation-date <iso>
67
+ [--source-commit <sha>] [--source-ref <ref>] [--skf-root <dir>]
68
+ [--extraction <file>] [--qmd-version <version>]
69
+ The skill's metadata.json an update leaves (see "Metadata" below),
70
+ written over --metadata through a temporary file and a rename.
71
+ Prints {"status": "written", "metadata": <path>, "version",
72
+ "exports": N, "scripts": N, "assets": N, "tool_versions": {...},
73
+ "dropped_stats": [...], "warnings": [...]}.
61
74
 
62
75
  records [--extraction <file>] [--export-details <file>]
63
76
  [--files-from <file>] [--patches <dir>] -o <file>
64
77
  Step 3's re-extraction records (see "Records" below), written to -o
65
78
  through a temporary file and a rename. Prints {"status": "written",
66
79
  "output", "files_extracted", "exports_extracted",
67
- "confidence_breakdown": {"T1", "T1-low", "T2"}, "warnings"}.
80
+ "confidence_breakdown": {"T1", "T1-low", "T2"}, "marked_not_public"
81
+ (the exports marked public: false), "warnings"}.
68
82
 
69
83
  gap-records --manifest <file> --provenance-map <file> [--source-root <dir>]
70
84
  [--drift-status ok|skipped|overridden] [--judgments <file>]
@@ -72,8 +86,17 @@ Subcommands:
72
86
  (--plan [--files-out <file>] | -o <file>)
73
87
  A gap-driven repair's spot-checks, routing and verification records
74
88
  (see "Gap records" below): --plan prints what to answer and which
75
- files targeted re-extraction scans; without it, the records are
76
- written to -o through a temporary file and a rename.
89
+ files §4a's extraction scans (targeted re-extraction, and the
90
+ signatures of the exports a spot-check located); without it, the
91
+ records are written to -o through a temporary file and a rename.
92
+
93
+ baseline-gaps --provenance-map <file> --extraction <file> --files <file>
94
+ --source-root <dir> -o <file>
95
+ Category B's baseline gaps (see "Baseline gaps" below): each map
96
+ entry of the files --files lists that the runner did not report and
97
+ its file still declares, written to -o through a temporary file and
98
+ a rename. Prints {"status": "written", "output": <-o>, "gaps": N,
99
+ "unchecked": N}.
77
100
 
78
101
  Helper files in place of typed slices (update-skill detect-changes §2.1):
79
102
 
@@ -94,9 +117,13 @@ Helper files in place of typed slices (update-skill detect-changes §2.1):
94
117
  changed); moved_exports for an
95
118
  export whose only field is `line`
96
119
  moved[] -> moved_exports {name, file:
97
- current_file, old_line:
120
+ current_file, old_file:
121
+ previous_file (only when it
122
+ differs), old_line:
98
123
  previous_line, new_line: line},
99
- unless it is a modified export
124
+ unless it is a modified export,
125
+ which then takes old_file and
126
+ old_line: previous_line
100
127
  label_changes[] is not drift, and a name
101
128
  ambiguous_names[] lists stays removed and added
102
129
  --category-c <file> the output of rename-candidates: its category_c
@@ -136,10 +163,10 @@ Input JSON shape (object on stdin or in --input file):
136
163
  "deleted": ["path3", ...]
137
164
  },
138
165
  "category_b": {
139
- "modified_exports": [{name, file, old_line, new_line}, ...],
166
+ "modified_exports": [{name, file, old_line, new_line, old_file?}, ...],
140
167
  "new_exports": [{name, file, line}, ...],
141
168
  "deleted_exports": [{name, file, old_line}, ...],
142
- "moved_exports": [{name, file, old_line, new_line}, ...]
169
+ "moved_exports": [{name, file, old_line, new_line, old_file?}, ...]
143
170
  },
144
171
  "category_c": {
145
172
  "renamed_files": [{old_path, new_path}, ...],
@@ -164,6 +191,10 @@ Manifest (build's output):
164
191
  "total_export_changes": N, "per_file": [...],
165
192
  "category_d": {the eight category_d lists, each path once}}
166
193
 
194
+ Each per_file item's exports_affected holds {name, change_type,
195
+ old_line, new_line}, and old_file for an export that moved across files
196
+ (a MOVED_EXPORT, or a MODIFIED_EXPORT that also changed).
197
+
167
198
  A tracked document (a doc row of file_entries[]) has no other detector:
168
199
  update-skill's Category A never lists it, so a document-only change still
169
200
  counts, and the update records its new hash.
@@ -213,14 +244,32 @@ Apply (update-skill write.md §3):
213
244
 
214
245
  Reads the provenance map the update started from (--provenance-map;
215
246
  without it, a degraded run's new map) and prints nothing but a summary:
216
- the new map goes to -o. Every entry the run leaves alone keeps its exact
217
- value; every entry it adds or rewrites carries signature_source, set
218
- from the tool that produced the signature: ast-grep gives T1 (with
219
- confidence T1 and the recipe's ast_node_type), a signature read by eye
220
- (source-read) gives T1-low (confidence T1-low, ast_node_type null).
221
- A rewritten entry keeps any key the update does not set (notes,
222
- deprecated). New entries take source_library from the map's entries,
223
- else --skill-name.
247
+ the new map goes to -o. Every entry it adds or rewrites carries
248
+ signature_source, set from the tool that produced the signature:
249
+ ast-grep gives T1 (with confidence T1 and the recipe's ast_node_type),
250
+ a signature read by eye (source-read) gives T1-low (confidence T1-low,
251
+ ast_node_type null). Every entry the run leaves alone keeps its exact
252
+ value, but for a signature_source it lacks or holds as a value other
253
+ than T1, T1-low, T2 or T3 (compared case-insensitively), as a map
254
+ written before the key existed does: once the mode's changes are made,
255
+ each such entry takes its confidence when that names T1-low, T2 or T3,
256
+ else T1 for an extraction_method of ast-grep or ast_bridge (compared
257
+ stripped and case-insensitively) and T1-low for any other (T1 needs an
258
+ ast-grep match). The summary's signature_source_backfilled counts these
259
+ fills by value ({"T1": 1, "T1-low": 2}, {} when none), with one warning
260
+ that lists each unrecognized value replaced (name: old -> new, the
261
+ first 10, then "and N more"); their names are not in its `updated`. A
262
+ rewritten entry keeps any key the update does not set (notes,
263
+ deprecated). A new entry's source_library is the map's library when its
264
+ entries name one (--skill-name when they name none). When they name
265
+ several, it is the library the entries of its file settle, else the one
266
+ the entries under the nearest folder above the file that holds entries
267
+ settle: the one library they name, or of several, the one that is a
268
+ path prefix of all the others (storybook beside storybook/test). The
269
+ record of a MOVED file, or of an export whose change names an old_file,
270
+ tries its new path, then its old. When neither settles it, or the entry
271
+ has no file, it is the map's first library, and one warning counts and
272
+ names those entries (the first 10, then "and N more").
224
273
 
225
274
  incremental (normal mode) reads --manifest (build's output), the renamed
226
275
  exports of --category-c and --ccc-pairs (or of --input, the category
@@ -233,14 +282,37 @@ Apply (update-skill write.md §3):
233
282
  details, then step 3's record (the runner's in the map's typed form).
234
283
  Per file of the manifest:
235
284
  MODIFIED NEW_EXPORT adds an entry, MODIFIED_EXPORT rewrites the
236
- export's fields from its fresh record, MOVED_EXPORT sets
237
- its line (and file), DELETED_EXPORT removes it
285
+ export's fields (file included) from its fresh record,
286
+ MOVED_EXPORT moves the entry to the file and the fresh
287
+ record's line, else new_line, DELETED_EXPORT removes it;
288
+ a change with an old_file finds its entry in that file
238
289
  ADDED every fresh record of the file becomes an entry
239
290
  DELETED every entry of the file is removed
240
291
  MOVED the old path's entries are replaced by the new path's
241
292
  fresh records, each keeping the keys of the entry of its
242
293
  name it replaces
243
- and each renamed export takes its new name and fresh fields. A docs-only
294
+ and each renamed export takes its new name and fresh fields. A
295
+ MOVED_EXPORT whose fresh record is the runner's (ast-grep) also takes
296
+ its confidence, extraction_method and ast_node_type (the entry's when
297
+ the record's is null), and its signature_source when the record holds
298
+ each of params and return_type the entry holds (else the entry's, or
299
+ T1-low when the entry has none it recognizes: the signature it keeps
300
+ was not the runner's), never its export_type, params or return_type;
301
+ one read by eye moves the line and file only.
302
+
303
+ Public-api skills (incremental only): a NEW_EXPORT or MODIFIED_EXPORT
304
+ the map does not hold, an ADDED file's record and a MOVED file's record
305
+ whose name no old entry carries become entries only when on the public
306
+ surface: the record's `public` mark (see "Records"), else, for a record
307
+ --reextract-records does not mark, the same rule over --extraction (its
308
+ scope.type). A name whose entry this run removes (a DELETED_EXPORT, a
309
+ DELETED file's or a MOVED file's old path's entry) is added whatever
310
+ its mark. Each one left out is listed in the summary's not_public,
311
+ {name, file}, with one warning for them all; an entry the map holds is
312
+ updated whatever its mark, and no entry is removed for it. A public-api
313
+ --extraction the rule cannot read (no entry_point_diff, an incomplete
314
+ run or one with errors, an unresolved entry point) adds every name, with
315
+ one warning. A docs-only
244
316
  run's records ({"mode": "docs-only", "changed_urls": [...], "exports":
245
317
  [{name, type, params, return_type, url}]}) replace the entries of each
246
318
  changed URL (source_file is the URL; confidence and signature_source
@@ -249,31 +321,91 @@ Apply (update-skill write.md §3):
249
321
  gap-driven reads --reextract-records ({"mode": "gap-driven",
250
322
  "verification": [...], "files": [...]}) and --merge-records ({"exports":
251
323
  [{export_name, export_type, params, return_type}]}, merge's output for
252
- an export no fresh record holds). Each verification record names
324
+ an export no record covers). Each verification record names gap_id,
253
325
  export_name, gap_category, severity, verification, in_map (the map holds
254
326
  an entry of that name), map_entry ({source_file, source_line} of the one
255
327
  entry the spot-check took, else null), source_citation, new_location,
256
- unknown_reason, pinned_definition_lines and reachability:
257
- verified (map_entry) unchanged
258
- moved (map_entry) source_line (and source_file) from new_location
328
+ unknown_reason, citation_check, map_locations (see "Gap records"),
329
+ pinned_definition_lines, reachability, and the located
330
+ export's export_type, params, return_type, extraction_method and
331
+ ast_node_type (from §4a's record at its line, see "Gap records"; null
332
+ when none holds it). It finds an export's entry by its name, else by the
333
+ entry's reexported_as (a renamed re-export the map holds under its
334
+ declared name, which the gap names by its public one):
335
+ verified (map_entry) unchanged, but a NEW_EXPORT's or
336
+ MODIFIED_EXPORT's signature (below) when its
337
+ record was found
338
+ moved (map_entry) source_line (and source_file) from new_location,
339
+ and a NEW_EXPORT's or MODIFIED_EXPORT's
340
+ signature (below) when its record was found
259
341
  re-extracted the entry from its `files` record (gap-driven.md §4a);
260
- none when its reachability is internal-unreachable
342
+ none when its reachability is internal-unreachable;
343
+ a record with a `local` name (matched by the
344
+ public name a re-export gives it) writes
345
+ export_name the declared `local` name, at its
346
+ line, and reexported_as the public one
261
347
  verified/moved, not in_map, a NEW_EXPORT or MODIFIED_EXPORT whose
262
348
  reachability is not internal-unreachable
263
349
  one source-read entry at the citation's line
264
350
  (verified) or new_location (moved), at most one
265
- per name and file, export_type and the
266
- signature from --merge-records
351
+ per name and file, with the signature (below)
352
+ and the source-read labels whatever its record's
353
+ tool
267
354
  unknown, not in_map, a NEW_EXPORT or MODIFIED_EXPORT, a Medium, Low or
268
355
  Info severity an entry from --merge-records with no
269
356
  source_file or source_line
270
357
  unknown, not in_map, any other severity (a missing one included)
271
358
  refused: exit 3, nothing written
272
359
  missing, unknown in_map, unknown on a MOVED_EXPORT
273
- unchanged, with a warning for a person
360
+ unchanged, with a warning for a person that
361
+ names an unknown's unknown_reason and its
362
+ words (UNKNOWN_REASONS); for a NEW_EXPORT or
363
+ MODIFIED_EXPORT whose severity is not Medium,
364
+ Low or Info (a missing one included), the
365
+ warning also names its gap_id and the one
366
+ fix a re-run acts on: for an ambiguous one,
367
+ fix its entries by hand, or give the gap a
368
+ `file:line` Source at its definition when
369
+ that file holds only one of them (the
370
+ re-run's map_match then finds that one); for
371
+ any other, fix the entry by hand (its
372
+ source_file and source_line), since the
373
+ re-run checks the same entry again. A
374
+ cited-elsewhere one, whatever its severity,
375
+ takes in their place a warning that names
376
+ its gap_id, its source_citation, its
377
+ citation_check (with its words), its
378
+ map_locations and the fix for each side
379
+ that may be wrong: when the map is right,
380
+ re-run Test Skill (the test compared
381
+ another file's declaration), and if the gap
382
+ comes back (not for a missing one), exclude
383
+ the cited file in the brief's scope when the
384
+ skill does not cover it; when the export
385
+ moved, fix the entry by hand (with several
386
+ map_locations, whichever of those entries
387
+ moved), then re-run update-skill. A
388
+ citation_check that read nothing (UNREAD:
389
+ no-source-root, verifier-missing,
390
+ spot-check-failed) takes its own fix in
391
+ their place: give update-skill the source
392
+ tree, re-install SKF, or fix what the check
393
+ failed on, then re-run update-skill
274
394
  rescoped the entry removed
395
+ The signature: when §4a found the export's record (its verification
396
+ record has an extraction_method), every part from it, a params or
397
+ return_type it gives as null removed from the entry; with none, an
398
+ entry the map holds keeps its signature, and an entry this run adds
399
+ takes --merge-records'. An entry the map holds keeps its confidence and
400
+ extraction_method; when its own extraction_method is ast-grep or
401
+ ast_bridge, it takes the signature_source of its record's labels (T1
402
+ for an ast-grep record, T1-low for a source-read one) and, when the
403
+ record's export_type is another, the record's ast_node_type, while a
404
+ source-read entry keeps T1-low and no node kind. Each signature_source
405
+ it changes is listed in `relabeled`, {name, file, from, to}, each with a
406
+ warning. An internal-unreachable NEW_EXPORT takes none.
275
407
  Under the drift override (--drift-head and --drift-pinned) each warning
276
- names the drift.
408
+ names the drift, after the outcome's cause.
277
409
 
278
410
  full (degraded mode) makes every fresh record an entry.
279
411
 
@@ -295,9 +427,81 @@ Apply (update-skill write.md §3):
295
427
  manual_sections_preserved. An older update-history key
296
428
  (update_operations, update_metadata) stays as it is.
297
429
 
430
+ Lists. With --metadata-in (the skill's metadata.json as the update
431
+ found it) and --lists-out, it also writes the lists metadata.json takes,
432
+ {"exports": [...], "scripts": [...], "assets": [...]}. exports: the
433
+ --metadata-in exports[] less each name whose entry this run removed (a
434
+ DELETED_EXPORT, a DELETED file's or a MOVED file's old path's, a
435
+ rescoped one, a changed URL's) or renamed, unless an entry of the new
436
+ map still holds it, then each name it added an entry for (the entry's
437
+ reexported_as when it has one, the public name) or renamed an entry to,
438
+ each once; a renamed name takes its old name's place. A name
439
+ left out as off the public surface (the summary's not_public) has no
440
+ entry, so it is not added. A full update, which rebuilds the map, also
441
+ drops each name the new map does not hold, and a reference app
442
+ (scope_type reference-app) gets []. scripts and assets: one row per
443
+ file_entries[] row of the new map whose file_type is script or asset
444
+ (a row with none takes the type its file_name's folder names), in the
445
+ map's order: the --metadata-in scripts[] or assets[] row of its
446
+ source_file (else of its file_name) as it stands, else a new row {file:
447
+ its file_name, purpose, source_file, confidence (default T1-low)},
448
+ whose purpose skf-detect-scripts-assets.py reads from the file under
449
+ --source-root (its file name when no --source-root is given or the
450
+ file is not there).
451
+
452
+ Metadata (update-skill write.md §2):
453
+
454
+ metadata.json as --metadata holds it, with what the update sets: version
455
+ (--version) and generation_date (--generation-date, YYYY-MM-DDTHH:MM:SSZ);
456
+ source_commit and source_ref when --source-commit or --source-ref is
457
+ given (an empty one sets null), else as they stand; exports, scripts and
458
+ assets from --lists (apply's --lists-out), scripts or assets left out
459
+ when its list is empty; confidence_distribution from --stats (the JSON
460
+ skf-render-metadata-stats.py writes with -o), and stats: the stats
461
+ helper's fields, then each other field --metadata's stats holds (a
462
+ component library's own counts, a reference app's notes, a field a
463
+ gap-driven patch added), the stats helper's fields an earlier run wrote
464
+ that this run's does not (effective_denominator,
465
+ pattern_surfaces_documented) dropped and listed in `dropped_stats`,
466
+ each with a warning. When this run's effective_denominator differs
467
+ from the old one (one of them absent included), it also drops each old
468
+ stats key named effective_denominator_* (an explanation of the old
469
+ count), listed in `dropped_stats` with the warning "metadata:
470
+ stats.<key> dropped: it explained an effective_denominator this run
471
+ changed or removed"; when the two are the same, such a key stays with
472
+ no warning, since it still describes the denominator. Each other old
473
+ stats key it keeps adds the warning "metadata: stats.<key> kept from an
474
+ earlier run; check it still describes this run's counts", unless
475
+ create-skill writes it for the skill's scope_type (STATS_CREATE_FIELDS:
476
+ a component-library's Component Library stats of
477
+ compile-assembly-rules.md, a reference-app's notes).
478
+
479
+ tool_versions names the tools this run used: the old tool_versions
480
+ ({"ast_grep": null, "qmd": null} when absent, or when it is not an
481
+ object, which adds a warning), with skf set to the version
482
+ skf-render-quick-metadata.py's probe_skf_version reads under
483
+ --skf-root (package.json, else VERSION), never the old value, or to
484
+ "unknown" with a warning, one when no --skf-root is given and another
485
+ naming the folder when it holds no version; ast_grep set to the
486
+ ast_grep.version of --extraction (the recipe runner's output), or to
487
+ "unknown" with a warning when that is null but an export of it has
488
+ extraction_method ast-grep; qmd set to --qmd-version, a leading "qmd "
489
+ (as `qmd --version` prints it) removed. A tool this run did not use
490
+ (no --qmd-version, no --extraction, or one with no export ast-grep
491
+ matched) keeps its old value. It removes last_update and update_type,
492
+ which the provenance map's update block records, and keeps every
493
+ other field. It refuses (exit 1, nothing written, one stderr line) a
494
+ --stats whose scripts_count or assets_count is not the length of
495
+ --lists' scripts or assets, a --generation-date of another form, an
496
+ --extraction that is not a JSON object, a file it cannot read, and a
497
+ skf-render-quick-metadata.py beside it that cannot be loaded.
498
+
298
499
  Summary on stdout: {"status": "written", "map": <-o>, "entries":
299
- {"added", "updated", "removed"}, "file_entries": {"added", "updated",
300
- "removed"}, "warnings": [...]}, or {"status": "refused",
500
+ {"added", "updated", "removed"}, "not_public": [{name, file}],
501
+ "relabeled": [{name, file, from, to}] (gap-driven),
502
+ "signature_source_backfilled": {tier: count}, "file_entries":
503
+ {"added", "updated", "removed"}, "warnings": [...]},
504
+ or {"status": "refused",
301
505
  "blocking_unresolved": [{export_name, severity}]} with exit 3.
302
506
 
303
507
  Records (update-skill re-extract.md §4, and gap-driven.md §4a for the files
@@ -324,6 +528,42 @@ it scans):
324
528
  no seed holds is not added, with a warning. confidence_breakdown counts
325
529
  the exports by confidence, and T2 the exports with a qmd_evidence.
326
530
 
531
+ Aliases. Each --extraction entry_point_diff `public` and
532
+ `other_definitions` item (an other in-scope definition of a public
533
+ name), and each member of a namespace item, whose `local` differs from
534
+ its `name` gives its name to the record of its `local` name and file,
535
+ when one is seeded:
536
+ that record's `aliases` lists each such public name once, sorted (an
537
+ export re-exported under another name, `export { MockUniversalStore as
538
+ experimental_MockUniversalStore }`, keeps its declared name and carries
539
+ the public one). A record no item renames has no `aliases`, and with
540
+ no entry_point_diff (Quick tier, a runner exit 3, an export read by
541
+ eye) no record has any.
542
+
543
+ Public surface. When --extraction's scope.type is public-api (in any
544
+ case) and it has an entry_point_diff, each export of a language family
545
+ whose entry_points.by_language status is `barrel` is marked `public`:
546
+ true when its name and file are a pair of the runner's entry_point_diff
547
+ `public` or `other_definitions` (its name, or the name `local` gives it
548
+ in that file) or `extraction_gaps`, or when a `public` item of its name
549
+ and family has no file; false otherwise. A `public` item with `via`
550
+ namespace and a
551
+ `members` list (what the submodule passes on, which users reach as
552
+ `pkg.sub.name`, and which counts.exports_public_api counts in its place)
553
+ gives its members' pairs (each member's name, or its `local` one, and
554
+ its file; a member with no file its name) in place of its own; one
555
+ whose `members` is null (a module the runner could not read), or that
556
+ has no `members` key (an older runner's), gives its own pair, as any
557
+ item does.
558
+ The rule is skf-extraction-inventory.py's public_surface, loaded from
559
+ this folder, by which create-skill marks its extraction inventory too.
560
+ In a normal update merge documents no export marked false that the map
561
+ does not hold, and apply adds none; gap-records drops the mark, and a
562
+ full (degraded) apply adds every record. Any other scope type, a family
563
+ with no barrel, and a run with no entry_point_diff, an incomplete one,
564
+ one with errors or one whose entry_points.unresolved is not empty mark
565
+ nothing.
566
+
327
567
  Gap records (update-skill gap-driven.md §3, §4 and §4a):
328
568
 
329
569
  Reads the change manifest skf-parse-gaps.py translate wrote. Its
@@ -336,8 +576,11 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
336
576
  but a MOVED_EXPORT needs the tree, with the first reason that fits:
337
577
  a DELETED_EXPORT "a public API recount from the tree (rule R1)", a
338
578
  provenance_completeness entry "a line from the tree (rule R3)", one
339
- whose map_match is not-found "a line and a signature from the tree",
340
- any other "a signature from the tree". Any such entry: {"status":
579
+ whose map_match is not-found with no candidate that has a
580
+ source_file "a line and a signature from the tree", any other (a
581
+ not-found with such a candidate, a citation of another file than
582
+ the map's entries, among them) "a signature from the tree". Any
583
+ such entry: {"status":
341
584
  "drift-blocked", "drift_blocked": [{gap_id, name, change_category,
342
585
  severity, reason}]}, and no spot-check runs.
343
586
  2. A DELETED_EXPORT without a `rescope` that holds its `amendment` and
@@ -353,10 +596,28 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
353
596
  its remediation's "definition line(s) (...)"
354
597
  NEW_EXPORT, MODIFIED_EXPORT
355
598
  map_match found checked at the map entry (its export_type)
356
- ambiguous unknown
599
+ ambiguous unknown, unknown_reason ambiguous (several
600
+ map entries of its name: no check runs)
357
601
  not-found checked at its source_citation (no export
358
- type) when it has one; when that pins no
359
- line (unknown or missing), or with no
602
+ type) when it has one. When the map holds
603
+ entries of its name (in_map, which a
604
+ not-found has only for a citation of
605
+ another file than theirs), one of them
606
+ with a source_file, and no check pins a
607
+ line (unknown or missing): unknown,
608
+ unknown_reason cited-elsewhere, whatever
609
+ its severity (a provenance_completeness
610
+ one too), and never re-extracted, so no
611
+ other file's declaration is read for it;
612
+ its record's citation_check keeps the
613
+ check's cause (missing, or its
614
+ unknown_reason) and its map_locations the
615
+ `file:line` (or file) of each candidate
616
+ with a source_file, both null on any other
617
+ record. Candidates with no source_file
618
+ (placeholders) do not count. Otherwise,
619
+ when that pins no line
620
+ (unknown or missing), or with no
360
621
  citation: targeted re-extraction (§4a) for a
361
622
  provenance_completeness entry with a source
362
623
  root, for a blocking one whatever its
@@ -366,36 +627,78 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
366
627
  A check gives `verified` (the line is a definition line, or a module
367
628
  or package entry whose file exists), `moved` (one definition line,
368
629
  not the recorded one: new_location), `missing` (no such file) or
369
- `unknown` (several definition lines, none the recorded one, or
370
- none). A file the rules cover no language of
371
- takes the answered `declaring_line`: the recorded line is verified,
372
- another is moved, null is unknown. With no verifier every check is
373
- unknown, with a `provenance: spot-checks not run` warning; a check
374
- the verifier fails on (a file it cannot read, or any error) is
375
- unknown, with a `provenance: spot-check failed for {name}: {error}`
376
- warning.
630
+ `unknown`, its unknown_reason `several-definitions` (several
631
+ definition lines, none the recorded one) or `no-definition` (none).
632
+ A file the rules cover no language of takes the answered
633
+ `declaring_line`: the recorded line is verified, another is moved,
634
+ null is unknown (`unchecked-language`). With no --source-root every
635
+ check is unknown (`no-source-root`); with no verifier every check is
636
+ unknown (`verifier-missing`), with a `provenance: spot-checks not
637
+ run` warning; a check the verifier fails on (a file it cannot read,
638
+ or any error) is unknown (`spot-check-failed`), with a `provenance:
639
+ spot-check failed for {name}: {error}` warning; a map entry with no
640
+ file or line is unknown (`no-line`). UNKNOWN_REASONS gives each its
641
+ words. Every other outcome has a null unknown_reason. A map_match
642
+ whose entry carries its own export_name (skf-parse-gaps matched it
643
+ by its reexported_as) is checked under that name.
644
+ A NEW_EXPORT or MODIFIED_EXPORT the map holds entries of its name
645
+ for (its map_match found or ambiguous, or not-found with a
646
+ candidate that has a source_file) never goes to targeted
647
+ re-extraction, whatever its severity: one that ends missing or
648
+ unknown keeps its map entries unchanged and adds none, and apply
649
+ warns (see "Apply").
650
+ A NEW_EXPORT or MODIFIED_EXPORT a check gives verified or moved, at
651
+ its map entry or at a citation the map holds no entry of its name
652
+ for, is located: its signature_at is the checked file and the
653
+ recorded line (verified) or the new one (moved). One whose citation
654
+ names another file than the map's entries of its name is not (apply
655
+ warns, as before), nor one whose file is absolute or outside
656
+ --source-root, links resolved, nor a NEW_EXPORT --judgments answers
657
+ internal-unreachable (one not yet answered is located, so its scan
658
+ may go unused).
377
659
  --plan stops here: {"status": "planned", "reextract": {"entries":
378
660
  [{gap_id, name, severity, resolved_paths}], "files": [the union of
379
- their resolved_paths]}, "warnings"}, the files also written to
380
- --files-out as a JSON array.
661
+ their resolved_paths, then of the located exports' files],
662
+ "signatures": [{gap_id, name, file, line} of each located export]},
663
+ "warnings"}, the files also written to --files-out as a JSON array.
381
664
  4. Targeted re-extraction (--remediation-records, `records`' output
382
665
  over those files): each routed entry with resolved_paths takes the
383
666
  first export of its name in the blocks of those files, in their
384
- order: `re-extracted`, new_location its record's location,
667
+ order, else the first whose `aliases` hold it (matched by name, or
668
+ by the public name the runner gives its local name in that file,
669
+ see "Records"): `re-extracted`, new_location its record's location,
385
670
  resolution_source remediation-paths, and the record copied into
386
- `files` as `records` wrote it, with the answered `docstring` added.
671
+ `files` as `records` wrote it, less its `public` mark (a normal
672
+ update's) and its `aliases`, with the answered `docstring` added. A
673
+ record matched by an alias is copied with `name` the public name and
674
+ `local` its own name. The aliases come from the runner's
675
+ entry_point_diff, so at Forge tier and above only: at Quick tier, or
676
+ when the runner exits 3 and §4a reads the files by eye, a gap naming
677
+ a renamed re-export by its public name matches no record.
387
678
  An entry with no match, or no path to scan, is unresolved, except a
388
679
  missing-export or missing-type one that is not blocking, which is
389
680
  unknown. Any unresolved entry: {"status": "unresolved",
390
681
  "unresolved": [{gap_id, name, severity, remediation_paths,
391
682
  rejected_paths, files_scanned, exports_found_in_scan}]}.
683
+ Each located export takes the one record of its name (or of its map
684
+ entry's own export_name) in the block of its file at its line, else
685
+ the one with no line: the record copied into `files` as above, under
686
+ the gap's name, less its `public` mark and `aliases`, with no
687
+ docstring asked
688
+ (one copy per name and location, or per record with no location),
689
+ and its type, params, return_type, extraction_method and
690
+ ast_node_type on the verification record. One with no such record
691
+ keeps them null, with a `provenance: {name}: located at
692
+ {file}:{line}, but no record of it there; signature not repaired`
693
+ warning (§4a's by-eye pass may add it to --export-details). Files to
694
+ match, of either kind, with no --remediation-records are an error.
392
695
  5. The public-reachability gate: every NEW_EXPORT that is verified,
393
696
  moved or re-extracted takes the answered `reachability`, `public` or
394
697
  `internal-unreachable`. An internal one keeps its outcome in its
395
- verification record, its record is not in `files`, `counts` tallies
396
- it under `reclassified` instead of its outcome, and it is listed in
397
- `reclassified` (merge queues it as a metadata update; apply writes no
398
- entry for it).
698
+ verification record (with no signature), its record is not in
699
+ `files`, `counts` tallies it under `reclassified` instead of its
700
+ outcome, and it is listed in `reclassified` (merge queues it as a
701
+ metadata update; apply writes no entry for it).
399
702
 
400
703
  Any answer missing in step 3 or 5 (a `docstring` too, for each
401
704
  re-extracted entry): {"status": "needs-judgment", "needs_judgment":
@@ -407,25 +710,57 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
407
710
  "unlabeled", "T2": 0} (each manifest entry by the extraction_method of
408
711
  its re-extracted record or, in the map, of its map entry; a pinned cited
409
712
  export the map lacks and the gate passed is T1-low), "verification":
410
- [one record per export-bearing entry: export_name, gap_category,
411
- severity, verification, in_map, map_entry, provenance_citation,
412
- source_citation, new_location, unknown_reason, pinned_definition_lines,
413
- resolution_source, reachability, export_type (always null: for a cited
414
- export the map lacks, apply takes it from merge.md's
415
- merge-records.json)], "files": [...]}, and prints {"status": "written",
713
+ [one record per export-bearing entry: gap_id, export_name,
714
+ gap_category, severity, verification, in_map, map_entry,
715
+ provenance_citation, source_citation, new_location, unknown_reason,
716
+ citation_check, map_locations, pinned_definition_lines,
717
+ resolution_source, reachability, and
718
+ export_type, params, return_type, extraction_method and ast_node_type
719
+ (a located export's record's, step 4; else null, and apply takes the
720
+ signature of an entry it adds from merge.md's merge-records.json)],
721
+ "files": [...]}, and prints {"status": "written",
416
722
  "output", "gap_count", "files_extracted", "exports_extracted",
417
723
  "confidence_breakdown", "counts" (per verification outcome, and
418
724
  `reclassified`), "reclassified", "forwarded", "targeted_reextraction",
419
- "warnings"}. When targeted re-extraction ran, `targeted_reextraction`
420
- is {resolved_count, files_scanned, exports_matched, tier}, also appended
421
- to --evidence as one JSON line keyed `targeted_reextraction`.
725
+ "warnings"}. When §4a ran (targeted re-extraction, or a located
726
+ export's signature), `targeted_reextraction` is {resolved_count,
727
+ files_scanned (the targeted re-extraction's files),
728
+ signature_files_scanned (the located exports' files), exports_matched,
729
+ tier}, also appended to --evidence as one JSON line keyed
730
+ `targeted_reextraction`.
731
+
732
+ Baseline gaps (update-skill detect-changes Category B, step 1):
733
+
734
+ The recipe runner never reports a module, an alias that renames, a
735
+ dunder such as `__version__` or another Python or Rust `_` name, so the
736
+ diff would read each such map entry as a deleted export. baseline-gaps
737
+ lists, for step 2 to read by eye, each entry of a file --files lists
738
+ (the modified files, Category A's modified-files.json) whose name and file
739
+ no export of --extraction (the runner's JSON, or {"exports": []} when
740
+ it could not run) holds, under its name or its re-export target, and
741
+ whose file still declares it. The selection and the declaration rule
742
+ are skf-verify-provenance-completeness.py's `baseline_gaps` (the rule
743
+ audit-skill's extraction snapshot applies), loaded from this folder: a
744
+ verifier that cannot be loaded is an error, never an empty list. A
745
+ name the runner's own `entry_point_diff.extraction_gaps` lists at that
746
+ file is left to that list. A map whose `entries` is missing or null
747
+ has none to check. Writes {"gaps": [{name, file, line, entry: null,
748
+ export_type?}], "unchecked": [{name, file, export_type}]}: a gap's line
749
+ declares the name (line 1 of the module or package so named for a
750
+ `module` or `package` entry), and `unchecked` holds each entry no rule
751
+ can check, to read by eye: its file neither Python nor TS/JS, a file
752
+ the lookup will not read (outside the source root, a test file or a
753
+ minified bundle, a symlink on its path), or a dotted `module` or
754
+ `package` name.
422
755
 
423
756
  Exit codes:
424
757
  0 operation succeeded (for gap-records, whatever its `status`)
425
758
  1 user error (malformed JSON, bad path, malformed provenance file, a
426
759
  helper file that is not the output it names, a gap-records answer of
427
- the wrong type, or targeted re-extraction to match with no
428
- --remediation-records)
760
+ the wrong type, files to match (targeted re-extraction or a located
761
+ export's signature) with no --remediation-records, or, for
762
+ baseline-gaps, a source root that is no folder or a verifier that
763
+ cannot be loaded)
429
764
  3 apply refused: a blocking gap reached the provenance write with no
430
765
  source line (nothing written)
431
766
  """
@@ -538,11 +873,20 @@ def category_b_from_diff(diff: dict) -> dict:
538
873
  lists["modified_exports"].append({"name": key[0], "file": key[1], "old_line": None, "new_line": None})
539
874
  modified.add(key)
540
875
  for item in _get_list(diff, "moved"):
541
- if isinstance(item, dict) and (item.get("name"), item.get("current_file")) not in modified:
542
- lists["moved_exports"].append({
543
- "name": item.get("name"), "file": item.get("current_file"),
544
- "old_line": item.get("previous_line"), "new_line": item.get("line"),
545
- })
876
+ if not isinstance(item, dict):
877
+ continue
878
+ key, old_file = (item.get("name"), item.get("current_file")), _norm_path(item.get("previous_file"))
879
+ across = old_file not in (None, _norm_path(item.get("current_file"))) # apply finds the entry at old_file
880
+ if key in modified:
881
+ if across: # moved across files and changed: a MODIFIED_EXPORT that names the file it left
882
+ for entry in lists["modified_exports"]:
883
+ if (entry["name"], entry["file"]) == key:
884
+ entry.update(old_line=item.get("previous_line"), old_file=old_file)
885
+ continue
886
+ move = {"name": item.get("name"), "file": item.get("current_file")}
887
+ if across:
888
+ move["old_file"] = old_file
889
+ lists["moved_exports"].append({**move, "old_line": item.get("previous_line"), "new_line": item.get("line")})
546
890
  return lists
547
891
 
548
892
 
@@ -738,12 +1082,15 @@ def _build_per_file(
738
1082
  path = entry.get("file")
739
1083
  if not path:
740
1084
  return
741
- exports_by_file.setdefault(path, []).append({
1085
+ change = {
742
1086
  "name": entry.get("name"),
743
1087
  "change_type": change_type,
744
1088
  "old_line": entry.get("old_line"),
745
1089
  "new_line": entry.get("new_line") or entry.get("line"),
746
- })
1090
+ }
1091
+ if entry.get("old_file"):
1092
+ change["old_file"] = entry["old_file"] # a move across files: a MOVED_EXPORT, or a MODIFIED_EXPORT
1093
+ exports_by_file.setdefault(path, []).append(change)
747
1094
 
748
1095
  for e in b_modified:
749
1096
  _record(e, "MODIFIED_EXPORT")
@@ -943,6 +1290,34 @@ def _file_of(record: dict) -> str | None:
943
1290
  return _norm_path(record.get("source_file", record.get("file", record.get("file_path"))))
944
1291
 
945
1292
 
1293
+ _INVENTORY = None
1294
+
1295
+
1296
+ def _inventory():
1297
+ """skf-extraction-inventory.py, loaded once from this folder: its typed_param and its public-surface rule, so
1298
+ create-skill and update-skill write one parameter form and keep one public surface."""
1299
+ global _INVENTORY
1300
+ if _INVENTORY is None:
1301
+ path = Path(__file__).resolve().parent / "skf-extraction-inventory.py"
1302
+ spec = importlib.util.spec_from_file_location("skf_extraction_inventory", path)
1303
+ if spec is None or spec.loader is None:
1304
+ raise ImportError(f"cannot load {path}")
1305
+ module = importlib.util.module_from_spec(spec)
1306
+ spec.loader.exec_module(module)
1307
+ _INVENTORY = module
1308
+ return _INVENTORY
1309
+
1310
+
1311
+ def public_surface(extraction: object) -> tuple[object | None, str | None]:
1312
+ """skf-extraction-inventory.py's public_surface over the recipe runner's output (see "Records" in the module
1313
+ docstring): (the public surface, None) for a public-api run, (None, why) when its surface cannot be read, and
1314
+ (None, None) for any other scope type. An input that is no object (a gap-driven or docs-only run gives none)
1315
+ is (None, None) without loading the inventory helper."""
1316
+ if not isinstance(extraction, dict):
1317
+ return None, None
1318
+ return _inventory().public_surface(extraction)
1319
+
1320
+
946
1321
  # --------------------------------------------------------------------------
947
1322
  # Rename candidates (update-skill detect-changes Category C)
948
1323
  # --------------------------------------------------------------------------
@@ -1150,6 +1525,8 @@ REPLACE_WAIT_SECONDS = 5.0 # how long a Windows rename onto a file held open is
1150
1525
  # The update block apply sets at the top level of the map, in this order.
1151
1526
  UPDATE_BLOCK_KEYS = ("last_update", "update_type", "test_report_run_id", "files_changed", "exports_affected",
1152
1527
  "confidence_tier", "manual_sections_preserved")
1528
+ # The parts of a signature a gap-driven repair writes from a verification record, else from merge's records.
1529
+ SIGNATURE_PARTS = ("export_type", "params", "return_type")
1153
1530
 
1154
1531
 
1155
1532
  def _labels(method: object, node: object) -> dict:
@@ -1192,6 +1569,7 @@ class _Fresh:
1192
1569
  self.runner = {}
1193
1570
  self.details = {}
1194
1571
  self.worker = {}
1572
+ self.surface, self.surface_problem = public_surface(extraction)
1195
1573
  for record in _exports_of(extraction, "--extraction file"):
1196
1574
  key = (_name_of(record), _file_of(record))
1197
1575
  if all(key):
@@ -1215,6 +1593,17 @@ class _Fresh:
1215
1593
  def all_keys(self) -> list[tuple]:
1216
1594
  return sorted({k for source in (self.runner, self.details, self.worker) for k in source})
1217
1595
 
1596
+ def public(self, name: str, path: str) -> bool | None:
1597
+ """The `public` mark `records` gave the export, else the public-surface rule over --extraction; None when
1598
+ neither marks it."""
1599
+ mark = (self.worker.get((name, path)) or {}).get("public")
1600
+ if isinstance(mark, bool):
1601
+ return mark
1602
+ if self.surface is None:
1603
+ return None
1604
+ record = self.runner.get((name, path)) or self.details.get((name, path)) or {}
1605
+ return self.surface.public(name, path, record.get("language"))
1606
+
1218
1607
  def record(self, name: str, path: str) -> dict | None:
1219
1608
  """The entry fields of an export's fresh record, or None when no source holds it."""
1220
1609
  key = (name, path)
@@ -1265,16 +1654,17 @@ def _without_none(fields: dict, keep: tuple = ()) -> dict:
1265
1654
 
1266
1655
  def _new_entry(fields: dict, library: str) -> dict:
1267
1656
  """An entry in create-skill's key order, from the fields a record gives."""
1268
- order = ("export_name", "export_type", "source_library", "params", "return_type", "source_file",
1657
+ order = ("export_name", "reexported_as", "export_type", "source_library", "params", "return_type", "source_file",
1269
1658
  "source_line", "confidence", "extraction_method", "ast_node_type", "signature_source")
1270
1659
  merged = {**fields, "source_library": fields.get("source_library") or library}
1271
1660
  out = {k: merged[k] for k in order if k in merged and (merged[k] is not None or k == "ast_node_type")}
1272
1661
  return out
1273
1662
 
1274
1663
 
1275
- def _rewrite(entry: dict, fields: dict) -> dict:
1276
- """The entry with `fields` set in place; keys the update does not set keep their values."""
1277
- out = dict(entry)
1664
+ def _rewrite(entry: dict, fields: dict, drop: tuple = ()) -> dict:
1665
+ """The entry with `fields` set in place and the keys `drop` names removed; keys the update does not set keep
1666
+ their values."""
1667
+ out = {k: v for k, v in entry.items() if k not in drop}
1278
1668
  for key, value in fields.items():
1279
1669
  if value is None and key not in ("ast_node_type",):
1280
1670
  continue
@@ -1298,18 +1688,87 @@ class _Map:
1298
1688
  "provenance_version": "2.0", "skill_name": skill_name, "skill_type": "single", "entries": []}
1299
1689
  if not isinstance(self.doc.get("entries"), list):
1300
1690
  raise ValueError("provenance `entries` must be an array")
1301
- libraries = [e.get("source_library") for e in self.doc["entries"]
1302
- if isinstance(e, dict) and isinstance(e.get("source_library"), str) and e["source_library"]]
1303
- self.library = libraries[0] if libraries else skill_name
1691
+ # the map's libraries in entry order, and each file's, as the map stood before the run (#808)
1692
+ self.libraries: list[str] = []
1693
+ self.libraries_by_file: dict[str, set[str]] = {}
1694
+ for entry in self.doc["entries"]:
1695
+ library = entry.get("source_library") if isinstance(entry, dict) else None
1696
+ if not (isinstance(library, str) and library):
1697
+ continue
1698
+ if library not in self.libraries:
1699
+ self.libraries.append(library)
1700
+ if _file_of(entry):
1701
+ self.libraries_by_file.setdefault(_file_of(entry), set()).add(library)
1702
+ self.library = self.libraries[0] if self.libraries else skill_name
1703
+ # each folder's libraries (the entries under it, "" the root) and _library_at per path, built on first use
1704
+ self._libraries_under: dict[str, set[str]] | None = None
1705
+ self._library_of: dict[str, str | None] = {}
1706
+ # (name, file) of each new entry library_for gave the first library for want of a better one
1707
+ self.library_unresolved: list[tuple[str | None, str | None]] = []
1304
1708
  self.added: list[str] = []
1305
1709
  self.updated: list[str] = []
1306
1710
  self.removed: list[str] = []
1711
+ self.renamed: list[tuple[str, str]] = [] # (old name, new name)
1307
1712
  self.touched_files: set[str] = set()
1308
1713
 
1309
- def find(self, name: str, path: str | None, line: int | None = None) -> int | None:
1310
- """The index of the entry with this name (and file, and line when several share both)."""
1311
- hits = [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and _name_of(e) == name
1312
- and (path is None or _file_of(e) == path)]
1714
+ def library_for(self, path: str | None, name: str | None = None, *, old: str | None = None) -> str:
1715
+ """The source_library of a new entry for `name` in `path` (the record of a MOVED file, or of an export
1716
+ whose change names an old_file, passes that old path as `old`, tried second). A map naming one library (or
1717
+ none: --skill-name) gives it; in one naming several, the library of the file, else of the nearest folder
1718
+ (see _library_at). With neither, or no path, the map's first library, and the entry goes to
1719
+ `library_unresolved` for the summary's warning."""
1720
+ if len(self.libraries) <= 1:
1721
+ return self.library
1722
+ for candidate in (path, old):
1723
+ library = self._library_at(_norm_path(candidate))
1724
+ if library is not None:
1725
+ return library
1726
+ self.library_unresolved.append((name, _norm_path(path) or _norm_path(old)))
1727
+ return self.library
1728
+
1729
+ @staticmethod
1730
+ def _root_library(libraries: set[str]) -> str | None:
1731
+ """The one library of the set, or of several the one that is a path prefix of all the others (`storybook`
1732
+ beside `storybook/test`); None when no library is."""
1733
+ return next((library for library in sorted(libraries)
1734
+ if all(other == library or other.startswith(library + "/") for other in libraries)), None)
1735
+
1736
+ def _library_at(self, path: str | None) -> str | None:
1737
+ """The library the map's entries of `path` settle (see _root_library), else the one the entries under the
1738
+ nearest folder above it that holds entries settle; None when neither does. Memoized per path."""
1739
+ if not path:
1740
+ return None
1741
+ if path in self._library_of:
1742
+ return self._library_of[path]
1743
+ if self._libraries_under is None:
1744
+ self._libraries_under = {}
1745
+ for file, libraries in self.libraries_by_file.items():
1746
+ folders = file.split("/")[:-1]
1747
+ for depth in range(len(folders) + 1):
1748
+ prefix = "/".join(folders[:depth]) + "/" if depth else ""
1749
+ self._libraries_under.setdefault(prefix, set()).update(libraries)
1750
+ library = self._root_library(self.libraries_by_file.get(path, set()))
1751
+ if library is None:
1752
+ folders = path.split("/")[:-1]
1753
+ for depth in range(len(folders), -1, -1):
1754
+ under = self._libraries_under.get("/".join(folders[:depth]) + "/" if depth else "")
1755
+ if under:
1756
+ library = self._root_library(under)
1757
+ break
1758
+ self._library_of[path] = library
1759
+ return library
1760
+
1761
+ def find(self, name: str, path: str | None, line: int | None = None, *, reexport: bool = False) -> int | None:
1762
+ """The index of the entry with this name (and file, and line when several share both); with `reexport`,
1763
+ when no entry has the name, of the entry whose reexported_as is the name (a gap names a renamed re-export
1764
+ by its public name)."""
1765
+ def hits_of(key) -> list[int]:
1766
+ return [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and key(e) == name
1767
+ and (path is None or _file_of(e) == path)]
1768
+
1769
+ hits = hits_of(_name_of)
1770
+ if not hits and reexport:
1771
+ hits = hits_of(lambda e: e.get("reexported_as"))
1313
1772
  if len(hits) > 1 and line is not None:
1314
1773
  hits = [i for i in hits if _line_of(self.doc["entries"][i].get("source_line")) == line]
1315
1774
  return hits[0] if len(hits) == 1 else None
@@ -1323,12 +1782,14 @@ class _Map:
1323
1782
  if entry.get("source_file"):
1324
1783
  self.touched_files.add(_norm_path(entry["source_file"]))
1325
1784
 
1326
- def update(self, index: int, fields: dict) -> None:
1785
+ def update(self, index: int, fields: dict, drop: tuple = ()) -> None:
1327
1786
  before = self.doc["entries"][index]
1328
- after = _rewrite(before, fields)
1787
+ after = _rewrite(before, fields, drop)
1329
1788
  if after != before:
1330
1789
  self.doc["entries"][index] = after
1331
1790
  self.updated.append(_name_of(after))
1791
+ if _name_of(before) and _name_of(after) and _name_of(before) != _name_of(after):
1792
+ self.renamed.append((_name_of(before), _name_of(after)))
1332
1793
  for path in (_file_of(before), _file_of(after)):
1333
1794
  if path:
1334
1795
  self.touched_files.add(path)
@@ -1342,19 +1803,39 @@ class _Map:
1342
1803
 
1343
1804
 
1344
1805
  def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fresh: _Fresh,
1345
- warnings: list[str]) -> tuple[int, int]:
1346
- """Normal mode: the manifest's file and export changes. Returns (files, exports) for the update block."""
1806
+ warnings: list[str], not_public: list[dict]) -> tuple[int, int]:
1807
+ """Normal mode: the manifest's file and export changes. Returns (files, exports) for the update block; each
1808
+ name it leaves out as off a public-api skill's public surface goes to `not_public`."""
1347
1809
  per_file = [f for f in _get_list(manifest or {}, "per_file") if isinstance(f, dict)]
1348
-
1349
- def refresh(index: int | None, name: str, path: str, *, add: bool) -> None:
1810
+ # the names of the entries this run removes (a DELETED_EXPORT, a DELETED file's, a MOVED file's old path's):
1811
+ # one that comes back in another file is re-added whatever its mark, so no held export is lost
1812
+ removed: set[str] = set()
1813
+ for item in per_file:
1814
+ path = _norm_path(item.get("old_path") if item.get("status") == "MOVED" else item.get("file_path"))
1815
+ if item.get("status") in ("DELETED", "MOVED") and path:
1816
+ removed.update(_name_of(pmap.doc["entries"][i]) for i in pmap.indexes_of(path))
1817
+ removed.update(_name_of(c) for c in item.get("exports_affected") or []
1818
+ if isinstance(c, dict) and c.get("change_type") == "DELETED_EXPORT")
1819
+ listed: set[tuple[str, str]] = set()
1820
+
1821
+ def off_surface(name: str, path: str) -> bool:
1822
+ """True for a name the run would add that is off the public surface, which it lists."""
1823
+ if name in removed or fresh.public(name, path) is not False:
1824
+ return False
1825
+ if (name, path) not in listed:
1826
+ listed.add((name, path))
1827
+ not_public.append({"name": name, "file": path})
1828
+ return True
1829
+
1830
+ def refresh(index: int | None, name: str, path: str, *, add: bool, old: str | None = None) -> None:
1350
1831
  record = fresh.record(name, path)
1351
1832
  if record is None:
1352
1833
  warnings.append(f"provenance: {name} in {path}: no extraction record, entry "
1353
1834
  f"{'kept as it was' if index is not None else 'not added'}")
1354
1835
  return
1355
1836
  if index is None:
1356
- if add:
1357
- pmap.add(_new_entry(record, pmap.library))
1837
+ if add and not off_surface(name, path):
1838
+ pmap.add(_new_entry(record, pmap.library_for(path, name, old=old)))
1358
1839
  return
1359
1840
  pmap.update(index, _without_none(record, keep=("ast_node_type",)))
1360
1841
 
@@ -1373,8 +1854,10 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
1373
1854
  for name, key_path in fresh.keys_of(path):
1374
1855
  record = fresh.record(name, key_path)
1375
1856
  carried = old_entries.get(name)
1857
+ if carried is None and off_surface(name, key_path):
1858
+ continue
1376
1859
  entry = _rewrite(carried, _without_none(record, keep=("ast_node_type",))) if carried \
1377
- else _new_entry(record, pmap.library)
1860
+ else _new_entry(record, pmap.library_for(key_path, name, old=old))
1378
1861
  pmap.add(entry)
1379
1862
  continue
1380
1863
  if status == "ADDED":
@@ -1386,7 +1869,8 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
1386
1869
  if not isinstance(change, dict) or not _name_of(change):
1387
1870
  continue
1388
1871
  name, kind = _name_of(change), change.get("change_type")
1389
- index = pmap.find(name, path, _line_of(change.get("old_line")))
1872
+ # a move across files (a MOVED_EXPORT, or a MODIFIED_EXPORT that also moved) names the file its entry is at
1873
+ index = pmap.find(name, _norm_path(change.get("old_file")) or path, _line_of(change.get("old_line")))
1390
1874
  if kind == "DELETED_EXPORT":
1391
1875
  if index is not None:
1392
1876
  pmap.remove([index])
@@ -1395,9 +1879,24 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
1395
1879
  line = record["source_line"] if record and record.get("source_line") is not None \
1396
1880
  else _line_of(change.get("new_line"))
1397
1881
  if index is not None and line is not None:
1398
- pmap.update(index, {"source_file": path, "source_line": line})
1882
+ fields = {"source_file": path, "source_line": line}
1883
+ if record is not None and record["extraction_method"] == "ast-grep":
1884
+ # the runner matched it: its labels, never its export_type, params or return_type
1885
+ fields.update(confidence=record["confidence"], extraction_method=record["extraction_method"])
1886
+ if record["ast_node_type"] is not None:
1887
+ fields["ast_node_type"] = record["ast_node_type"]
1888
+ entry = pmap.doc["entries"][index]
1889
+ # its signature_source only when the record holds each signature part the entry does
1890
+ if all(record.get(part) is not None for part in ("params", "return_type")
1891
+ if entry.get(part) is not None):
1892
+ fields["signature_source"] = record["signature_source"]
1893
+ elif _tier(entry.get("signature_source")) is None:
1894
+ # the signature it keeps was read by eye: set that, or the backfill would derive T1
1895
+ # from the ast-grep label set here
1896
+ fields["signature_source"] = "T1-low"
1897
+ pmap.update(index, fields)
1399
1898
  elif kind in ("NEW_EXPORT", "MODIFIED_EXPORT"):
1400
- refresh(index, name, path, add=True)
1899
+ refresh(index, name, path, add=True, old=_norm_path(change.get("old_file")))
1401
1900
  for rename in _get_list(categories or {}, "category_c", "renamed_exports"):
1402
1901
  if not isinstance(rename, dict):
1403
1902
  continue
@@ -1434,7 +1933,7 @@ def _apply_docs_only(pmap: _Map, records: dict) -> tuple[int, int]:
1434
1933
  "confidence": "T3", "signature_source": "T3"}
1435
1934
  carried = old.get(_name_of(record))
1436
1935
  entry = _rewrite(carried, _without_none(fields)) if carried else _new_entry(
1437
- _without_none(fields), pmap.library)
1936
+ _without_none(fields), pmap.library_for(url, _name_of(record)))
1438
1937
  pmap.add(entry)
1439
1938
  return len(changed), len(exports)
1440
1939
 
@@ -1444,8 +1943,9 @@ def _drift_note(drift: tuple | None) -> str:
1444
1943
 
1445
1944
 
1446
1945
  def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, drift: tuple | None,
1447
- warnings: list[str]) -> tuple[int, int] | list[dict]:
1448
- """Gap-driven mode: one write per verification record. A list return is the refusal."""
1946
+ warnings: list[str], relabeled: list[dict]) -> tuple[int, int] | list[dict]:
1947
+ """Gap-driven mode: one write per verification record. A list return is the refusal; each entry whose
1948
+ signature_source the repair changes goes to `relabeled`."""
1449
1949
  verification = [r for r in records.get("verification") or [] if isinstance(r, dict) and _name_of(r)]
1450
1950
  reextracted = _Fresh(records.get("files") or [], None, None)
1451
1951
  merged = {}
@@ -1456,18 +1956,108 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
1456
1956
  def in_map(r: dict) -> bool:
1457
1957
  return bool(r.get("in_map")) or bool(r.get("map_entry"))
1458
1958
 
1959
+ def signature(r: dict) -> tuple[dict, tuple]:
1960
+ """(the signature parts to set, the parts to remove) of an export: with the record §4a's extraction gave at
1961
+ its line (its verification record has an extraction_method), every part from it, a null params or
1962
+ return_type removed; with none, merge's."""
1963
+ if r.get("extraction_method") is not None:
1964
+ return _without_none({part: r.get(part) for part in SIGNATURE_PARTS}), \
1965
+ tuple(part for part in ("params", "return_type") if r.get(part) is None)
1966
+ extra = merged.get(_name_of(r), {})
1967
+ return _without_none({part: extra.get(part) for part in SIGNATURE_PARTS}), ()
1968
+
1969
+ def resign(r: dict, index: int, fields: dict) -> None:
1970
+ """Write an entry the map holds with `fields` and, when §4a found the export's record, its signature; an
1971
+ ast-grep entry takes the record's signature_source, and its node kind when the record's export_type is
1972
+ another (a source-read entry keeps T1-low and no kind); confidence and extraction_method stay."""
1973
+ entry = pmap.doc["entries"][index]
1974
+ drop: tuple = ()
1975
+ method = r.get("extraction_method")
1976
+ if r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT") and method is not None \
1977
+ and r.get("reachability") != "internal-unreachable":
1978
+ parts, gone = signature(r)
1979
+ fields = {**fields, **parts}
1980
+ drop = tuple(part for part in gone if entry.get(part) is not None)
1981
+ if entry.get("extraction_method") in ("ast-grep", "ast_bridge"):
1982
+ fields["signature_source"] = _labels(method, None)["signature_source"]
1983
+ if parts.get("export_type") not in (None, entry.get("export_type")):
1984
+ fields["ast_node_type"] = r.get("ast_node_type")
1985
+ old, new = entry.get("signature_source"), fields["signature_source"]
1986
+ if old is not None and old != new:
1987
+ path = _file_of(entry)
1988
+ relabeled.append({"name": _name_of(entry), "file": path, "from": old, "to": new})
1989
+ warnings.append(f"provenance: {_name_of(entry)} in {path}: signature_source {old} relabeled "
1990
+ f"{new}, from the {method} record of its signature")
1991
+ pmap.update(index, fields, drop)
1992
+
1993
+ note = _drift_note(drift)
1994
+
1995
+ def why(r: dict) -> str:
1996
+ """The cause of an `unknown` outcome its record gives, `<unknown_reason>: <words>`, else ''."""
1997
+ reason = r.get("unknown_reason")
1998
+ return f"{reason}: {UNKNOWN_REASONS[reason]}" if r.get("verification") == "unknown" \
1999
+ and reason in UNKNOWN_REASONS else ""
2000
+
2001
+ def unchecked(r: dict) -> str | None:
2002
+ """The warning for a blocking NEW_EXPORT or MODIFIED_EXPORT the map holds that its spot-check could not
2003
+ locate (missing, or unknown for a cause its unknown_reason names), which stays unchanged: the gap, the
2004
+ export, the outcome, its cause and the one fix a re-run acts on; None for any other record."""
2005
+ outcome = r.get("verification")
2006
+ if outcome not in ("missing", "unknown") or not in_map(r) or not _blocking(r.get("severity")) \
2007
+ or r.get("gap_category") not in ("NEW_EXPORT", "MODIFIED_EXPORT"):
2008
+ return None
2009
+ name, gap = _name_of(r), r.get("gap_id")
2010
+ path = _location(r.get("map_entry"))[0]
2011
+ cause = f"its map entry's file{f' {path}' if path else ''} is gone" if outcome == "missing" else why(r)
2012
+ detail = "; ".join(part for part in (cause, note) if part)
2013
+ if r.get("unknown_reason") == "ambiguous": # a Source at the definition lets the re-run's lookup pick one
2014
+ fix = ("its entries are left unchanged. Fix its entries by hand, or give the gap a `file:line` Source at "
2015
+ "its definition when that file holds only one of them, then re-run update-skill")
2016
+ else: # the re-run checks the same map entry again: only a hand fix changes it
2017
+ fix = ("its entry is left unchanged. Fix the entry by hand (set its source_file and source_line to the "
2018
+ "line that defines it), then re-run update-skill")
2019
+ return f"provenance: {name}{f' ({gap})' if gap else ''}: {outcome}{f' ({detail})' if detail else ''}; {fix}"
2020
+
2021
+ def elsewhere(r: dict) -> str | None:
2022
+ """The warning for a NEW_EXPORT or MODIFIED_EXPORT whose gap cites another file than the map's entries of its
2023
+ name (unknown, cited-elsewhere), whatever its severity: the gap, the export, the cited file and its check's
2024
+ cause, the record's map_locations and the fix for each side that may be wrong; None for any other record."""
2025
+ if r.get("verification") != "unknown" or r.get("unknown_reason") != "cited-elsewhere":
2026
+ return None
2027
+ name, gap = _name_of(r), r.get("gap_id")
2028
+ cited_file, cited_line = _location(r.get("source_citation"))
2029
+ cited = _where(cited_file, cited_line) or cited_file
2030
+ check = r.get("citation_check") if isinstance(r.get("citation_check"), str) else None
2031
+ words = "no such file" if check == "missing" else UNKNOWN_REASONS.get(check or "")
2032
+ where = [w for w in r.get("map_locations") or [] if isinstance(w, str) and w]
2033
+ detail = "; ".join(part for part in (
2034
+ why(r), f"it cites {cited}" if cited else "",
2035
+ f"its check there: {check}{f' ({words})' if words else ''}" if check else "",
2036
+ f"the map holds it at {', '.join(where)}" if where else "", note) if part)
2037
+ several = len(where) > 1
2038
+ kept = "its entries are left unchanged" if several else "its entry is left unchanged"
2039
+ head = f"provenance: {name}{f' ({gap})' if gap else ''}: unknown ({detail}); {kept}, and nothing is " \
2040
+ "re-extracted for it."
2041
+ if check in UNREAD: # nothing was read at the cited file: make the check read it first
2042
+ return f"{head} Nothing was read at the cited file: {UNREAD[check]}, then re-run update-skill"
2043
+ moved = "when an export moved, fix by hand whichever of those entries moved" if several \
2044
+ else "when the export moved, fix the map entry by hand"
2045
+ scope = "" if check == "missing" else (", and if the gap comes back, exclude the cited file in the brief's "
2046
+ "scope when the skill does not cover it")
2047
+ return (f"{head} When the map is right, re-run Test Skill, since the test compared another file's "
2048
+ f"declaration{scope}; {moved} (set its source_file and source_line to the line that defines it), "
2049
+ f"then re-run update-skill")
2050
+
1459
2051
  blocking = [{"export_name": _name_of(r), "severity": r.get("severity")} for r in verification
1460
2052
  if r.get("verification") == "unknown" and not in_map(r)
1461
- and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT")
1462
- and str(r.get("severity") or "").strip().lower() not in NON_BLOCKING]
2053
+ and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT") and _blocking(r.get("severity"))]
1463
2054
  if blocking:
1464
2055
  return blocking
1465
- note = _drift_note(drift)
1466
2056
  pinned_keys: set[tuple] = set()
1467
2057
  for r in verification:
1468
2058
  name, outcome, category = _name_of(r), r.get("verification"), r.get("gap_category")
1469
2059
  entry_file, entry_line = _location(r.get("map_entry"))
1470
- index = pmap.find(name, entry_file, entry_line) if r.get("map_entry") else None
2060
+ index = pmap.find(name, entry_file, entry_line, reexport=True) if r.get("map_entry") else None
1471
2061
  lines = r.get("pinned_definition_lines")
1472
2062
  r5 = f"; definition lines per the test report: {lines}" if category == "MOVED_EXPORT" else ""
1473
2063
  if outcome == "rescoped":
@@ -1476,10 +2066,10 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
1476
2066
  elif outcome == "verified" and index is not None:
1477
2067
  if drift and category == "MOVED_EXPORT":
1478
2068
  warnings.append(f"provenance: {name}: verified at HEAD only ({note}{r5})")
2069
+ resign(r, index, {})
1479
2070
  elif outcome == "moved" and index is not None:
1480
2071
  path, line = _location(r.get("new_location"))
1481
- if path and line is not None:
1482
- pmap.update(index, {"source_file": path, "source_line": line})
2072
+ resign(r, index, {"source_file": path, "source_line": line} if path and line is not None else {})
1483
2073
  elif outcome == "re-extracted":
1484
2074
  if r.get("reachability") == "internal-unreachable":
1485
2075
  continue # not public API: merge queued it as a metadata update
@@ -1488,9 +2078,13 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
1488
2078
  if record is None:
1489
2079
  warnings.append(f"provenance: {name}: re-extracted, but no record of it in the run's files")
1490
2080
  continue
1491
- existing = pmap.find(name, path)
2081
+ local = (reextracted.worker.get((name, path)) or {}).get("local")
2082
+ if isinstance(local, str) and local and local != name:
2083
+ # matched by the public name a re-export gives it: the entry keeps its declared name
2084
+ record = {**record, "export_name": local, "reexported_as": name}
2085
+ existing = pmap.find(name, path, reexport=True)
1492
2086
  if existing is None:
1493
- pmap.add(_new_entry(record, pmap.library))
2087
+ pmap.add(_new_entry(record, pmap.library_for(path, name)))
1494
2088
  else:
1495
2089
  pmap.update(existing, _without_none(record, keep=("ast_node_type",)))
1496
2090
  elif outcome in ("verified", "moved") and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
@@ -1498,29 +2092,28 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
1498
2092
  continue
1499
2093
  where = r.get("source_citation") if outcome == "verified" else r.get("new_location")
1500
2094
  path, line = _location(where)
1501
- if not path or line is None or (name, path) in pinned_keys or pmap.find(name, path) is not None:
2095
+ if not path or line is None or (name, path) in pinned_keys \
2096
+ or pmap.find(name, path, reexport=True) is not None:
1502
2097
  continue
1503
2098
  pinned_keys.add((name, path))
1504
- extra = merged.get(name, {})
2099
+ # the signature its record gives, else merge's; the source-read labels whatever the record's tool
1505
2100
  pmap.add(_new_entry(_without_none({
1506
- "export_name": name,
1507
- "export_type": extra.get("export_type") or r.get("export_type"),
1508
- "params": extra.get("params"), "return_type": extra.get("return_type"),
2101
+ "export_name": name, **signature(r)[0],
1509
2102
  "source_file": path, "source_line": line, **_labels("source-read", None)},
1510
- keep=("ast_node_type",)), pmap.library))
2103
+ keep=("ast_node_type",)), pmap.library_for(path, name)))
1511
2104
  elif outcome == "unknown" and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
1512
- extra = merged.get(name, {})
1513
2105
  pmap.add(_new_entry(_without_none({
1514
- "export_name": name, "export_type": extra.get("export_type") or r.get("export_type"),
1515
- "params": extra.get("params"), "return_type": extra.get("return_type"),
1516
- **_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library))
2106
+ "export_name": name, **signature(r)[0],
2107
+ **_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library_for(None, name)))
1517
2108
  else:
1518
2109
  if r.get("unknown_reason") == "drift-override":
1519
2110
  warnings.append(f"provenance: {name}: unknown ({note}; no line taken from HEAD{r5})")
2111
+ elif (warning := elsewhere(r) or unchecked(r)) is not None:
2112
+ warnings.append(warning)
1520
2113
  elif drift:
1521
- warnings.append(f"provenance: {name}: {outcome} ({note}{r5})")
2114
+ warnings.append(f"provenance: {name}: {outcome} ({why(r) + '; ' if why(r) else ''}{note}{r5})")
1522
2115
  else:
1523
- warnings.append(f"provenance: {name}: {outcome}{r5}")
2116
+ warnings.append(f"provenance: {name}: {outcome}{f' ({why(r)})' if why(r) else ''}{r5}")
1524
2117
  return len(pmap.touched_files), len(verification)
1525
2118
 
1526
2119
 
@@ -1583,14 +2176,165 @@ def _apply_files(pmap: _Map, compare: dict | None, new_files: dict | None, promo
1583
2176
  return done
1584
2177
 
1585
2178
 
2179
+ REFERENCE_APP = "reference-app"
2180
+
2181
+
2182
+ def _export_name(item: object) -> str | None:
2183
+ """The name of a metadata.json exports[] item: a string, or an object's name or export_name."""
2184
+ if isinstance(item, str):
2185
+ return item or None
2186
+ return _name_of(item) if isinstance(item, dict) else None
2187
+
2188
+
2189
+ def exports_after(metadata: object, pmap: _Map, update_type: str) -> list:
2190
+ """metadata.json's exports[] after the update (see "Exports" in the module docstring)."""
2191
+ if not isinstance(metadata, dict):
2192
+ raise ValueError("--metadata-in file must hold a JSON object (the skill's metadata.json)")
2193
+ if metadata.get("scope_type") == REFERENCE_APP:
2194
+ return []
2195
+ current = metadata.get("exports") if isinstance(metadata.get("exports"), list) else []
2196
+ held = {_name_of(e) for e in pmap.doc["entries"] if isinstance(e, dict)} - {None}
2197
+ # an entry added under its declared name, re-exported under a public one: exports[] lists the public name
2198
+ public = {_name_of(e): e["reexported_as"] for e in pmap.doc["entries"]
2199
+ if isinstance(e, dict) and isinstance(e.get("reexported_as"), str) and e["reexported_as"]}
2200
+ renamed = dict(pmap.renamed)
2201
+ gone = ({n for n in pmap.removed if n} | set(renamed)) - held
2202
+ if update_type == "full":
2203
+ gone |= {n for n in map(_export_name, current) if n and n not in held}
2204
+ out: list = []
2205
+ seen: set[str] = set()
2206
+
2207
+ def put(item: object, name: str | None) -> None:
2208
+ if name and name not in seen:
2209
+ seen.add(name)
2210
+ out.append(item)
2211
+
2212
+ for item in current:
2213
+ name = _export_name(item)
2214
+ if name in gone:
2215
+ new = renamed.get(name)
2216
+ put(new, new if new in held else None)
2217
+ else:
2218
+ put(item, name)
2219
+ for name in [public.get(n, n) for n in pmap.added] + [new for _old, new in pmap.renamed]:
2220
+ put(name, name if name in held or name in public.values() else None)
2221
+ return out
2222
+
2223
+
2224
+ _DETECTOR = None
2225
+
2226
+
2227
+ def _detector():
2228
+ """skf-detect-scripts-assets.py, loaded once from this folder: the purpose a new script's or asset's row
2229
+ takes, read from the file as create-skill's detector reads it."""
2230
+ global _DETECTOR
2231
+ if _DETECTOR is None:
2232
+ path = Path(__file__).resolve().parent / "skf-detect-scripts-assets.py"
2233
+ spec = importlib.util.spec_from_file_location("skf_detect_scripts_assets", path)
2234
+ if spec is None or spec.loader is None or not path.is_file():
2235
+ raise ValueError(f"cannot load {path.name} beside {Path(__file__).name}: re-install SKF")
2236
+ module = importlib.util.module_from_spec(spec)
2237
+ spec.loader.exec_module(module)
2238
+ _DETECTOR = module
2239
+ return _DETECTOR
2240
+
2241
+
2242
+ def _purpose(source_file: str, kind: str, source_root: Path | None) -> str:
2243
+ """A new script's or asset's purpose: the detector's, read from the file under `source_root`, else its name."""
2244
+ path = source_root / source_file if source_root is not None else None
2245
+ if path is None or not path.is_file():
2246
+ return Path(source_file).name
2247
+ detector = _detector()
2248
+ hint = detector.asset_type(path, source_root) if kind == "asset" else None
2249
+ return detector.extract_purpose(path, asset_type_hint=hint)
2250
+
2251
+
2252
+ def file_rows_after(metadata: dict, pmap: _Map, source_root: Path | None) -> dict[str, list]:
2253
+ """metadata.json's scripts[] and assets[] after the update (see "Lists" in the module docstring)."""
2254
+ by_source: dict[str, dict] = {}
2255
+ by_file: dict[str, dict] = {}
2256
+ for key in ("scripts", "assets"):
2257
+ rows = metadata.get(key)
2258
+ for row in rows if isinstance(rows, list) else []:
2259
+ if not isinstance(row, dict):
2260
+ continue
2261
+ source = _norm_path(row.get("source_file"))
2262
+ if source:
2263
+ by_source.setdefault(source, row)
2264
+ if isinstance(row.get("file"), str):
2265
+ by_file.setdefault(row["file"], row)
2266
+ lists: dict[str, list] = {"scripts": [], "assets": []}
2267
+ entries = pmap.doc.get("file_entries")
2268
+ for entry in entries if isinstance(entries, list) else []:
2269
+ if not isinstance(entry, dict) or _file_type(entry) not in ("script", "asset"):
2270
+ continue
2271
+ kind = _file_type(entry)
2272
+ source = _norm_path(entry.get("source_file"))
2273
+ held = by_source.get(source) if source else None
2274
+ if held is None and isinstance(entry.get("file_name"), str):
2275
+ held = by_file.get(entry["file_name"])
2276
+ if held is None:
2277
+ held = {"file": entry.get("file_name") or f"{kind}s/{Path(source or '').name}",
2278
+ "purpose": _purpose(source, kind, source_root) if source else "",
2279
+ "source_file": source, "confidence": entry.get("confidence") or "T1-low"}
2280
+ lists[f"{kind}s"].append(held)
2281
+ return lists
2282
+
2283
+
2284
+ # the signature_source tiers skf-render-metadata-stats.py bins, in their spelling (it compares them case-insensitively)
2285
+ SIGNATURE_SOURCES = ("T1", "T1-low", "T2", "T3")
2286
+ WARNING_LIST_CAP = 10 # the items a warning lists before "and N more"
2287
+
2288
+
2289
+ def _capped(items: list[str]) -> str:
2290
+ """The items joined with ", ", the first WARNING_LIST_CAP of them and "and N more" for the rest."""
2291
+ more = len(items) - WARNING_LIST_CAP
2292
+ return ", ".join(items[:WARNING_LIST_CAP] + ([f"and {more} more"] if more > 0 else []))
2293
+
2294
+
2295
+ def _tier(value: object) -> str | None:
2296
+ """A signature_source or confidence tier in its SIGNATURE_SOURCES spelling; None for any other value."""
2297
+ if not isinstance(value, str):
2298
+ return None
2299
+ return next((tier for tier in SIGNATURE_SOURCES if tier.upper() == value.strip().upper()), None)
2300
+
2301
+
2302
+ def _backfill_signature_source(pmap: _Map) -> tuple[dict[str, int], list[tuple[object, object, str]]]:
2303
+ """Give each entry whose signature_source is missing or not a tier the stats helper bins (a map written before
2304
+ the key existed, #802) the one its labels give: its confidence when that names T1-low, T2 or T3, else T1 for
2305
+ an extraction_method of ast-grep or ast_bridge (compared stripped and case-insensitively) and T1-low for any
2306
+ other (relabel-rule.md: T1 needs evidence an ast-grep rule matched). Returns (the fills counted by value, in
2307
+ tier order; (name, old value, new value) of each unrecognized value replaced, which the map no longer holds);
2308
+ the names are not listed as updated."""
2309
+ counts: dict[str, int] = {}
2310
+ replaced: list[tuple[object, object, str]] = []
2311
+ for entry in pmap.doc["entries"]:
2312
+ if not isinstance(entry, dict) or _tier(entry.get("signature_source")) is not None:
2313
+ continue
2314
+ confidence, method = _tier(entry.get("confidence")), entry.get("extraction_method")
2315
+ if confidence in ("T1-low", "T2", "T3"):
2316
+ value = confidence
2317
+ elif isinstance(method, str) and method.strip().lower() in ("ast-grep", "ast_bridge"):
2318
+ value = "T1"
2319
+ else:
2320
+ value = "T1-low"
2321
+ if entry.get("signature_source") is not None:
2322
+ replaced.append((entry.get("export_name"), entry["signature_source"], value))
2323
+ entry["signature_source"] = value
2324
+ counts[value] = counts.get(value, 0) + 1
2325
+ return {tier: counts[tier] for tier in SIGNATURE_SOURCES if tier in counts}, replaced
2326
+
2327
+
1586
2328
  def apply_update(*, update_type: str, provenance: dict | None, skill_name: str, manifest: dict | None = None,
1587
2329
  categories: dict | None = None, extraction: dict | None = None, details: dict | None = None,
1588
2330
  records: dict | None = None, merge_records: dict | None = None, compare: dict | None = None,
1589
2331
  new_files: dict | None = None, promoted: list | None = None, source_root: Path | None = None,
1590
2332
  generation_date: str, test_report_run_id: str | None, confidence_tier: str,
1591
2333
  manual_sections_preserved: int, source_commit: str | None = None,
1592
- source_ref: str | None = None, drift: tuple | None = None) -> tuple[dict, dict]:
1593
- """(new map, summary) of one update (see "Apply" in the module docstring).
2334
+ source_ref: str | None = None, drift: tuple | None = None,
2335
+ metadata: dict | None = None) -> tuple[dict, dict]:
2336
+ """(new map, summary) of one update (see "Apply" in the module docstring). With `metadata` (the skill's
2337
+ metadata.json), the summary's `lists` holds the exports[], scripts[] and assets[] it takes after the update.
1594
2338
 
1595
2339
  A refusal returns ({}, summary) with status "refused" and no map.
1596
2340
  Raises ValueError for inputs it cannot read.
@@ -1602,9 +2346,11 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
1602
2346
  records = records if isinstance(records, dict) else {}
1603
2347
  pmap = _Map(provenance, skill_name)
1604
2348
  warnings: list[str] = []
2349
+ not_public: list[dict] = []
2350
+ relabeled: list[dict] = []
1605
2351
  mode = records.get("mode")
1606
2352
  if update_type == "gap-driven":
1607
- outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings)
2353
+ outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings, relabeled)
1608
2354
  if isinstance(outcome, list):
1609
2355
  return {}, {"status": "refused", "blocking_unresolved": outcome}
1610
2356
  files, exports = outcome
@@ -1617,13 +2363,35 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
1617
2363
  index = pmap.find(name, path)
1618
2364
  record = fresh.record(name, path)
1619
2365
  if index is None:
1620
- pmap.add(_new_entry(record, pmap.library))
2366
+ pmap.add(_new_entry(record, pmap.library_for(path, name)))
1621
2367
  else:
1622
2368
  pmap.update(index, _without_none(record, keep=("ast_node_type",)))
1623
2369
  files = len({path for _name, path in fresh.all_keys()})
1624
2370
  exports = len(fresh.all_keys())
1625
2371
  else:
1626
- files, exports = _apply_normal(pmap, manifest, categories, fresh, warnings)
2372
+ files, exports = _apply_normal(pmap, manifest, categories, fresh, warnings, not_public)
2373
+ if fresh.surface_problem:
2374
+ warnings.append(f"provenance: the public surface of this public-api skill could not be applied "
2375
+ f"({fresh.surface_problem}): every new export was added")
2376
+ if not_public:
2377
+ warnings.append(_not_public_warning(not_public))
2378
+ if pmap.library_unresolved:
2379
+ count = len(pmap.library_unresolved)
2380
+ named = _capped([f"{name} in {path}" if path else f"{name} (no file)"
2381
+ for name, path in pmap.library_unresolved])
2382
+ warnings.append(f"provenance: {count} new entr{'y' if count == 1 else 'ies'} took source_library "
2383
+ f"{pmap.library}, the map's first, as the map names several libraries "
2384
+ f"({', '.join(pmap.libraries)}) and each entry's file or folder does not settle which: "
2385
+ f"{named}; check each entry's source_library")
2386
+ backfilled, replaced = _backfill_signature_source(pmap)
2387
+ if backfilled:
2388
+ count = sum(backfilled.values())
2389
+ listed = [f"{name}: {old if isinstance(old, str) else json.dumps(old)} -> {new}" for name, old, new in replaced]
2390
+ values = f"; unrecognized values replaced: {_capped(listed)}" if replaced else ""
2391
+ warnings.append(f"provenance: backfilled signature_source on {count} entr{'y' if count == 1 else 'ies'} "
2392
+ f"that had none or an unrecognized value (a map written before the key existed), from each "
2393
+ f"one's confidence and extraction_method: "
2394
+ f"{', '.join(f'{n} {value}' for value, n in backfilled.items())}{values}")
1627
2395
  file_changes = _apply_files(pmap, compare, new_files, promoted, source_root)
1628
2396
  doc = pmap.doc
1629
2397
  for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
@@ -1643,12 +2411,168 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
1643
2411
  summary = {
1644
2412
  "status": "written",
1645
2413
  "entries": {"added": pmap.added, "updated": pmap.updated, "removed": pmap.removed},
2414
+ "not_public": not_public,
2415
+ "relabeled": relabeled,
2416
+ "signature_source_backfilled": backfilled,
1646
2417
  "file_entries": file_changes,
1647
2418
  "warnings": warnings,
1648
2419
  }
2420
+ if metadata is not None:
2421
+ summary["lists"] = {"exports": exports_after(metadata, pmap, update_type),
2422
+ **file_rows_after(metadata, pmap, source_root)}
1649
2423
  return doc, summary
1650
2424
 
1651
2425
 
2426
+ GENERATION_DATE_RE = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ")
2427
+ # the stats fields skf-render-metadata-stats.py writes, some only for a shape or a judgment
2428
+ STATS_HELPER_FIELDS = ("exports_documented", "exports_public_api", "exports_internal", "exports_total",
2429
+ "public_api_coverage", "total_coverage", "scripts_count", "assets_count",
2430
+ "pattern_surfaces_documented", "effective_denominator")
2431
+ # the other stats fields create-skill writes, by the skill's scope_type: a component library's own counts (the
2432
+ # Component Library stats of compile-assembly-rules.md, its stats_extra) and a reference app's notes
2433
+ STATS_CREATE_FIELDS = {
2434
+ "component-library": ("components_registered", "components_documented", "props_interfaces_extracted",
2435
+ "components_unique", "demo_files_excluded", "design_variants"),
2436
+ "reference-app": ("notes",),
2437
+ }
2438
+ # a stats key that explains effective_denominator, dropped when this run's denominator differs from the old one
2439
+ DENOMINATOR_NOTE_PREFIX = "effective_denominator_"
2440
+ SKF_NO_ROOT = ("metadata: tool_versions.skf is unknown: no --skf-root was given; pass the installed SKF's folder "
2441
+ "({project-root}/_bmad/skf)")
2442
+ AST_GREP_UNKNOWN = ("metadata: tool_versions.ast_grep is unknown: --extraction holds exports ast-grep matched but no "
2443
+ "ast_grep.version")
2444
+ _QMD_PREFIX = re.compile(r"^qmd\s+", re.IGNORECASE) # what `qmd --version` prints first
2445
+
2446
+
2447
+ def _tool_version(value: object) -> str | None:
2448
+ return value.strip() if isinstance(value, str) and value.strip() else None
2449
+
2450
+
2451
+ def _skf_unknown(skf_root: str | Path) -> str:
2452
+ return (f"metadata: tool_versions.skf is unknown: {Path(skf_root).as_posix()} holds no package.json or VERSION "
2453
+ f"file with a version")
2454
+
2455
+
2456
+ def _kept_warning(key: str) -> str:
2457
+ return f"metadata: stats.{key} kept from an earlier run; check it still describes this run's counts"
2458
+
2459
+
2460
+ def tool_versions_after(old: object, *, skf_version: str | None, skf_root: str | Path | None,
2461
+ ast_grep_version: str | None, qmd_version: str | None) -> tuple[dict, list[str]]:
2462
+ """(the tool_versions an update leaves, its warnings) (see "Metadata" in the module docstring)."""
2463
+ warnings: list[str] = []
2464
+ if isinstance(old, dict):
2465
+ tools = dict(old)
2466
+ else:
2467
+ tools = {"ast_grep": None, "qmd": None}
2468
+ if old is not None:
2469
+ kind = {bool: "a boolean", str: "a string", list: "an array"}.get(type(old), "a number")
2470
+ warnings.append(f"metadata: tool_versions was {kind}, not an object; replaced with the versions this run "
2471
+ f"records")
2472
+ ast_grep = _tool_version(ast_grep_version)
2473
+ if ast_grep is not None:
2474
+ tools["ast_grep"] = ast_grep
2475
+ if ast_grep == "unknown":
2476
+ warnings.append(AST_GREP_UNKNOWN)
2477
+ qmd = _tool_version(qmd_version)
2478
+ if qmd is not None:
2479
+ qmd = _tool_version(_QMD_PREFIX.sub("", qmd, count=1))
2480
+ if qmd is not None:
2481
+ tools["qmd"] = qmd
2482
+ skf = _tool_version(skf_version)
2483
+ if skf is None or skf == "unknown":
2484
+ skf = "unknown"
2485
+ warnings.append(SKF_NO_ROOT if skf_root is None else _skf_unknown(skf_root))
2486
+ tools["skf"] = skf
2487
+ return tools, warnings
2488
+
2489
+
2490
+ def stats_after(old_stats: dict, new_stats: dict, scope_type: object) -> tuple[dict, list[str], list[str]]:
2491
+ """(the stats an update leaves, dropped_stats, their warnings) (see "Metadata" in the module docstring)."""
2492
+ exempt = STATS_CREATE_FIELDS.get(scope_type, ()) if isinstance(scope_type, str) else ()
2493
+ denominator = new_stats.get("effective_denominator")
2494
+ recounted = old_stats.get("effective_denominator") != denominator
2495
+ kept: dict = {}
2496
+ dropped: list[str] = []
2497
+ warnings: list[str] = []
2498
+ for key, value in old_stats.items():
2499
+ if key in new_stats:
2500
+ continue
2501
+ if key in STATS_HELPER_FIELDS:
2502
+ dropped.append(key)
2503
+ warnings.append(f"metadata: stats.{key} dropped: this run's stats helper did not write it")
2504
+ continue
2505
+ if key.startswith(DENOMINATOR_NOTE_PREFIX) and recounted:
2506
+ dropped.append(key)
2507
+ warnings.append(f"metadata: stats.{key} dropped: it explained an effective_denominator this run changed "
2508
+ f"or removed")
2509
+ continue
2510
+ kept[key] = value
2511
+ # an explanation of a denominator this run left as it was still describes it
2512
+ if key not in exempt and not (key.startswith(DENOMINATOR_NOTE_PREFIX) and denominator is not None):
2513
+ warnings.append(_kept_warning(key))
2514
+ return {**new_stats, **kept}, dropped, warnings
2515
+
2516
+
2517
+ def update_metadata(metadata: object, lists: object, stats: object, *, version: str, generation_date: str,
2518
+ source_commit: str | None = None, source_ref: str | None = None,
2519
+ skf_version: str | None = None, skf_root: str | Path | None = None,
2520
+ ast_grep_version: str | None = None, qmd_version: str | None = None) -> tuple[dict, dict]:
2521
+ """(the metadata.json an update leaves, {"dropped_stats": [...], "warnings": [...]}) (see "Metadata" in the
2522
+ module docstring). Raises ValueError."""
2523
+ if not isinstance(metadata, dict):
2524
+ raise ValueError("--metadata file must hold a JSON object (the skill's metadata.json)")
2525
+ if not isinstance(lists, dict) or not all(isinstance(lists.get(k), list) for k in ("exports", "scripts",
2526
+ "assets")):
2527
+ raise ValueError("--lists file must hold the exports, scripts and assets lists apply wrote (--lists-out)")
2528
+ if not isinstance(stats, dict) or not isinstance(stats.get("stats"), dict) \
2529
+ or not isinstance(stats.get("confidence_distribution"), dict):
2530
+ raise ValueError("--stats file must hold the stats helper's JSON: a stats and a confidence_distribution "
2531
+ "object")
2532
+ if not version.strip():
2533
+ raise ValueError("--version must not be empty")
2534
+ if not GENERATION_DATE_RE.fullmatch(generation_date):
2535
+ raise ValueError(f"--generation-date must be YYYY-MM-DDTHH:MM:SSZ, read from the clock; got "
2536
+ f"{generation_date!r}")
2537
+ for kind in ("scripts", "assets"):
2538
+ counted = stats["stats"].get(f"{kind}_count")
2539
+ if counted != len(lists[kind]):
2540
+ raise ValueError(f"the stats count {counted} {kind} but apply's lists hold {len(lists[kind])}: run the "
2541
+ f"stats helper again with the lists' {kind}")
2542
+ out = {k: v for k, v in metadata.items() if k not in ("last_update", "update_type")}
2543
+ out["version"] = version
2544
+ out["generation_date"] = generation_date
2545
+ for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
2546
+ if value is not None:
2547
+ out[key] = value or None
2548
+ out["exports"] = lists["exports"]
2549
+ for kind in ("scripts", "assets"):
2550
+ if lists[kind]:
2551
+ out[kind] = lists[kind]
2552
+ else:
2553
+ out.pop(kind, None)
2554
+ out["tool_versions"], warnings = tool_versions_after(
2555
+ metadata.get("tool_versions"), skf_version=skf_version, skf_root=skf_root,
2556
+ ast_grep_version=ast_grep_version, qmd_version=qmd_version)
2557
+ old_stats = metadata.get("stats") if isinstance(metadata.get("stats"), dict) else {}
2558
+ out["stats"], dropped, stats_warnings = stats_after(old_stats, stats["stats"], metadata.get("scope_type"))
2559
+ warnings += stats_warnings
2560
+ out["confidence_distribution"] = stats["confidence_distribution"]
2561
+ return out, {"dropped_stats": dropped, "warnings": warnings}
2562
+
2563
+
2564
+ NOT_PUBLIC_NAMED = 10 # the names the not_public warning spells out; the summary lists them all
2565
+
2566
+
2567
+ def _not_public_warning(not_public: list[dict]) -> str:
2568
+ """The one warning for every name apply left out as off a public-api skill's public surface."""
2569
+ names = [item["name"] for item in not_public]
2570
+ shown = ", ".join(names[:NOT_PUBLIC_NAMED])
2571
+ more = f" and {len(names) - NOT_PUBLIC_NAMED} more" if len(names) > NOT_PUBLIC_NAMED else ""
2572
+ return (f"provenance: {len(names)} new export(s) off the public surface of this public-api skill not added "
2573
+ f"(the summary's not_public lists each with its file): {shown}{more}")
2574
+
2575
+
1652
2576
  def _write_json_atomic(path: Path, value) -> None:
1653
2577
  """Write `value` as indented JSON through a temporary file beside `path` and one rename, retried on Windows,
1654
2578
  where a scanner may hold the target open for a moment."""
@@ -1681,22 +2605,12 @@ def _write_json_atomic(path: Path, value) -> None:
1681
2605
  # a patch sets only where the seed has none.
1682
2606
  PATCH_FIELDS = ("signature", "members", "docstring", "qmd_evidence")
1683
2607
  PATCH_IF_NULL = ("params", "return_type")
1684
- _INVENTORY = None
1685
2608
 
1686
2609
 
1687
2610
  def _typed_param(param, language: object) -> str | None:
1688
- """One parameter as the provenance map writes it: skf-extraction-inventory.py's typed_param, loaded once from
1689
- this folder, so create-skill and update-skill write one form."""
1690
- global _INVENTORY
1691
- if _INVENTORY is None:
1692
- path = Path(__file__).resolve().parent / "skf-extraction-inventory.py"
1693
- spec = importlib.util.spec_from_file_location("skf_extraction_inventory", path)
1694
- if spec is None or spec.loader is None:
1695
- raise ImportError(f"cannot load {path}")
1696
- module = importlib.util.module_from_spec(spec)
1697
- spec.loader.exec_module(module)
1698
- _INVENTORY = module
1699
- return _INVENTORY.typed_param(param, language if isinstance(language, str) else None)
2611
+ """One parameter as the provenance map writes it: skf-extraction-inventory.py's typed_param, so create-skill
2612
+ and update-skill write one form."""
2613
+ return _inventory().typed_param(param, language if isinstance(language, str) else None)
1700
2614
 
1701
2615
 
1702
2616
  def _typed_params(value, language: object) -> list | None:
@@ -1784,6 +2698,25 @@ def build_records(extraction: dict | None, details: dict | None, files: list | N
1784
2698
  if patch.get(field) is not None:
1785
2699
  seeded[key][field] = patch[field]
1786
2700
  fill_if_null(key, patch)
2701
+ diff = extraction.get("entry_point_diff") if isinstance(extraction, dict) else None
2702
+ aliases: dict[tuple, set] = {}
2703
+ rows = [*(diff.get("public") or []), *(diff.get("other_definitions") or [])] if isinstance(diff, dict) else []
2704
+ for item in rows:
2705
+ members = item.get("members") if isinstance(item, dict) and item.get("via") == "namespace" else None
2706
+ for named in [item, *(members if isinstance(members, list) else [])]:
2707
+ if not isinstance(named, dict):
2708
+ continue
2709
+ name, local = named.get("name"), named.get("local")
2710
+ key = (local, _norm_path(named.get("file")))
2711
+ if isinstance(name, str) and name and isinstance(local, str) and local != name and key in seeded:
2712
+ aliases.setdefault(key, set()).add(name) # the public name the runner gives `local` in that file
2713
+ for key, names in aliases.items():
2714
+ seeded[key]["aliases"] = sorted(names)
2715
+ surface, _problem = public_surface(extraction) # apply warns when a public-api run gives no surface
2716
+ for key in seeded if surface is not None else ():
2717
+ mark = surface.public(key[0], key[1], languages.get(key))
2718
+ if mark is not None:
2719
+ seeded[key]["public"] = mark
1787
2720
  paths = list(dict.fromkeys([p for p in (_norm_path(f) for f in files or []) if p]
1788
2721
  + sorted({key[1] for key in seeded})))
1789
2722
  blocks = [{"file_path": path, "exports": [seeded[key] for key in sorted(seeded) if key[1] == path]}
@@ -1797,6 +2730,7 @@ def build_records(extraction: dict | None, details: dict | None, files: list | N
1797
2730
  "T1-low": sum(1 for r in exports if r["confidence"] == "T1-low"),
1798
2731
  "T2": sum(1 for r in exports if r.get("qmd_evidence") is not None),
1799
2732
  },
2733
+ "marked_not_public": sum(1 for r in exports if r.get("public") is False),
1800
2734
  "warnings": warnings,
1801
2735
  }
1802
2736
  return {"mode": "normal", "files": blocks}, summary
@@ -1818,6 +2752,26 @@ DRIFT_IN_MAP = "a signature from the tree"
1818
2752
  REACHABILITY = ("public", "internal-unreachable")
1819
2753
  VERIFIER_MISSING = "provenance: spot-checks not run: skf-verify-provenance-completeness.py is missing; re-install SKF"
1820
2754
  OUTCOMES = ("verified", "moved", "missing", "re-extracted", "rescoped", "unknown")
2755
+ # Why an entry ends `unknown` (its unknown_reason, see "Gap records"), with the words apply's warning gives it.
2756
+ UNKNOWN_REASONS = {
2757
+ "ambiguous": "several map entries of its name",
2758
+ "no-definition": "no line of its file defines it, by the definition-line rules",
2759
+ "several-definitions": "several lines of its file define it, none the recorded one",
2760
+ "unchecked-language": "the definition-line rules cover no language of its file, and no line was answered as "
2761
+ "declaring it",
2762
+ "no-line": "its map entry gives no file or line to check",
2763
+ "spot-check-failed": "the spot-check failed",
2764
+ "verifier-missing": "skf-verify-provenance-completeness.py is missing",
2765
+ "no-source-root": "no source tree to check",
2766
+ "drift-override": "the drift override kept its line from being moved",
2767
+ "cited-elsewhere": "the gap cites another file than the provenance map's entries of its name",
2768
+ }
2769
+ # A cited-elsewhere entry's citation_check that read nothing at the cited file, with the fix that lets a re-run read it.
2770
+ UNREAD = {
2771
+ "no-source-root": "give update-skill the source tree",
2772
+ "verifier-missing": "re-install SKF",
2773
+ "spot-check-failed": "fix what its spot-check failed on, which the `spot-check failed` warning names",
2774
+ }
1821
2775
  _DEFINITION_LINES = re.compile(r"definition lines?\s*\(([\d,\s]+)\)", re.IGNORECASE)
1822
2776
  _VERIFIER = None
1823
2777
 
@@ -1869,40 +2823,48 @@ def _drift_reason(entry: dict) -> str | None:
1869
2823
  return None
1870
2824
  if entry.get("provenance_completeness"):
1871
2825
  return DRIFT_COMPLETENESS
1872
- status = (entry.get("map_match") or {}).get("status")
1873
- return DRIFT_IN_MAP if status in ("found", "ambiguous") else DRIFT_NOT_IN_MAP
2826
+ match = entry.get("map_match") if isinstance(entry.get("map_match"), dict) else {}
2827
+ status = match.get("status")
2828
+ # a not-found with a candidate that has a file is a citation of another file than the map's entries: held (#824)
2829
+ held = status in ("found", "ambiguous") or status == "not-found" and any(
2830
+ isinstance(c, dict) and _norm_path(c.get("source_file")) for c in match.get("candidates") or [])
2831
+ return DRIFT_IN_MAP if held else DRIFT_NOT_IN_MAP
1874
2832
 
1875
2833
 
1876
2834
  def _spot_check(root: Path | None, path: str | None, name: str, line: int | None, export_type: object,
1877
- answers: dict) -> tuple[str, int | None, bool, str | None]:
2835
+ answers: dict) -> tuple[str, int | None, bool, str | None, str | None]:
1878
2836
  """(outcome, the line a `moved` export moves to, whether it waits on a `declaring_line` answer, why the
1879
- check failed)."""
2837
+ check failed, why it is `unknown`: an UNKNOWN_REASONS key)."""
2838
+ if root is None:
2839
+ return "unknown", None, False, None, "no-source-root"
1880
2840
  verifier = _verifier()
1881
- if verifier is None or root is None or not path or line is None:
1882
- return "unknown", None, False, None
2841
+ if verifier is None:
2842
+ return "unknown", None, False, None, "verifier-missing"
2843
+ if not path or line is None:
2844
+ return "unknown", None, False, None, "no-line"
1883
2845
  try:
1884
2846
  report = verifier.definition_lines_report(path, name, root, line,
1885
2847
  export_type if isinstance(export_type, str) else None)
1886
2848
  except Exception as exc: # noqa: BLE001 - a file it cannot read, or any verifier failure: this entry only
1887
- return "unknown", None, False, f"{type(exc).__name__}: {exc}"
2849
+ return "unknown", None, False, f"{type(exc).__name__}: {exc}", "spot-check-failed"
1888
2850
  check = report.get("line_check")
1889
2851
  if check == "file-missing":
1890
- return "missing", None, False, None
2852
+ return "missing", None, False, None, None
1891
2853
  if check == "skipped-export-type":
1892
- return "verified", None, False, None
2854
+ return "verified", None, False, None, None
1893
2855
  if check == "skipped-language":
1894
2856
  if "declaring_line" not in answers:
1895
- return "unknown", None, True, None
2857
+ return "unknown", None, True, None, None
1896
2858
  declared = answers["declaring_line"]
1897
2859
  if declared is None:
1898
- return "unknown", None, False, None
1899
- return ("verified", None, False, None) if declared == line else ("moved", declared, False, None)
2860
+ return "unknown", None, False, None, "unchecked-language"
2861
+ return ("verified", None, False, None, None) if declared == line else ("moved", declared, False, None, None)
1900
2862
  if report.get("line_is_definition"):
1901
- return "verified", None, False, None
2863
+ return "verified", None, False, None, None
1902
2864
  defs = report.get("definition_lines") or []
1903
2865
  if len(defs) == 1 and defs[0] != line:
1904
- return "moved", defs[0], False, None
1905
- return "unknown", None, False, None
2866
+ return "moved", defs[0], False, None, None
2867
+ return "unknown", None, False, None, "several-definitions" if defs else "no-definition"
1906
2868
 
1907
2869
 
1908
2870
  def _pinned_lines(remediation: object) -> list[int] | None:
@@ -1912,6 +2874,18 @@ def _pinned_lines(remediation: object) -> list[int] | None:
1912
2874
  return lines or None
1913
2875
 
1914
2876
 
2877
+ def _inside(root: Path | None, path: str | None) -> bool:
2878
+ """True for a relative path that stays under the source root, links resolved."""
2879
+ rel = Path(path) if path else None
2880
+ if root is None or rel is None or rel.is_absolute() or ".." in rel.parts:
2881
+ return False
2882
+ try:
2883
+ (root / rel).resolve().relative_to(root.resolve())
2884
+ except (OSError, ValueError):
2885
+ return False
2886
+ return True
2887
+
2888
+
1915
2889
  def _where(path: object, line: object) -> str | None:
1916
2890
  file, number = _norm_path(path), _line_of(line)
1917
2891
  return f"{file}:{number}" if file and number is not None else None
@@ -1931,7 +2905,14 @@ class _GapEntry:
1931
2905
  self.map_entry = {"source_file": found.get("source_file"), "source_line": found.get("source_line")} \
1932
2906
  if found else None
1933
2907
  self.map_export_type = found.get("export_type") if found else None
2908
+ # the map entry's own name when skf-parse-gaps matched it by its reexported_as (the gap names the public one)
2909
+ self.declared = _name_of(found) if found and _name_of(found) else self.name
1934
2910
  self.in_map = bool(match.get("candidates"))
2911
+ # where map_match's candidates sit, `file:line` (or the file); a placeholder with no file names none
2912
+ self.map_locations = list(dict.fromkeys(
2913
+ _where(c.get("source_file"), c.get("source_line")) or _norm_path(c.get("source_file"))
2914
+ for c in match.get("candidates") or [] if isinstance(c, dict) and _norm_path(c.get("source_file"))))
2915
+ self.citation_check: str | None = None # a cited-elsewhere entry's citation check: missing, or its reason
1935
2916
  citation = entry.get("source_citation")
1936
2917
  self.citation = citation if isinstance(citation, dict) else None
1937
2918
  self.recorded: tuple[str | None, int | None] = (None, None) # the file and line a check read
@@ -1939,8 +2920,9 @@ class _GapEntry:
1939
2920
  self.new_line: int | None = None
1940
2921
  self.unknown_reason: str | None = None
1941
2922
  self.reextract = False
1942
- self.record: dict | None = None # the §4a record it matched
2923
+ self.record: dict | None = None # the §4a record it matched, or took its signature from
1943
2924
  self.record_path: str | None = None
2925
+ self.signature_at: tuple[str, int] | None = None # where a check located it: its signature's record
1944
2926
  self.waits = False # on a declaring_line answer
1945
2927
  self.failure: str | None = None # why its spot-check failed
1946
2928
 
@@ -1954,8 +2936,25 @@ class _GapEntry:
1954
2936
 
1955
2937
  def check(self, root: Path | None, path: object, line: object, export_type: object) -> None:
1956
2938
  self.recorded = (_norm_path(path), _line_of(line))
1957
- self.outcome, self.new_line, self.waits, self.failure = _spot_check(
1958
- root, self.recorded[0], self.name, self.recorded[1], export_type, self.answers)
2939
+ self.outcome, self.new_line, self.waits, self.failure, self.unknown_reason = _spot_check(
2940
+ root, self.recorded[0], self.declared, self.recorded[1], export_type, self.answers)
2941
+
2942
+ def locate(self, root: Path | None) -> None:
2943
+ """Record where a check located the export (the checked file, at the recorded line or the one it moved
2944
+ to), so its signature comes from §4a's record there; never a file outside the source root."""
2945
+ line = self.new_line if self.outcome == "moved" else self.recorded[1]
2946
+ if self.outcome in ("verified", "moved") and line is not None and _inside(root, self.recorded[0]):
2947
+ self.signature_at = (self.recorded[0], line)
2948
+
2949
+ def take_signature(self, blocks: dict[str, list[dict]]) -> None:
2950
+ """Take the one record of the entry's name in the block of the file it was located in that is at its line,
2951
+ else the one with no line; none otherwise."""
2952
+ path, line = self.signature_at
2953
+ named = [r for r in blocks.get(path, []) if _name_of(r) in (self.name, self.declared)]
2954
+ at = [r for r in named if _line_of(r.get("location")) == line] \
2955
+ or [r for r in named if _line_of(r.get("location")) is None]
2956
+ if len(at) == 1:
2957
+ self.record, self.record_path = at[0], path
1959
2958
 
1960
2959
  def route(self, root: Path | None, drift: bool) -> None:
1961
2960
  """Spot-check the entry, or send it to targeted re-extraction (see "Gap records" in the module docstring)."""
@@ -1974,14 +2973,23 @@ class _GapEntry:
1974
2973
  return
1975
2974
  if self.match_status == "found":
1976
2975
  self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
2976
+ self.locate(root)
1977
2977
  return
1978
- if self.match_status == "ambiguous":
1979
- self.outcome = "unknown"
2978
+ if self.match_status == "ambiguous": # several map entries of its name: no check, apply warns
2979
+ self.outcome, self.unknown_reason = "unknown", "ambiguous"
1980
2980
  return
1981
2981
  if self.citation:
1982
2982
  self.check(root, self.citation.get("file"), self.citation.get("line"), None)
1983
2983
  if self.waits or self.outcome in ("verified", "moved"):
2984
+ if not self.in_map: # a citation of another file than the map's entries of its name takes none
2985
+ self.locate(root)
1984
2986
  return
2987
+ if self.match_status == "not-found" and self.map_locations:
2988
+ # the map holds its name at other files: the cited file's declaration may be another package's, so it is
2989
+ # never re-extracted there (#824); the record keeps the check's cause, and apply warns with what to fix
2990
+ self.citation_check = "missing" if self.outcome == "missing" else self.unknown_reason
2991
+ self.outcome, self.unknown_reason = "unknown", "cited-elsewhere"
2992
+ return
1985
2993
  if self.entry.get("provenance_completeness") and root is not None:
1986
2994
  self.reextract = True
1987
2995
  elif self.blocking:
@@ -1992,14 +3000,20 @@ class _GapEntry:
1992
3000
  self.outcome = "unknown"
1993
3001
 
1994
3002
  def match(self, blocks: dict[str, list[dict]]) -> tuple[int, int]:
1995
- """Take the first export of the entry's name in the blocks of its resolved files; (files scanned, exports
1996
- found in them)."""
3003
+ """Take the first export of the entry's name in the blocks of its resolved files, else the first whose
3004
+ `aliases` hold its name (the public name the runner gives its local name in that file); (files scanned,
3005
+ exports found in them)."""
1997
3006
  exports = 0
3007
+ aliased: tuple[dict, str] | None = None
1998
3008
  for path in self.paths:
1999
3009
  for record in blocks.get(path, []):
2000
3010
  exports += 1
2001
3011
  if self.record is None and _name_of(record) == self.name:
2002
3012
  self.record, self.record_path = record, path
3013
+ elif aliased is None and isinstance(record.get("aliases"), list) and self.name in record["aliases"]:
3014
+ aliased = (record, path)
3015
+ if self.record is None and aliased is not None:
3016
+ self.record, self.record_path = aliased
2003
3017
  if self.record is not None:
2004
3018
  self.outcome = "re-extracted"
2005
3019
  return len(self.paths), exports
@@ -2028,8 +3042,14 @@ class _GapEntry:
2028
3042
  return (self.record or {}).get("location")
2029
3043
  return None
2030
3044
 
3045
+ def signature(self) -> dict | None:
3046
+ """The record a located export takes its signature from, or None."""
3047
+ return self.record if self.signature_at and not self.internal else None
3048
+
2031
3049
  def verification(self) -> dict:
3050
+ signed = self.signature() or {}
2032
3051
  recorded = _where(*self.recorded)
3052
+ elsewhere = self.outcome == "unknown" and self.unknown_reason == "cited-elsewhere"
2033
3053
  if self.outcome == "re-extracted":
2034
3054
  citation = self.new_location()
2035
3055
  elif self.outcome in ("verified", "moved", "missing", "rescoped") and recorded:
@@ -2046,13 +3066,19 @@ class _GapEntry:
2046
3066
  "provenance_citation": citation,
2047
3067
  "source_citation": self.citation,
2048
3068
  "new_location": self.new_location(),
2049
- "unknown_reason": self.unknown_reason,
3069
+ "unknown_reason": self.unknown_reason if self.outcome == "unknown" else None,
3070
+ "citation_check": self.citation_check if elsewhere else None,
3071
+ "map_locations": self.map_locations if elsewhere else None,
2050
3072
  "pinned_definition_lines": _pinned_lines(self.entry.get("remediation"))
2051
3073
  if self.category == "MOVED_EXPORT" else None,
2052
3074
  "resolution_source": "remediation-paths" if self.outcome == "re-extracted" else None,
2053
3075
  "reachability": self.answers.get("reachability") if self.category == "NEW_EXPORT" and self.located
2054
3076
  else None,
2055
- "export_type": None,
3077
+ "export_type": signed.get("type", signed.get("export_type")),
3078
+ "params": signed.get("params", signed.get("parameters")),
3079
+ "return_type": signed.get("return_type"),
3080
+ "extraction_method": signed.get("extraction_method"),
3081
+ "ast_node_type": signed.get("ast_node_type"),
2056
3082
  }
2057
3083
 
2058
3084
  def label(self, provenance: list[dict]) -> str:
@@ -2065,7 +3091,7 @@ class _GapEntry:
2065
3091
  elif self.map_entry is not None:
2066
3092
  file, line = _norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"])
2067
3093
  for entry in provenance:
2068
- if (_name_of(entry) == self.name and _file_of(entry) == file
3094
+ if (_name_of(entry) == self.declared and _file_of(entry) == file
2069
3095
  and _line_of(entry.get("source_line")) == line):
2070
3096
  method = entry.get("extraction_method")
2071
3097
  break
@@ -2114,17 +3140,22 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
2114
3140
  if waiting:
2115
3141
  return None, {"status": "needs-judgment", "needs_judgment": waiting, "warnings": warnings}
2116
3142
  to_scan = [g for g in routed if g.reextract]
2117
- files = list(dict.fromkeys(p for g in to_scan for p in g.paths))
3143
+ located = [g for g in routed if g.signature_at and not g.internal] # an internal export takes no signature
3144
+ files = list(dict.fromkeys([p for g in to_scan for p in g.paths] + [g.signature_at[0] for g in located]))
2118
3145
  if plan:
2119
3146
  return None, {"status": "planned", "reextract": {
2120
3147
  "entries": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
2121
- "resolved_paths": g.paths} for g in to_scan], "files": files}, "warnings": warnings}
3148
+ "resolved_paths": g.paths} for g in to_scan], "files": files,
3149
+ "signatures": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "file": g.signature_at[0],
3150
+ "line": g.signature_at[1]} for g in located]}, "warnings": warnings}
2122
3151
  blocks: dict[str, list[dict]] = {}
2123
3152
  scanned = 0
3153
+ targeted_files = {p for g in to_scan for p in g.paths}
3154
+ signature_files = {g.signature_at[0] for g in located}
2124
3155
  if files:
2125
3156
  if remediation is None:
2126
- raise ValueError("targeted re-extraction has files to match: pass --remediation-records, the output "
2127
- "of `records` over them")
3157
+ raise ValueError("gap-records has files to match (targeted re-extraction, or a located export's "
3158
+ "signature): pass --remediation-records, the output of `records` over them")
2128
3159
  if not isinstance(remediation, dict) or not isinstance(remediation.get("files"), list):
2129
3160
  raise ValueError("--remediation-records has no `files` array: is it the output of `records`?")
2130
3161
  for block in remediation["files"]:
@@ -2146,20 +3177,36 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
2146
3177
  "files_scanned": files_scanned, "exports_found_in_scan": found})
2147
3178
  if unresolved:
2148
3179
  return None, {"status": "unresolved", "unresolved": unresolved, "warnings": warnings}
3180
+ for g in located:
3181
+ g.take_signature(blocks)
2149
3182
  needs = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": g.needs(),
2150
3183
  "file": _location(g.new_location())[0] if g.outcome != "verified" else g.recorded[0],
2151
3184
  "line": _location(g.new_location())[1] if g.outcome != "verified" else g.recorded[1]}
2152
3185
  for g in routed if g.needs()]
2153
3186
  if needs:
2154
3187
  return None, {"status": "needs-judgment", "needs_judgment": needs, "warnings": warnings}
3188
+ warnings += [f"provenance: {g.name}: located at {g.signature_at[0]}:{g.signature_at[1]}, but no record of it "
3189
+ f"there; signature not repaired" for g in located if g.record is None]
2155
3190
  kept: dict[str, list[dict]] = {}
2156
3191
  for g in routed:
2157
- if g.record is not None and not g.internal:
2158
- record = dict(g.record)
2159
- if g.answers.get("docstring") is not None:
2160
- record["docstring"] = g.answers["docstring"]
2161
- if record not in kept.setdefault(g.record_path, []):
2162
- kept[g.record_path].append(record)
3192
+ if g.record is None or g.internal:
3193
+ continue
3194
+ record = dict(g.record)
3195
+ record.pop("public", None) # a repair documents and maps every record: the mark is a normal update's
3196
+ record.pop("aliases", None) # the match is made: the copy names the one it took
3197
+ if _name_of(record) != g.name: # matched by its public name: documented under it, its own name as `local`
3198
+ record = {"name": g.name, "local": _name_of(record),
3199
+ **{k: v for k, v in record.items() if k not in ("name", "export_name", "local")}}
3200
+ if g.outcome == "re-extracted" and g.answers.get("docstring") is not None:
3201
+ record["docstring"] = g.answers["docstring"] # a located export's signature record asks none
3202
+ block = kept.setdefault(g.record_path, [])
3203
+ location = record.get("location")
3204
+ same = next((r for r in block if (_name_of(r) == _name_of(record) and r.get("location") == location
3205
+ if location is not None else r == record)), None)
3206
+ if same is None:
3207
+ block.append(record)
3208
+ elif same.get("docstring") is None and record.get("docstring") is not None:
3209
+ same["docstring"] = record["docstring"]
2163
3210
  breakdown = {"T1": 0, "T1-low": 0, "unlabeled": 0, "T2": 0}
2164
3211
  provenance_entries = _entries_of(provenance)
2165
3212
  for g in routed:
@@ -2170,13 +3217,15 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
2170
3217
  "files_extracted": scanned,
2171
3218
  "exports_extracted": len(entries),
2172
3219
  "confidence_breakdown": breakdown,
2173
- "verification": [g.verification() for g in routed],
3220
+ "verification": [{"gap_id": g.entry.get("gap_id"), **g.verification()} for g in routed],
2174
3221
  "files": [{"file_path": path, "exports": exports} for path, exports in kept.items()],
2175
3222
  }
2176
3223
  matched = [g for g in to_scan if g.record is not None]
2177
- targeted = {"resolved_count": len(matched), "files_scanned": scanned,
3224
+ scanned_paths = set(blocks)
3225
+ targeted = {"resolved_count": len(matched), "files_scanned": len(scanned_paths & targeted_files),
3226
+ "signature_files_scanned": len(scanned_paths & signature_files),
2178
3227
  "exports_matched": len({(g.record_path, _name_of(g.record)) for g in matched}),
2179
- "tier": tier} if to_scan else None
3228
+ "tier": tier} if to_scan or located else None
2180
3229
  summary = {
2181
3230
  "status": "written",
2182
3231
  "gap_count": len(entries),
@@ -2356,10 +3405,93 @@ def _apply_categories(args: argparse.Namespace) -> dict | None:
2356
3405
  return {**(categories or {}), "category_c": cat_c}
2357
3406
 
2358
3407
 
3408
+ def _metadata_in(path: str | None) -> dict | None:
3409
+ """apply's --metadata-in: the skill's metadata.json, an object."""
3410
+ if not path:
3411
+ return None
3412
+ metadata = _load_json_file(Path(path), "--metadata-in file")
3413
+ if not isinstance(metadata, dict):
3414
+ raise ValueError("--metadata-in file must hold a JSON object (the skill's metadata.json)")
3415
+ return metadata
3416
+
3417
+
3418
+ _QUICK_METADATA = None
3419
+
3420
+
3421
+ def _quick_metadata():
3422
+ """skf-render-quick-metadata.py, loaded once from this folder as skf-extraction-inventory.py loads it: its
3423
+ probe_skf_version reads the installed SKF's version, so create-skill and update-skill read it one way. Raises
3424
+ ValueError, one line, when it cannot be loaded."""
3425
+ global _QUICK_METADATA
3426
+ if _QUICK_METADATA is None:
3427
+ path = Path(__file__).resolve().parent / "skf-render-quick-metadata.py"
3428
+ cannot = f"cannot load {path.name} beside {Path(__file__).name}"
3429
+ if not path.is_file():
3430
+ raise ValueError(f"{cannot}: it is missing; re-install SKF")
3431
+ spec = importlib.util.spec_from_file_location("skf_render_quick_metadata", path)
3432
+ if spec is None or spec.loader is None:
3433
+ raise ValueError(f"{cannot}: re-install SKF")
3434
+ module = importlib.util.module_from_spec(spec)
3435
+ try:
3436
+ spec.loader.exec_module(module)
3437
+ except (ImportError, SyntaxError, OSError) as exc:
3438
+ raise ValueError(f"{cannot} ({type(exc).__name__}: {' '.join(str(exc).split())}): re-install "
3439
+ f"SKF") from exc
3440
+ if not callable(getattr(module, "probe_skf_version", None)):
3441
+ raise ValueError(f"{cannot}: it has no probe_skf_version; re-install SKF")
3442
+ _QUICK_METADATA = module
3443
+ return _QUICK_METADATA
3444
+
3445
+
3446
+ def _runner_ast_grep(path: str | None) -> str | None:
3447
+ """The ast-grep version the recipe runner's output (--extraction) records: "unknown" when it records none but
3448
+ holds an export ast-grep matched, and None when ast-grep did not run (no export it matched, as in the
3449
+ {"exports": []} step 2 writes when the runner cannot run)."""
3450
+ if not path:
3451
+ return None
3452
+ extraction = _load_json_file(Path(path), "--extraction file")
3453
+ if not isinstance(extraction, dict):
3454
+ raise ValueError("--extraction file must hold a JSON object (the recipe runner's output)")
3455
+ ast_grep = extraction.get("ast_grep")
3456
+ version = _tool_version(ast_grep.get("version")) if isinstance(ast_grep, dict) else None
3457
+ if version is not None:
3458
+ return version
3459
+ exports = extraction.get("exports") if isinstance(extraction.get("exports"), list) else []
3460
+ matched = any(isinstance(e, dict) and e.get("extraction_method") == "ast-grep" for e in exports)
3461
+ return "unknown" if matched else None
3462
+
3463
+
3464
+ def _cmd_metadata(args: argparse.Namespace) -> int:
3465
+ target = Path(args.metadata)
3466
+ try:
3467
+ skf_version = _quick_metadata().probe_skf_version(args.skf_root) if args.skf_root else None
3468
+ metadata, summary = update_metadata(
3469
+ _load_json_file(target, "--metadata file"), _load_json_file(Path(args.lists), "--lists file"),
3470
+ _load_json_file(Path(args.stats), "--stats file"), version=args.version,
3471
+ generation_date=args.generation_date, source_commit=args.source_commit, source_ref=args.source_ref,
3472
+ skf_version=skf_version, skf_root=args.skf_root, ast_grep_version=_runner_ast_grep(args.extraction),
3473
+ qmd_version=args.qmd_version)
3474
+ _write_json_atomic(target, metadata)
3475
+ except ValueError as exc:
3476
+ print(f"error: {exc}", file=sys.stderr)
3477
+ return 1
3478
+ except OSError as exc:
3479
+ print(f"error: cannot write {args.metadata}: {exc}", file=sys.stderr)
3480
+ return 1
3481
+ print(json.dumps({"status": "written", "metadata": target.as_posix(), "version": metadata["version"],
3482
+ "exports": len(metadata["exports"]), "scripts": len(metadata.get("scripts") or []),
3483
+ "assets": len(metadata.get("assets") or []), "tool_versions": metadata["tool_versions"],
3484
+ **summary}))
3485
+ return 0
3486
+
3487
+
2359
3488
  def _cmd_apply(args: argparse.Namespace) -> int:
2360
3489
  if bool(args.drift_head) != bool(args.drift_pinned):
2361
3490
  print("error: --drift-head and --drift-pinned go together", file=sys.stderr)
2362
3491
  return 1
3492
+ if bool(args.metadata_in) != bool(args.lists_out):
3493
+ print("error: --metadata-in and --lists-out go together", file=sys.stderr)
3494
+ return 1
2363
3495
  try:
2364
3496
  promoted = _optional_json(args.promoted_docs, "--promoted-docs file")
2365
3497
  if promoted is not None and not isinstance(promoted, list):
@@ -2385,6 +3517,7 @@ def _cmd_apply(args: argparse.Namespace) -> int:
2385
3517
  source_commit=args.source_commit,
2386
3518
  source_ref=args.source_ref,
2387
3519
  drift=(args.drift_head, args.drift_pinned) if args.drift_head else None,
3520
+ metadata=_metadata_in(args.metadata_in),
2388
3521
  )
2389
3522
  except ValueError as exc:
2390
3523
  print(f"error: {exc}", file=sys.stderr)
@@ -2392,11 +3525,15 @@ def _cmd_apply(args: argparse.Namespace) -> int:
2392
3525
  if summary["status"] == "refused":
2393
3526
  print(json.dumps(summary))
2394
3527
  return EXIT_REFUSED
2395
- try:
2396
- _write_json_atomic(Path(args.output), doc)
2397
- except OSError as exc:
2398
- print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
2399
- return 1
3528
+ lists = summary.pop("lists", None)
3529
+ for path, value in ((args.not_public_out, summary["not_public"]), (args.lists_out, lists),
3530
+ (args.output, doc)):
3531
+ try:
3532
+ if path:
3533
+ _write_json_atomic(Path(path), value)
3534
+ except OSError as exc:
3535
+ print(f"error: cannot write {path}: {exc}", file=sys.stderr)
3536
+ return 1
2400
3537
  print(json.dumps({**summary, "map": args.output}))
2401
3538
  return 0
2402
3539
 
@@ -2464,6 +3601,76 @@ def _cmd_gap_records(args: argparse.Namespace) -> int:
2464
3601
  return 0
2465
3602
 
2466
3603
 
3604
+ # --------------------------------------------------------------------------
3605
+ # Baseline gaps (update-skill detect-changes Category B)
3606
+ # --------------------------------------------------------------------------
3607
+
3608
+
3609
+ VERIFIER_FILE = "skf-verify-provenance-completeness.py"
3610
+
3611
+
3612
+ def _baseline_verifier():
3613
+ """The verifier beside this script, for its `baseline_gaps`. Unlike `_verifier()`, which lets a gap-driven
3614
+ spot-check go on as unknown, one that cannot be loaded (or has no `baseline_gaps`) is a ValueError: with no
3615
+ verifier no gap is listed, and every name the runner leaves out would read as a deleted export."""
3616
+ path = Path(__file__).resolve().parent / VERIFIER_FILE
3617
+ where = f"cannot load {VERIFIER_FILE} beside {Path(__file__).name}"
3618
+ try:
3619
+ spec = importlib.util.spec_from_file_location("skf_verify_provenance_completeness_baseline", path)
3620
+ if spec is None or spec.loader is None or not path.is_file():
3621
+ raise ImportError(f"no file at {path}")
3622
+ module = importlib.util.module_from_spec(spec)
3623
+ spec.loader.exec_module(module)
3624
+ except Exception as exc: # whatever the script raises, the call reports it
3625
+ raise ValueError(f"{where}: {exc}; re-install SKF") from exc
3626
+ if not callable(getattr(module, "baseline_gaps", None)):
3627
+ raise ValueError(f"{where}: it has no baseline_gaps; re-install SKF")
3628
+ return module
3629
+
3630
+
3631
+ def baseline_gaps(provenance: dict, extraction: dict, files: list, source_root: Path, verifier) -> dict:
3632
+ """{"gaps", "unchecked"}: the map entries of `files` the runner's `extraction` lacks whose file still declares
3633
+ them (see "Baseline gaps" in the module docstring)."""
3634
+ if not isinstance(extraction, dict):
3635
+ raise ValueError("--extraction file has no `exports` array: is it the helper's output?")
3636
+ held = {(name, file) for e in _exports_of(extraction, "--extraction file")
3637
+ if (name := _name_of(e)) is not None and (file := _file_of(e)) is not None}
3638
+ diff = extraction.get("entry_point_diff")
3639
+ listed = {(item["name"], _norm_path(item["file"]))
3640
+ for item in (diff.get("extraction_gaps") or [] if isinstance(diff, dict) else [])
3641
+ if isinstance(item, dict) and isinstance(item.get("name"), str) and _norm_path(item.get("file"))}
3642
+ if provenance.get("entries") is not None and not isinstance(provenance["entries"], list):
3643
+ raise ValueError("provenance `entries` must be an array") # missing or null: no entry (#684)
3644
+ wanted = {f for f in (_norm_path(p) for p in files) if f is not None}
3645
+ gaps, unchecked = verifier.baseline_gaps(provenance, source_root, held, wanted, listed)
3646
+ return {"gaps": gaps, "unchecked": unchecked}
3647
+
3648
+
3649
+ def _cmd_baseline_gaps(args: argparse.Namespace) -> int:
3650
+ try:
3651
+ source_root = Path(args.source_root)
3652
+ if not source_root.is_dir():
3653
+ raise ValueError(f"--source-root {args.source_root} is not a folder")
3654
+ files = _load_json_file(Path(args.files), "--files file")
3655
+ if not (isinstance(files, list) and all(isinstance(f, str) for f in files)):
3656
+ raise ValueError(f"--files file {args.files} must hold a JSON array of paths")
3657
+ provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file")
3658
+ if not isinstance(provenance, dict):
3659
+ raise ValueError(f"--provenance-map file {args.provenance_map} must hold a JSON object")
3660
+ extraction = _load_json_file(Path(args.extraction), "--extraction file")
3661
+ result = baseline_gaps(provenance, extraction, files, source_root, _baseline_verifier())
3662
+ _write_json_atomic(Path(args.output), result)
3663
+ except ValueError as exc:
3664
+ print(f"error: {exc}", file=sys.stderr)
3665
+ return 1
3666
+ except OSError as exc:
3667
+ print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
3668
+ return 1
3669
+ print(json.dumps({"status": "written", "output": args.output, "gaps": len(result["gaps"]),
3670
+ "unchecked": len(result["unchecked"])}))
3671
+ return 0
3672
+
3673
+
2467
3674
  def _build_parser() -> argparse.ArgumentParser:
2468
3675
  parser = argparse.ArgumentParser(
2469
3676
  prog="skf-build-change-manifest",
@@ -2537,9 +3744,50 @@ def _build_parser() -> argparse.ArgumentParser:
2537
3744
  p_apply.add_argument("--source-ref", default=None, help="the map's source_ref (empty: null)")
2538
3745
  p_apply.add_argument("--drift-head", default=None, help="under the drift override: HEAD's short SHA")
2539
3746
  p_apply.add_argument("--drift-pinned", default=None, help="under the drift override: the pinned short SHA")
3747
+ p_apply.add_argument("--not-public-out", metavar="FILE",
3748
+ help="also write the summary's not_public list here (a JSON array)")
3749
+ p_apply.add_argument("--metadata-in", metavar="FILE",
3750
+ help="the skill's metadata.json, whose exports[], scripts[] and assets[] the update "
3751
+ "starts from")
3752
+ p_apply.add_argument("--lists-out", metavar="FILE",
3753
+ help="write the exports[], scripts[] and assets[] metadata.json takes after the update "
3754
+ "here (a JSON object)")
2540
3755
  p_apply.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the new map")
2541
3756
  p_apply.set_defaults(func=_cmd_apply)
2542
3757
 
3758
+ p_meta = sub.add_parser(
3759
+ "metadata", help="write the metadata.json an update leaves",
3760
+ description="Rewrite the skill's metadata.json in place. It sets version and generation_date, "
3761
+ "source_commit and source_ref when given, exports, scripts and assets from --lists (a script "
3762
+ "or asset list left out when empty), confidence_distribution from --stats and the stats "
3763
+ "helper's stats fields, dropping one an earlier run wrote that --stats lacks; tool_versions.skf "
3764
+ "from --skf-root (unknown, with a warning, when it gives no version), and tool_versions.ast_grep "
3765
+ "and tool_versions.qmd from --extraction and --qmd-version when this run used them, else as "
3766
+ "they stand. When --stats' effective_denominator differs from the old one (or one is absent), it "
3767
+ "drops each old stats key named effective_denominator_*; each dropped key is in dropped_stats, "
3768
+ "with a warning. It keeps every other old stats key, with a warning for each one create-skill "
3769
+ "does not write for the skill's scope_type. It removes last_update and "
3770
+ "update_type and keeps every other field, a patched one included. It refuses a --stats whose "
3771
+ "scripts_count or assets_count is not the length of --lists' scripts or assets.")
3772
+ p_meta.add_argument("--metadata", required=True, metavar="FILE",
3773
+ help="the skill's metadata.json, rewritten in place")
3774
+ p_meta.add_argument("--lists", required=True, metavar="FILE", help="the lists apply wrote (--lists-out)")
3775
+ p_meta.add_argument("--stats", required=True, metavar="FILE",
3776
+ help="the JSON skf-render-metadata-stats.py wrote (its -o file)")
3777
+ p_meta.add_argument("--version", required=True, help="the version this update writes")
3778
+ p_meta.add_argument("--generation-date", required=True, help="the clock's UTC time, YYYY-MM-DDTHH:MM:SSZ")
3779
+ p_meta.add_argument("--source-commit", default=None, help="the commit this update read (an empty one: null)")
3780
+ p_meta.add_argument("--source-ref", default=None, help="the ref this update read (an empty one: null)")
3781
+ p_meta.add_argument("--skf-root", metavar="DIR",
3782
+ help="the installed SKF ({project-root}/_bmad/skf): tool_versions.skf, from its package.json "
3783
+ "or VERSION file")
3784
+ p_meta.add_argument("--extraction", metavar="FILE",
3785
+ help="the recipe runner's output this run wrote: tool_versions.ast_grep, its ast_grep.version")
3786
+ p_meta.add_argument("--qmd-version", default=None,
3787
+ help="what qmd --version printed, when this run queried QMD: tool_versions.qmd (a leading "
3788
+ "\"qmd \" removed)")
3789
+ p_meta.set_defaults(func=_cmd_metadata)
3790
+
2543
3791
  p_records = sub.add_parser("records", help="write step 3's re-extraction records from the runner's and the "
2544
3792
  "workers' files")
2545
3793
  p_records.add_argument("--extraction", metavar="FILE", help="the recipe runner's output")
@@ -2559,9 +3807,9 @@ def _build_parser() -> argparse.ArgumentParser:
2559
3807
  help="the workspace drift check's status; overridden runs the drift gate")
2560
3808
  p_gap.add_argument("--judgments", metavar="FILE", help="the answers to its needs_judgment[], keyed by gap id")
2561
3809
  p_gap.add_argument("--remediation-records", metavar="FILE",
2562
- help="`records` output over the files targeted re-extraction scanned")
3810
+ help="`records` output over the files the plan listed")
2563
3811
  p_gap.add_argument("--plan", action="store_true",
2564
- help="print the answers it needs and the files targeted re-extraction scans; write no records")
3812
+ help="print the answers it needs and the files §4a's extraction scans; write no records")
2565
3813
  p_gap.add_argument("--files-out", metavar="FILE", help="with --plan: write those files here, a JSON array")
2566
3814
  p_gap.add_argument("--evidence", metavar="FILE",
2567
3815
  help="append the targeted_reextraction record to this JSON-lines file when it ran")
@@ -2569,6 +3817,17 @@ def _build_parser() -> argparse.ArgumentParser:
2569
3817
  p_gap.add_argument("-o", "--output", metavar="FILE", help="where to write the records (without --plan)")
2570
3818
  p_gap.set_defaults(func=_cmd_gap_records)
2571
3819
 
3820
+ p_base = sub.add_parser("baseline-gaps", help="list the map entries the runner left out that their modified "
3821
+ "file still declares, for Category B's step 2 to read")
3822
+ p_base.add_argument("--provenance-map", required=True, metavar="FILE", help="the map the update starts from")
3823
+ p_base.add_argument("--extraction", required=True, metavar="FILE",
3824
+ help="the recipe runner's output ({\"exports\": []} when it could not run)")
3825
+ p_base.add_argument("--files", required=True, metavar="FILE",
3826
+ help="the files whose entries are checked: Category A's modified-files.json")
3827
+ p_base.add_argument("--source-root", required=True, metavar="DIR", help="the source tree the files are read in")
3828
+ p_base.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the gaps")
3829
+ p_base.set_defaults(func=_cmd_baseline_gaps)
3830
+
2572
3831
  for p in (p_build, p_ratio):
2573
3832
  p.add_argument(
2574
3833
  "--input", default=None, help="path to JSON input (default: stdin)"