bmad-module-skill-forge 3.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +3 -3
- package/docs/_data/pinned.yaml +19 -13
- package/docs/architecture.md +2 -2
- package/docs/concepts.md +4 -4
- package/docs/examples.md +33 -26
- package/docs/getting-started.md +3 -3
- package/docs/how-it-works.md +2 -2
- package/docs/index.md +1 -1
- package/docs/skill-model.md +41 -39
- package/docs/troubleshooting.md +8 -0
- package/docs/verifying-a-skill.md +46 -21
- package/docs/why-skf.md +1 -1
- package/docs/workflows.md +5 -5
- package/package.json +1 -1
- package/src/knowledge/qmd-registry.md +1 -1
- package/src/knowledge/version-paths.md +1 -1
- package/src/shared/health-check.md +2 -50
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +12 -4
- package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +212 -59
- package/src/shared/scripts/skf-build-change-manifest.py +1437 -178
- package/src/shared/scripts/skf-classify-changed-files.py +78 -0
- package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
- package/src/shared/scripts/skf-count-tokens.py +54 -10
- package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
- package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
- package/src/shared/scripts/skf-detect-workspaces.py +67 -0
- package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
- package/src/shared/scripts/skf-extract-public-api.py +1673 -189
- package/src/shared/scripts/skf-extraction-inventory.py +584 -46
- package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
- package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
- package/src/shared/scripts/skf-github-probe.py +47 -8
- package/src/shared/scripts/skf-hash-content.py +14 -8
- package/src/shared/scripts/skf-load-provenance.py +3 -1
- package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
- package/src/shared/scripts/skf-parse-gaps.py +135 -26
- package/src/shared/scripts/skf-preflight.py +15 -1
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
- package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
- package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
- package/src/shared/scripts/skf-resolve-customization.py +197 -0
- package/src/shared/scripts/skf-shard-body.py +239 -41
- package/src/shared/scripts/skf-structural-diff.py +133 -18
- package/src/shared/scripts/skf-update-active-symlink.py +115 -62
- package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
- package/src/shared/scripts/skf-validate-output.py +12 -3
- package/src/shared/scripts/skf-verify-provenance-completeness.py +1439 -79
- package/src/shared/scripts/skf-write-skill-brief.py +41 -8
- package/src/shared/tool-requirements.yaml +8 -2
- package/src/skf-analyze-source/SKILL.md +2 -2
- package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
- package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
- package/src/skf-audit-skill/SKILL.md +2 -2
- package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
- package/src/skf-audit-skill/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +2 -1
- package/src/skf-audit-skill/references/re-index.md +14 -9
- package/src/skf-audit-skill/references/report.md +2 -2
- package/src/skf-audit-skill/references/semantic-diff.md +15 -15
- package/src/skf-audit-skill/references/structural-diff.md +10 -5
- package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
- package/src/skf-brief-skill/SKILL.md +3 -3
- package/src/skf-brief-skill/assets/scope-templates.md +1 -1
- package/src/skf-brief-skill/assets/skill-brief-schema.md +4 -11
- package/src/skf-brief-skill/references/analyze-target.md +6 -6
- package/src/skf-brief-skill/references/confirm-brief.md +1 -1
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
- package/src/skf-brief-skill/references/version-resolution.md +5 -9
- package/src/skf-campaign/SKILL.md +2 -2
- package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
- package/src/skf-create-skill/SKILL.md +2 -2
- package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
- package/src/skf-create-skill/assets/skill-sections.md +7 -7
- package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
- package/src/skf-create-skill/references/compile.md +31 -60
- package/src/skf-create-skill/references/component-extraction.md +3 -3
- package/src/skf-create-skill/references/enrich.md +2 -2
- package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
- package/src/skf-create-skill/references/extract.md +23 -19
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
- package/src/skf-create-skill/references/extraction-patterns.md +10 -11
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +21 -5
- package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
- package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
- package/src/skf-create-stack-skill/SKILL.md +2 -2
- package/src/skf-create-stack-skill/references/validate.md +1 -1
- package/src/skf-drop-skill/SKILL.md +2 -2
- package/src/skf-drop-skill/references/execute.md +5 -1
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +2 -2
- package/src/skf-drop-skill/references/select.md +7 -3
- package/src/skf-export-skill/SKILL.md +2 -2
- package/src/skf-export-skill/assets/snippet-format.md +5 -5
- package/src/skf-export-skill/references/generate-snippet.md +22 -15
- package/src/skf-export-skill/references/load-skill.md +1 -1
- package/src/skf-export-skill/references/token-report.md +29 -9
- package/src/skf-forger/SKILL.md +2 -2
- package/src/skf-quick-skill/SKILL.md +2 -2
- package/src/skf-refine-architecture/SKILL.md +6 -8
- package/src/skf-rename-skill/SKILL.md +2 -2
- package/src/skf-setup/SKILL.md +2 -2
- package/src/skf-test-skill/SKILL.md +2 -2
- package/src/skf-test-skill/assets/output-section-formats.md +6 -4
- package/src/skf-test-skill/references/coherence-check.md +5 -4
- package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
- package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
- package/src/skf-test-skill/references/coverage-check.md +69 -111
- package/src/skf-test-skill/references/init.md +5 -6
- package/src/skf-test-skill/references/migration-section-rules.md +26 -14
- package/src/skf-test-skill/references/report.md +11 -9
- package/src/skf-test-skill/references/scoring-rules.md +8 -6
- package/src/skf-test-skill/references/source-access-protocol.md +15 -18
- package/src/skf-test-skill/scripts/build-result-context.py +114 -21
- package/src/skf-test-skill/scripts/gap-ledger.py +580 -80
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +646 -107
- package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
- package/src/skf-test-skill/scripts/reconcile-coverage.py +450 -23
- package/src/skf-test-skill/scripts/score-signatures.py +356 -61
- package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
- package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
- package/src/skf-test-skill/templates/test-report-template.md +1 -0
- package/src/skf-update-skill/SKILL.md +2 -2
- package/src/skf-update-skill/references/detect-changes.md +32 -13
- package/src/skf-update-skill/references/gap-driven.md +37 -22
- package/src/skf-update-skill/references/init.md +6 -2
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +17 -12
- package/src/skf-update-skill/references/re-extract.md +4 -2
- package/src/skf-update-skill/references/report.md +7 -4
- package/src/skf-update-skill/references/write.md +77 -36
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
- package/src/skf-verify-stack/SKILL.md +2 -2
- package/tools/cli/lib/tool-check.js +158 -2
|
@@ -55,16 +55,30 @@ Subcommands:
|
|
|
55
55
|
apply --update-type incremental|gap-driven|full --skill-name <name>
|
|
56
56
|
--generation-date <iso> --confidence-tier <tier>
|
|
57
57
|
--manual-sections-preserved <n> -o <file> [inputs...]
|
|
58
|
+
[--not-public-out <file>] [--metadata-in <file> --lists-out <file>]
|
|
58
59
|
The provenance map an update writes (see "Apply" below): the old
|
|
59
60
|
map with this run's changes, written to -o through a temporary file
|
|
60
|
-
and a rename
|
|
61
|
+
and a rename, the summary's not_public list to --not-public-out and
|
|
62
|
+
metadata.json's exports[], scripts[] and assets[] to --lists-out
|
|
63
|
+
when given. Prints a summary with the warnings for a person.
|
|
64
|
+
|
|
65
|
+
metadata --metadata <metadata.json> --lists <file> --stats <file>
|
|
66
|
+
--version <version> --generation-date <iso>
|
|
67
|
+
[--source-commit <sha>] [--source-ref <ref>] [--skf-root <dir>]
|
|
68
|
+
[--extraction <file>] [--qmd-version <version>]
|
|
69
|
+
The skill's metadata.json an update leaves (see "Metadata" below),
|
|
70
|
+
written over --metadata through a temporary file and a rename.
|
|
71
|
+
Prints {"status": "written", "metadata": <path>, "version",
|
|
72
|
+
"exports": N, "scripts": N, "assets": N, "tool_versions": {...},
|
|
73
|
+
"dropped_stats": [...], "warnings": [...]}.
|
|
61
74
|
|
|
62
75
|
records [--extraction <file>] [--export-details <file>]
|
|
63
76
|
[--files-from <file>] [--patches <dir>] -o <file>
|
|
64
77
|
Step 3's re-extraction records (see "Records" below), written to -o
|
|
65
78
|
through a temporary file and a rename. Prints {"status": "written",
|
|
66
79
|
"output", "files_extracted", "exports_extracted",
|
|
67
|
-
"confidence_breakdown": {"T1", "T1-low", "T2"}, "
|
|
80
|
+
"confidence_breakdown": {"T1", "T1-low", "T2"}, "marked_not_public"
|
|
81
|
+
(the exports marked public: false), "warnings"}.
|
|
68
82
|
|
|
69
83
|
gap-records --manifest <file> --provenance-map <file> [--source-root <dir>]
|
|
70
84
|
[--drift-status ok|skipped|overridden] [--judgments <file>]
|
|
@@ -72,8 +86,17 @@ Subcommands:
|
|
|
72
86
|
(--plan [--files-out <file>] | -o <file>)
|
|
73
87
|
A gap-driven repair's spot-checks, routing and verification records
|
|
74
88
|
(see "Gap records" below): --plan prints what to answer and which
|
|
75
|
-
files
|
|
76
|
-
|
|
89
|
+
files §4a's extraction scans (targeted re-extraction, and the
|
|
90
|
+
signatures of the exports a spot-check located); without it, the
|
|
91
|
+
records are written to -o through a temporary file and a rename.
|
|
92
|
+
|
|
93
|
+
baseline-gaps --provenance-map <file> --extraction <file> --files <file>
|
|
94
|
+
--source-root <dir> -o <file>
|
|
95
|
+
Category B's baseline gaps (see "Baseline gaps" below): each map
|
|
96
|
+
entry of the files --files lists that the runner did not report and
|
|
97
|
+
its file still declares, written to -o through a temporary file and
|
|
98
|
+
a rename. Prints {"status": "written", "output": <-o>, "gaps": N,
|
|
99
|
+
"unchecked": N}.
|
|
77
100
|
|
|
78
101
|
Helper files in place of typed slices (update-skill detect-changes §2.1):
|
|
79
102
|
|
|
@@ -94,9 +117,13 @@ Helper files in place of typed slices (update-skill detect-changes §2.1):
|
|
|
94
117
|
changed); moved_exports for an
|
|
95
118
|
export whose only field is `line`
|
|
96
119
|
moved[] -> moved_exports {name, file:
|
|
97
|
-
current_file,
|
|
120
|
+
current_file, old_file:
|
|
121
|
+
previous_file (only when it
|
|
122
|
+
differs), old_line:
|
|
98
123
|
previous_line, new_line: line},
|
|
99
|
-
unless it is a modified export
|
|
124
|
+
unless it is a modified export,
|
|
125
|
+
which then takes old_file and
|
|
126
|
+
old_line: previous_line
|
|
100
127
|
label_changes[] is not drift, and a name
|
|
101
128
|
ambiguous_names[] lists stays removed and added
|
|
102
129
|
--category-c <file> the output of rename-candidates: its category_c
|
|
@@ -136,10 +163,10 @@ Input JSON shape (object on stdin or in --input file):
|
|
|
136
163
|
"deleted": ["path3", ...]
|
|
137
164
|
},
|
|
138
165
|
"category_b": {
|
|
139
|
-
"modified_exports": [{name, file, old_line, new_line}, ...],
|
|
166
|
+
"modified_exports": [{name, file, old_line, new_line, old_file?}, ...],
|
|
140
167
|
"new_exports": [{name, file, line}, ...],
|
|
141
168
|
"deleted_exports": [{name, file, old_line}, ...],
|
|
142
|
-
"moved_exports": [{name, file, old_line, new_line}, ...]
|
|
169
|
+
"moved_exports": [{name, file, old_line, new_line, old_file?}, ...]
|
|
143
170
|
},
|
|
144
171
|
"category_c": {
|
|
145
172
|
"renamed_files": [{old_path, new_path}, ...],
|
|
@@ -164,6 +191,10 @@ Manifest (build's output):
|
|
|
164
191
|
"total_export_changes": N, "per_file": [...],
|
|
165
192
|
"category_d": {the eight category_d lists, each path once}}
|
|
166
193
|
|
|
194
|
+
Each per_file item's exports_affected holds {name, change_type,
|
|
195
|
+
old_line, new_line}, and old_file for an export that moved across files
|
|
196
|
+
(a MOVED_EXPORT, or a MODIFIED_EXPORT that also changed).
|
|
197
|
+
|
|
167
198
|
A tracked document (a doc row of file_entries[]) has no other detector:
|
|
168
199
|
update-skill's Category A never lists it, so a document-only change still
|
|
169
200
|
counts, and the update records its new hash.
|
|
@@ -213,14 +244,32 @@ Apply (update-skill write.md §3):
|
|
|
213
244
|
|
|
214
245
|
Reads the provenance map the update started from (--provenance-map;
|
|
215
246
|
without it, a degraded run's new map) and prints nothing but a summary:
|
|
216
|
-
the new map goes to -o. Every entry
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
247
|
+
the new map goes to -o. Every entry it adds or rewrites carries
|
|
248
|
+
signature_source, set from the tool that produced the signature:
|
|
249
|
+
ast-grep gives T1 (with confidence T1 and the recipe's ast_node_type),
|
|
250
|
+
a signature read by eye (source-read) gives T1-low (confidence T1-low,
|
|
251
|
+
ast_node_type null). Every entry the run leaves alone keeps its exact
|
|
252
|
+
value, but for a signature_source it lacks or holds as a value other
|
|
253
|
+
than T1, T1-low, T2 or T3 (compared case-insensitively), as a map
|
|
254
|
+
written before the key existed does: once the mode's changes are made,
|
|
255
|
+
each such entry takes its confidence when that names T1-low, T2 or T3,
|
|
256
|
+
else T1 for an extraction_method of ast-grep or ast_bridge (compared
|
|
257
|
+
stripped and case-insensitively) and T1-low for any other (T1 needs an
|
|
258
|
+
ast-grep match). The summary's signature_source_backfilled counts these
|
|
259
|
+
fills by value ({"T1": 1, "T1-low": 2}, {} when none), with one warning
|
|
260
|
+
that lists each unrecognized value replaced (name: old -> new, the
|
|
261
|
+
first 10, then "and N more"); their names are not in its `updated`. A
|
|
262
|
+
rewritten entry keeps any key the update does not set (notes,
|
|
263
|
+
deprecated). A new entry's source_library is the map's library when its
|
|
264
|
+
entries name one (--skill-name when they name none). When they name
|
|
265
|
+
several, it is the library the entries of its file settle, else the one
|
|
266
|
+
the entries under the nearest folder above the file that holds entries
|
|
267
|
+
settle: the one library they name, or of several, the one that is a
|
|
268
|
+
path prefix of all the others (storybook beside storybook/test). The
|
|
269
|
+
record of a MOVED file, or of an export whose change names an old_file,
|
|
270
|
+
tries its new path, then its old. When neither settles it, or the entry
|
|
271
|
+
has no file, it is the map's first library, and one warning counts and
|
|
272
|
+
names those entries (the first 10, then "and N more").
|
|
224
273
|
|
|
225
274
|
incremental (normal mode) reads --manifest (build's output), the renamed
|
|
226
275
|
exports of --category-c and --ccc-pairs (or of --input, the category
|
|
@@ -233,14 +282,37 @@ Apply (update-skill write.md §3):
|
|
|
233
282
|
details, then step 3's record (the runner's in the map's typed form).
|
|
234
283
|
Per file of the manifest:
|
|
235
284
|
MODIFIED NEW_EXPORT adds an entry, MODIFIED_EXPORT rewrites the
|
|
236
|
-
export's fields from its fresh record,
|
|
237
|
-
|
|
285
|
+
export's fields (file included) from its fresh record,
|
|
286
|
+
MOVED_EXPORT moves the entry to the file and the fresh
|
|
287
|
+
record's line, else new_line, DELETED_EXPORT removes it;
|
|
288
|
+
a change with an old_file finds its entry in that file
|
|
238
289
|
ADDED every fresh record of the file becomes an entry
|
|
239
290
|
DELETED every entry of the file is removed
|
|
240
291
|
MOVED the old path's entries are replaced by the new path's
|
|
241
292
|
fresh records, each keeping the keys of the entry of its
|
|
242
293
|
name it replaces
|
|
243
|
-
and each renamed export takes its new name and fresh fields. A
|
|
294
|
+
and each renamed export takes its new name and fresh fields. A
|
|
295
|
+
MOVED_EXPORT whose fresh record is the runner's (ast-grep) also takes
|
|
296
|
+
its confidence, extraction_method and ast_node_type (the entry's when
|
|
297
|
+
the record's is null), and its signature_source when the record holds
|
|
298
|
+
each of params and return_type the entry holds (else the entry's, or
|
|
299
|
+
T1-low when the entry has none it recognizes: the signature it keeps
|
|
300
|
+
was not the runner's), never its export_type, params or return_type;
|
|
301
|
+
one read by eye moves the line and file only.
|
|
302
|
+
|
|
303
|
+
Public-api skills (incremental only): a NEW_EXPORT or MODIFIED_EXPORT
|
|
304
|
+
the map does not hold, an ADDED file's record and a MOVED file's record
|
|
305
|
+
whose name no old entry carries become entries only when on the public
|
|
306
|
+
surface: the record's `public` mark (see "Records"), else, for a record
|
|
307
|
+
--reextract-records does not mark, the same rule over --extraction (its
|
|
308
|
+
scope.type). A name whose entry this run removes (a DELETED_EXPORT, a
|
|
309
|
+
DELETED file's or a MOVED file's old path's entry) is added whatever
|
|
310
|
+
its mark. Each one left out is listed in the summary's not_public,
|
|
311
|
+
{name, file}, with one warning for them all; an entry the map holds is
|
|
312
|
+
updated whatever its mark, and no entry is removed for it. A public-api
|
|
313
|
+
--extraction the rule cannot read (no entry_point_diff, an incomplete
|
|
314
|
+
run or one with errors, an unresolved entry point) adds every name, with
|
|
315
|
+
one warning. A docs-only
|
|
244
316
|
run's records ({"mode": "docs-only", "changed_urls": [...], "exports":
|
|
245
317
|
[{name, type, params, return_type, url}]}) replace the entries of each
|
|
246
318
|
changed URL (source_file is the URL; confidence and signature_source
|
|
@@ -249,31 +321,91 @@ Apply (update-skill write.md §3):
|
|
|
249
321
|
gap-driven reads --reextract-records ({"mode": "gap-driven",
|
|
250
322
|
"verification": [...], "files": [...]}) and --merge-records ({"exports":
|
|
251
323
|
[{export_name, export_type, params, return_type}]}, merge's output for
|
|
252
|
-
an export no
|
|
324
|
+
an export no record covers). Each verification record names gap_id,
|
|
253
325
|
export_name, gap_category, severity, verification, in_map (the map holds
|
|
254
326
|
an entry of that name), map_entry ({source_file, source_line} of the one
|
|
255
327
|
entry the spot-check took, else null), source_citation, new_location,
|
|
256
|
-
unknown_reason,
|
|
257
|
-
|
|
258
|
-
|
|
328
|
+
unknown_reason, citation_check, map_locations (see "Gap records"),
|
|
329
|
+
pinned_definition_lines, reachability, and the located
|
|
330
|
+
export's export_type, params, return_type, extraction_method and
|
|
331
|
+
ast_node_type (from §4a's record at its line, see "Gap records"; null
|
|
332
|
+
when none holds it). It finds an export's entry by its name, else by the
|
|
333
|
+
entry's reexported_as (a renamed re-export the map holds under its
|
|
334
|
+
declared name, which the gap names by its public one):
|
|
335
|
+
verified (map_entry) unchanged, but a NEW_EXPORT's or
|
|
336
|
+
MODIFIED_EXPORT's signature (below) when its
|
|
337
|
+
record was found
|
|
338
|
+
moved (map_entry) source_line (and source_file) from new_location,
|
|
339
|
+
and a NEW_EXPORT's or MODIFIED_EXPORT's
|
|
340
|
+
signature (below) when its record was found
|
|
259
341
|
re-extracted the entry from its `files` record (gap-driven.md §4a);
|
|
260
|
-
none when its reachability is internal-unreachable
|
|
342
|
+
none when its reachability is internal-unreachable;
|
|
343
|
+
a record with a `local` name (matched by the
|
|
344
|
+
public name a re-export gives it) writes
|
|
345
|
+
export_name the declared `local` name, at its
|
|
346
|
+
line, and reexported_as the public one
|
|
261
347
|
verified/moved, not in_map, a NEW_EXPORT or MODIFIED_EXPORT whose
|
|
262
348
|
reachability is not internal-unreachable
|
|
263
349
|
one source-read entry at the citation's line
|
|
264
350
|
(verified) or new_location (moved), at most one
|
|
265
|
-
per name and file,
|
|
266
|
-
|
|
351
|
+
per name and file, with the signature (below)
|
|
352
|
+
and the source-read labels whatever its record's
|
|
353
|
+
tool
|
|
267
354
|
unknown, not in_map, a NEW_EXPORT or MODIFIED_EXPORT, a Medium, Low or
|
|
268
355
|
Info severity an entry from --merge-records with no
|
|
269
356
|
source_file or source_line
|
|
270
357
|
unknown, not in_map, any other severity (a missing one included)
|
|
271
358
|
refused: exit 3, nothing written
|
|
272
359
|
missing, unknown in_map, unknown on a MOVED_EXPORT
|
|
273
|
-
unchanged, with a warning for a person
|
|
360
|
+
unchanged, with a warning for a person that
|
|
361
|
+
names an unknown's unknown_reason and its
|
|
362
|
+
words (UNKNOWN_REASONS); for a NEW_EXPORT or
|
|
363
|
+
MODIFIED_EXPORT whose severity is not Medium,
|
|
364
|
+
Low or Info (a missing one included), the
|
|
365
|
+
warning also names its gap_id and the one
|
|
366
|
+
fix a re-run acts on: for an ambiguous one,
|
|
367
|
+
fix its entries by hand, or give the gap a
|
|
368
|
+
`file:line` Source at its definition when
|
|
369
|
+
that file holds only one of them (the
|
|
370
|
+
re-run's map_match then finds that one); for
|
|
371
|
+
any other, fix the entry by hand (its
|
|
372
|
+
source_file and source_line), since the
|
|
373
|
+
re-run checks the same entry again. A
|
|
374
|
+
cited-elsewhere one, whatever its severity,
|
|
375
|
+
takes in their place a warning that names
|
|
376
|
+
its gap_id, its source_citation, its
|
|
377
|
+
citation_check (with its words), its
|
|
378
|
+
map_locations and the fix for each side
|
|
379
|
+
that may be wrong: when the map is right,
|
|
380
|
+
re-run Test Skill (the test compared
|
|
381
|
+
another file's declaration), and if the gap
|
|
382
|
+
comes back (not for a missing one), exclude
|
|
383
|
+
the cited file in the brief's scope when the
|
|
384
|
+
skill does not cover it; when the export
|
|
385
|
+
moved, fix the entry by hand (with several
|
|
386
|
+
map_locations, whichever of those entries
|
|
387
|
+
moved), then re-run update-skill. A
|
|
388
|
+
citation_check that read nothing (UNREAD:
|
|
389
|
+
no-source-root, verifier-missing,
|
|
390
|
+
spot-check-failed) takes its own fix in
|
|
391
|
+
their place: give update-skill the source
|
|
392
|
+
tree, re-install SKF, or fix what the check
|
|
393
|
+
failed on, then re-run update-skill
|
|
274
394
|
rescoped the entry removed
|
|
395
|
+
The signature: when §4a found the export's record (its verification
|
|
396
|
+
record has an extraction_method), every part from it, a params or
|
|
397
|
+
return_type it gives as null removed from the entry; with none, an
|
|
398
|
+
entry the map holds keeps its signature, and an entry this run adds
|
|
399
|
+
takes --merge-records'. An entry the map holds keeps its confidence and
|
|
400
|
+
extraction_method; when its own extraction_method is ast-grep or
|
|
401
|
+
ast_bridge, it takes the signature_source of its record's labels (T1
|
|
402
|
+
for an ast-grep record, T1-low for a source-read one) and, when the
|
|
403
|
+
record's export_type is another, the record's ast_node_type, while a
|
|
404
|
+
source-read entry keeps T1-low and no node kind. Each signature_source
|
|
405
|
+
it changes is listed in `relabeled`, {name, file, from, to}, each with a
|
|
406
|
+
warning. An internal-unreachable NEW_EXPORT takes none.
|
|
275
407
|
Under the drift override (--drift-head and --drift-pinned) each warning
|
|
276
|
-
names the drift.
|
|
408
|
+
names the drift, after the outcome's cause.
|
|
277
409
|
|
|
278
410
|
full (degraded mode) makes every fresh record an entry.
|
|
279
411
|
|
|
@@ -295,9 +427,81 @@ Apply (update-skill write.md §3):
|
|
|
295
427
|
manual_sections_preserved. An older update-history key
|
|
296
428
|
(update_operations, update_metadata) stays as it is.
|
|
297
429
|
|
|
430
|
+
Lists. With --metadata-in (the skill's metadata.json as the update
|
|
431
|
+
found it) and --lists-out, it also writes the lists metadata.json takes,
|
|
432
|
+
{"exports": [...], "scripts": [...], "assets": [...]}. exports: the
|
|
433
|
+
--metadata-in exports[] less each name whose entry this run removed (a
|
|
434
|
+
DELETED_EXPORT, a DELETED file's or a MOVED file's old path's, a
|
|
435
|
+
rescoped one, a changed URL's) or renamed, unless an entry of the new
|
|
436
|
+
map still holds it, then each name it added an entry for (the entry's
|
|
437
|
+
reexported_as when it has one, the public name) or renamed an entry to,
|
|
438
|
+
each once; a renamed name takes its old name's place. A name
|
|
439
|
+
left out as off the public surface (the summary's not_public) has no
|
|
440
|
+
entry, so it is not added. A full update, which rebuilds the map, also
|
|
441
|
+
drops each name the new map does not hold, and a reference app
|
|
442
|
+
(scope_type reference-app) gets []. scripts and assets: one row per
|
|
443
|
+
file_entries[] row of the new map whose file_type is script or asset
|
|
444
|
+
(a row with none takes the type its file_name's folder names), in the
|
|
445
|
+
map's order: the --metadata-in scripts[] or assets[] row of its
|
|
446
|
+
source_file (else of its file_name) as it stands, else a new row {file:
|
|
447
|
+
its file_name, purpose, source_file, confidence (default T1-low)},
|
|
448
|
+
whose purpose skf-detect-scripts-assets.py reads from the file under
|
|
449
|
+
--source-root (its file name when no --source-root is given or the
|
|
450
|
+
file is not there).
|
|
451
|
+
|
|
452
|
+
Metadata (update-skill write.md §2):
|
|
453
|
+
|
|
454
|
+
metadata.json as --metadata holds it, with what the update sets: version
|
|
455
|
+
(--version) and generation_date (--generation-date, YYYY-MM-DDTHH:MM:SSZ);
|
|
456
|
+
source_commit and source_ref when --source-commit or --source-ref is
|
|
457
|
+
given (an empty one sets null), else as they stand; exports, scripts and
|
|
458
|
+
assets from --lists (apply's --lists-out), scripts or assets left out
|
|
459
|
+
when its list is empty; confidence_distribution from --stats (the JSON
|
|
460
|
+
skf-render-metadata-stats.py writes with -o), and stats: the stats
|
|
461
|
+
helper's fields, then each other field --metadata's stats holds (a
|
|
462
|
+
component library's own counts, a reference app's notes, a field a
|
|
463
|
+
gap-driven patch added), the stats helper's fields an earlier run wrote
|
|
464
|
+
that this run's does not (effective_denominator,
|
|
465
|
+
pattern_surfaces_documented) dropped and listed in `dropped_stats`,
|
|
466
|
+
each with a warning. When this run's effective_denominator differs
|
|
467
|
+
from the old one (one of them absent included), it also drops each old
|
|
468
|
+
stats key named effective_denominator_* (an explanation of the old
|
|
469
|
+
count), listed in `dropped_stats` with the warning "metadata:
|
|
470
|
+
stats.<key> dropped: it explained an effective_denominator this run
|
|
471
|
+
changed or removed"; when the two are the same, such a key stays with
|
|
472
|
+
no warning, since it still describes the denominator. Each other old
|
|
473
|
+
stats key it keeps adds the warning "metadata: stats.<key> kept from an
|
|
474
|
+
earlier run; check it still describes this run's counts", unless
|
|
475
|
+
create-skill writes it for the skill's scope_type (STATS_CREATE_FIELDS:
|
|
476
|
+
a component-library's Component Library stats of
|
|
477
|
+
compile-assembly-rules.md, a reference-app's notes).
|
|
478
|
+
|
|
479
|
+
tool_versions names the tools this run used: the old tool_versions
|
|
480
|
+
({"ast_grep": null, "qmd": null} when absent, or when it is not an
|
|
481
|
+
object, which adds a warning), with skf set to the version
|
|
482
|
+
skf-render-quick-metadata.py's probe_skf_version reads under
|
|
483
|
+
--skf-root (package.json, else VERSION), never the old value, or to
|
|
484
|
+
"unknown" with a warning, one when no --skf-root is given and another
|
|
485
|
+
naming the folder when it holds no version; ast_grep set to the
|
|
486
|
+
ast_grep.version of --extraction (the recipe runner's output), or to
|
|
487
|
+
"unknown" with a warning when that is null but an export of it has
|
|
488
|
+
extraction_method ast-grep; qmd set to --qmd-version, a leading "qmd "
|
|
489
|
+
(as `qmd --version` prints it) removed. A tool this run did not use
|
|
490
|
+
(no --qmd-version, no --extraction, or one with no export ast-grep
|
|
491
|
+
matched) keeps its old value. It removes last_update and update_type,
|
|
492
|
+
which the provenance map's update block records, and keeps every
|
|
493
|
+
other field. It refuses (exit 1, nothing written, one stderr line) a
|
|
494
|
+
--stats whose scripts_count or assets_count is not the length of
|
|
495
|
+
--lists' scripts or assets, a --generation-date of another form, an
|
|
496
|
+
--extraction that is not a JSON object, a file it cannot read, and a
|
|
497
|
+
skf-render-quick-metadata.py beside it that cannot be loaded.
|
|
498
|
+
|
|
298
499
|
Summary on stdout: {"status": "written", "map": <-o>, "entries":
|
|
299
|
-
{"added", "updated", "removed"}, "
|
|
300
|
-
"
|
|
500
|
+
{"added", "updated", "removed"}, "not_public": [{name, file}],
|
|
501
|
+
"relabeled": [{name, file, from, to}] (gap-driven),
|
|
502
|
+
"signature_source_backfilled": {tier: count}, "file_entries":
|
|
503
|
+
{"added", "updated", "removed"}, "warnings": [...]},
|
|
504
|
+
or {"status": "refused",
|
|
301
505
|
"blocking_unresolved": [{export_name, severity}]} with exit 3.
|
|
302
506
|
|
|
303
507
|
Records (update-skill re-extract.md §4, and gap-driven.md §4a for the files
|
|
@@ -324,6 +528,42 @@ it scans):
|
|
|
324
528
|
no seed holds is not added, with a warning. confidence_breakdown counts
|
|
325
529
|
the exports by confidence, and T2 the exports with a qmd_evidence.
|
|
326
530
|
|
|
531
|
+
Aliases. Each --extraction entry_point_diff `public` and
|
|
532
|
+
`other_definitions` item (an other in-scope definition of a public
|
|
533
|
+
name), and each member of a namespace item, whose `local` differs from
|
|
534
|
+
its `name` gives its name to the record of its `local` name and file,
|
|
535
|
+
when one is seeded:
|
|
536
|
+
that record's `aliases` lists each such public name once, sorted (an
|
|
537
|
+
export re-exported under another name, `export { MockUniversalStore as
|
|
538
|
+
experimental_MockUniversalStore }`, keeps its declared name and carries
|
|
539
|
+
the public one). A record no item renames has no `aliases`, and with
|
|
540
|
+
no entry_point_diff (Quick tier, a runner exit 3, an export read by
|
|
541
|
+
eye) no record has any.
|
|
542
|
+
|
|
543
|
+
Public surface. When --extraction's scope.type is public-api (in any
|
|
544
|
+
case) and it has an entry_point_diff, each export of a language family
|
|
545
|
+
whose entry_points.by_language status is `barrel` is marked `public`:
|
|
546
|
+
true when its name and file are a pair of the runner's entry_point_diff
|
|
547
|
+
`public` or `other_definitions` (its name, or the name `local` gives it
|
|
548
|
+
in that file) or `extraction_gaps`, or when a `public` item of its name
|
|
549
|
+
and family has no file; false otherwise. A `public` item with `via`
|
|
550
|
+
namespace and a
|
|
551
|
+
`members` list (what the submodule passes on, which users reach as
|
|
552
|
+
`pkg.sub.name`, and which counts.exports_public_api counts in its place)
|
|
553
|
+
gives its members' pairs (each member's name, or its `local` one, and
|
|
554
|
+
its file; a member with no file its name) in place of its own; one
|
|
555
|
+
whose `members` is null (a module the runner could not read), or that
|
|
556
|
+
has no `members` key (an older runner's), gives its own pair, as any
|
|
557
|
+
item does.
|
|
558
|
+
The rule is skf-extraction-inventory.py's public_surface, loaded from
|
|
559
|
+
this folder, by which create-skill marks its extraction inventory too.
|
|
560
|
+
In a normal update merge documents no export marked false that the map
|
|
561
|
+
does not hold, and apply adds none; gap-records drops the mark, and a
|
|
562
|
+
full (degraded) apply adds every record. Any other scope type, a family
|
|
563
|
+
with no barrel, and a run with no entry_point_diff, an incomplete one,
|
|
564
|
+
one with errors or one whose entry_points.unresolved is not empty mark
|
|
565
|
+
nothing.
|
|
566
|
+
|
|
327
567
|
Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
328
568
|
|
|
329
569
|
Reads the change manifest skf-parse-gaps.py translate wrote. Its
|
|
@@ -336,8 +576,11 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
|
336
576
|
but a MOVED_EXPORT needs the tree, with the first reason that fits:
|
|
337
577
|
a DELETED_EXPORT "a public API recount from the tree (rule R1)", a
|
|
338
578
|
provenance_completeness entry "a line from the tree (rule R3)", one
|
|
339
|
-
whose map_match is not-found
|
|
340
|
-
|
|
579
|
+
whose map_match is not-found with no candidate that has a
|
|
580
|
+
source_file "a line and a signature from the tree", any other (a
|
|
581
|
+
not-found with such a candidate, a citation of another file than
|
|
582
|
+
the map's entries, among them) "a signature from the tree". Any
|
|
583
|
+
such entry: {"status":
|
|
341
584
|
"drift-blocked", "drift_blocked": [{gap_id, name, change_category,
|
|
342
585
|
severity, reason}]}, and no spot-check runs.
|
|
343
586
|
2. A DELETED_EXPORT without a `rescope` that holds its `amendment` and
|
|
@@ -353,10 +596,28 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
|
353
596
|
its remediation's "definition line(s) (...)"
|
|
354
597
|
NEW_EXPORT, MODIFIED_EXPORT
|
|
355
598
|
map_match found checked at the map entry (its export_type)
|
|
356
|
-
ambiguous unknown
|
|
599
|
+
ambiguous unknown, unknown_reason ambiguous (several
|
|
600
|
+
map entries of its name: no check runs)
|
|
357
601
|
not-found checked at its source_citation (no export
|
|
358
|
-
type) when it has one
|
|
359
|
-
|
|
602
|
+
type) when it has one. When the map holds
|
|
603
|
+
entries of its name (in_map, which a
|
|
604
|
+
not-found has only for a citation of
|
|
605
|
+
another file than theirs), one of them
|
|
606
|
+
with a source_file, and no check pins a
|
|
607
|
+
line (unknown or missing): unknown,
|
|
608
|
+
unknown_reason cited-elsewhere, whatever
|
|
609
|
+
its severity (a provenance_completeness
|
|
610
|
+
one too), and never re-extracted, so no
|
|
611
|
+
other file's declaration is read for it;
|
|
612
|
+
its record's citation_check keeps the
|
|
613
|
+
check's cause (missing, or its
|
|
614
|
+
unknown_reason) and its map_locations the
|
|
615
|
+
`file:line` (or file) of each candidate
|
|
616
|
+
with a source_file, both null on any other
|
|
617
|
+
record. Candidates with no source_file
|
|
618
|
+
(placeholders) do not count. Otherwise,
|
|
619
|
+
when that pins no line
|
|
620
|
+
(unknown or missing), or with no
|
|
360
621
|
citation: targeted re-extraction (§4a) for a
|
|
361
622
|
provenance_completeness entry with a source
|
|
362
623
|
root, for a blocking one whatever its
|
|
@@ -366,36 +627,78 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
|
366
627
|
A check gives `verified` (the line is a definition line, or a module
|
|
367
628
|
or package entry whose file exists), `moved` (one definition line,
|
|
368
629
|
not the recorded one: new_location), `missing` (no such file) or
|
|
369
|
-
`unknown` (several
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
unknown, with a `provenance: spot-
|
|
376
|
-
warning
|
|
630
|
+
`unknown`, its unknown_reason `several-definitions` (several
|
|
631
|
+
definition lines, none the recorded one) or `no-definition` (none).
|
|
632
|
+
A file the rules cover no language of takes the answered
|
|
633
|
+
`declaring_line`: the recorded line is verified, another is moved,
|
|
634
|
+
null is unknown (`unchecked-language`). With no --source-root every
|
|
635
|
+
check is unknown (`no-source-root`); with no verifier every check is
|
|
636
|
+
unknown (`verifier-missing`), with a `provenance: spot-checks not
|
|
637
|
+
run` warning; a check the verifier fails on (a file it cannot read,
|
|
638
|
+
or any error) is unknown (`spot-check-failed`), with a `provenance:
|
|
639
|
+
spot-check failed for {name}: {error}` warning; a map entry with no
|
|
640
|
+
file or line is unknown (`no-line`). UNKNOWN_REASONS gives each its
|
|
641
|
+
words. Every other outcome has a null unknown_reason. A map_match
|
|
642
|
+
whose entry carries its own export_name (skf-parse-gaps matched it
|
|
643
|
+
by its reexported_as) is checked under that name.
|
|
644
|
+
A NEW_EXPORT or MODIFIED_EXPORT the map holds entries of its name
|
|
645
|
+
for (its map_match found or ambiguous, or not-found with a
|
|
646
|
+
candidate that has a source_file) never goes to targeted
|
|
647
|
+
re-extraction, whatever its severity: one that ends missing or
|
|
648
|
+
unknown keeps its map entries unchanged and adds none, and apply
|
|
649
|
+
warns (see "Apply").
|
|
650
|
+
A NEW_EXPORT or MODIFIED_EXPORT a check gives verified or moved, at
|
|
651
|
+
its map entry or at a citation the map holds no entry of its name
|
|
652
|
+
for, is located: its signature_at is the checked file and the
|
|
653
|
+
recorded line (verified) or the new one (moved). One whose citation
|
|
654
|
+
names another file than the map's entries of its name is not (apply
|
|
655
|
+
warns, as before), nor one whose file is absolute or outside
|
|
656
|
+
--source-root, links resolved, nor a NEW_EXPORT --judgments answers
|
|
657
|
+
internal-unreachable (one not yet answered is located, so its scan
|
|
658
|
+
may go unused).
|
|
377
659
|
--plan stops here: {"status": "planned", "reextract": {"entries":
|
|
378
660
|
[{gap_id, name, severity, resolved_paths}], "files": [the union of
|
|
379
|
-
their resolved_paths
|
|
380
|
-
|
|
661
|
+
their resolved_paths, then of the located exports' files],
|
|
662
|
+
"signatures": [{gap_id, name, file, line} of each located export]},
|
|
663
|
+
"warnings"}, the files also written to --files-out as a JSON array.
|
|
381
664
|
4. Targeted re-extraction (--remediation-records, `records`' output
|
|
382
665
|
over those files): each routed entry with resolved_paths takes the
|
|
383
666
|
first export of its name in the blocks of those files, in their
|
|
384
|
-
order
|
|
667
|
+
order, else the first whose `aliases` hold it (matched by name, or
|
|
668
|
+
by the public name the runner gives its local name in that file,
|
|
669
|
+
see "Records"): `re-extracted`, new_location its record's location,
|
|
385
670
|
resolution_source remediation-paths, and the record copied into
|
|
386
|
-
`files` as `records` wrote it,
|
|
671
|
+
`files` as `records` wrote it, less its `public` mark (a normal
|
|
672
|
+
update's) and its `aliases`, with the answered `docstring` added. A
|
|
673
|
+
record matched by an alias is copied with `name` the public name and
|
|
674
|
+
`local` its own name. The aliases come from the runner's
|
|
675
|
+
entry_point_diff, so at Forge tier and above only: at Quick tier, or
|
|
676
|
+
when the runner exits 3 and §4a reads the files by eye, a gap naming
|
|
677
|
+
a renamed re-export by its public name matches no record.
|
|
387
678
|
An entry with no match, or no path to scan, is unresolved, except a
|
|
388
679
|
missing-export or missing-type one that is not blocking, which is
|
|
389
680
|
unknown. Any unresolved entry: {"status": "unresolved",
|
|
390
681
|
"unresolved": [{gap_id, name, severity, remediation_paths,
|
|
391
682
|
rejected_paths, files_scanned, exports_found_in_scan}]}.
|
|
683
|
+
Each located export takes the one record of its name (or of its map
|
|
684
|
+
entry's own export_name) in the block of its file at its line, else
|
|
685
|
+
the one with no line: the record copied into `files` as above, under
|
|
686
|
+
the gap's name, less its `public` mark and `aliases`, with no
|
|
687
|
+
docstring asked
|
|
688
|
+
(one copy per name and location, or per record with no location),
|
|
689
|
+
and its type, params, return_type, extraction_method and
|
|
690
|
+
ast_node_type on the verification record. One with no such record
|
|
691
|
+
keeps them null, with a `provenance: {name}: located at
|
|
692
|
+
{file}:{line}, but no record of it there; signature not repaired`
|
|
693
|
+
warning (§4a's by-eye pass may add it to --export-details). Files to
|
|
694
|
+
match, of either kind, with no --remediation-records are an error.
|
|
392
695
|
5. The public-reachability gate: every NEW_EXPORT that is verified,
|
|
393
696
|
moved or re-extracted takes the answered `reachability`, `public` or
|
|
394
697
|
`internal-unreachable`. An internal one keeps its outcome in its
|
|
395
|
-
verification record, its record is not in
|
|
396
|
-
it under `reclassified` instead of its
|
|
397
|
-
`reclassified` (merge queues it as a
|
|
398
|
-
entry for it).
|
|
698
|
+
verification record (with no signature), its record is not in
|
|
699
|
+
`files`, `counts` tallies it under `reclassified` instead of its
|
|
700
|
+
outcome, and it is listed in `reclassified` (merge queues it as a
|
|
701
|
+
metadata update; apply writes no entry for it).
|
|
399
702
|
|
|
400
703
|
Any answer missing in step 3 or 5 (a `docstring` too, for each
|
|
401
704
|
re-extracted entry): {"status": "needs-judgment", "needs_judgment":
|
|
@@ -407,25 +710,57 @@ Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
|
407
710
|
"unlabeled", "T2": 0} (each manifest entry by the extraction_method of
|
|
408
711
|
its re-extracted record or, in the map, of its map entry; a pinned cited
|
|
409
712
|
export the map lacks and the gate passed is T1-low), "verification":
|
|
410
|
-
[one record per export-bearing entry:
|
|
411
|
-
severity, verification, in_map, map_entry,
|
|
412
|
-
source_citation, new_location, unknown_reason,
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
713
|
+
[one record per export-bearing entry: gap_id, export_name,
|
|
714
|
+
gap_category, severity, verification, in_map, map_entry,
|
|
715
|
+
provenance_citation, source_citation, new_location, unknown_reason,
|
|
716
|
+
citation_check, map_locations, pinned_definition_lines,
|
|
717
|
+
resolution_source, reachability, and
|
|
718
|
+
export_type, params, return_type, extraction_method and ast_node_type
|
|
719
|
+
(a located export's record's, step 4; else null, and apply takes the
|
|
720
|
+
signature of an entry it adds from merge.md's merge-records.json)],
|
|
721
|
+
"files": [...]}, and prints {"status": "written",
|
|
416
722
|
"output", "gap_count", "files_extracted", "exports_extracted",
|
|
417
723
|
"confidence_breakdown", "counts" (per verification outcome, and
|
|
418
724
|
`reclassified`), "reclassified", "forwarded", "targeted_reextraction",
|
|
419
|
-
"warnings"}. When targeted re-extraction
|
|
420
|
-
|
|
421
|
-
|
|
725
|
+
"warnings"}. When §4a ran (targeted re-extraction, or a located
|
|
726
|
+
export's signature), `targeted_reextraction` is {resolved_count,
|
|
727
|
+
files_scanned (the targeted re-extraction's files),
|
|
728
|
+
signature_files_scanned (the located exports' files), exports_matched,
|
|
729
|
+
tier}, also appended to --evidence as one JSON line keyed
|
|
730
|
+
`targeted_reextraction`.
|
|
731
|
+
|
|
732
|
+
Baseline gaps (update-skill detect-changes Category B, step 1):
|
|
733
|
+
|
|
734
|
+
The recipe runner never reports a module, an alias that renames, a
|
|
735
|
+
dunder such as `__version__` or another Python or Rust `_` name, so the
|
|
736
|
+
diff would read each such map entry as a deleted export. baseline-gaps
|
|
737
|
+
lists, for step 2 to read by eye, each entry of a file --files lists
|
|
738
|
+
(the modified files, Category A's modified-files.json) whose name and file
|
|
739
|
+
no export of --extraction (the runner's JSON, or {"exports": []} when
|
|
740
|
+
it could not run) holds, under its name or its re-export target, and
|
|
741
|
+
whose file still declares it. The selection and the declaration rule
|
|
742
|
+
are skf-verify-provenance-completeness.py's `baseline_gaps` (the rule
|
|
743
|
+
audit-skill's extraction snapshot applies), loaded from this folder: a
|
|
744
|
+
verifier that cannot be loaded is an error, never an empty list. A
|
|
745
|
+
name the runner's own `entry_point_diff.extraction_gaps` lists at that
|
|
746
|
+
file is left to that list. A map whose `entries` is missing or null
|
|
747
|
+
has none to check. Writes {"gaps": [{name, file, line, entry: null,
|
|
748
|
+
export_type?}], "unchecked": [{name, file, export_type}]}: a gap's line
|
|
749
|
+
declares the name (line 1 of the module or package so named for a
|
|
750
|
+
`module` or `package` entry), and `unchecked` holds each entry no rule
|
|
751
|
+
can check, to read by eye: its file neither Python nor TS/JS, a file
|
|
752
|
+
the lookup will not read (outside the source root, a test file or a
|
|
753
|
+
minified bundle, a symlink on its path), or a dotted `module` or
|
|
754
|
+
`package` name.
|
|
422
755
|
|
|
423
756
|
Exit codes:
|
|
424
757
|
0 operation succeeded (for gap-records, whatever its `status`)
|
|
425
758
|
1 user error (malformed JSON, bad path, malformed provenance file, a
|
|
426
759
|
helper file that is not the output it names, a gap-records answer of
|
|
427
|
-
the wrong type,
|
|
428
|
-
--remediation-records
|
|
760
|
+
the wrong type, files to match (targeted re-extraction or a located
|
|
761
|
+
export's signature) with no --remediation-records, or, for
|
|
762
|
+
baseline-gaps, a source root that is no folder or a verifier that
|
|
763
|
+
cannot be loaded)
|
|
429
764
|
3 apply refused: a blocking gap reached the provenance write with no
|
|
430
765
|
source line (nothing written)
|
|
431
766
|
"""
|
|
@@ -538,11 +873,20 @@ def category_b_from_diff(diff: dict) -> dict:
|
|
|
538
873
|
lists["modified_exports"].append({"name": key[0], "file": key[1], "old_line": None, "new_line": None})
|
|
539
874
|
modified.add(key)
|
|
540
875
|
for item in _get_list(diff, "moved"):
|
|
541
|
-
if isinstance(item, dict)
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
876
|
+
if not isinstance(item, dict):
|
|
877
|
+
continue
|
|
878
|
+
key, old_file = (item.get("name"), item.get("current_file")), _norm_path(item.get("previous_file"))
|
|
879
|
+
across = old_file not in (None, _norm_path(item.get("current_file"))) # apply finds the entry at old_file
|
|
880
|
+
if key in modified:
|
|
881
|
+
if across: # moved across files and changed: a MODIFIED_EXPORT that names the file it left
|
|
882
|
+
for entry in lists["modified_exports"]:
|
|
883
|
+
if (entry["name"], entry["file"]) == key:
|
|
884
|
+
entry.update(old_line=item.get("previous_line"), old_file=old_file)
|
|
885
|
+
continue
|
|
886
|
+
move = {"name": item.get("name"), "file": item.get("current_file")}
|
|
887
|
+
if across:
|
|
888
|
+
move["old_file"] = old_file
|
|
889
|
+
lists["moved_exports"].append({**move, "old_line": item.get("previous_line"), "new_line": item.get("line")})
|
|
546
890
|
return lists
|
|
547
891
|
|
|
548
892
|
|
|
@@ -738,12 +1082,15 @@ def _build_per_file(
|
|
|
738
1082
|
path = entry.get("file")
|
|
739
1083
|
if not path:
|
|
740
1084
|
return
|
|
741
|
-
|
|
1085
|
+
change = {
|
|
742
1086
|
"name": entry.get("name"),
|
|
743
1087
|
"change_type": change_type,
|
|
744
1088
|
"old_line": entry.get("old_line"),
|
|
745
1089
|
"new_line": entry.get("new_line") or entry.get("line"),
|
|
746
|
-
}
|
|
1090
|
+
}
|
|
1091
|
+
if entry.get("old_file"):
|
|
1092
|
+
change["old_file"] = entry["old_file"] # a move across files: a MOVED_EXPORT, or a MODIFIED_EXPORT
|
|
1093
|
+
exports_by_file.setdefault(path, []).append(change)
|
|
747
1094
|
|
|
748
1095
|
for e in b_modified:
|
|
749
1096
|
_record(e, "MODIFIED_EXPORT")
|
|
@@ -943,6 +1290,34 @@ def _file_of(record: dict) -> str | None:
|
|
|
943
1290
|
return _norm_path(record.get("source_file", record.get("file", record.get("file_path"))))
|
|
944
1291
|
|
|
945
1292
|
|
|
1293
|
+
_INVENTORY = None
|
|
1294
|
+
|
|
1295
|
+
|
|
1296
|
+
def _inventory():
|
|
1297
|
+
"""skf-extraction-inventory.py, loaded once from this folder: its typed_param and its public-surface rule, so
|
|
1298
|
+
create-skill and update-skill write one parameter form and keep one public surface."""
|
|
1299
|
+
global _INVENTORY
|
|
1300
|
+
if _INVENTORY is None:
|
|
1301
|
+
path = Path(__file__).resolve().parent / "skf-extraction-inventory.py"
|
|
1302
|
+
spec = importlib.util.spec_from_file_location("skf_extraction_inventory", path)
|
|
1303
|
+
if spec is None or spec.loader is None:
|
|
1304
|
+
raise ImportError(f"cannot load {path}")
|
|
1305
|
+
module = importlib.util.module_from_spec(spec)
|
|
1306
|
+
spec.loader.exec_module(module)
|
|
1307
|
+
_INVENTORY = module
|
|
1308
|
+
return _INVENTORY
|
|
1309
|
+
|
|
1310
|
+
|
|
1311
|
+
def public_surface(extraction: object) -> tuple[object | None, str | None]:
|
|
1312
|
+
"""skf-extraction-inventory.py's public_surface over the recipe runner's output (see "Records" in the module
|
|
1313
|
+
docstring): (the public surface, None) for a public-api run, (None, why) when its surface cannot be read, and
|
|
1314
|
+
(None, None) for any other scope type. An input that is no object (a gap-driven or docs-only run gives none)
|
|
1315
|
+
is (None, None) without loading the inventory helper."""
|
|
1316
|
+
if not isinstance(extraction, dict):
|
|
1317
|
+
return None, None
|
|
1318
|
+
return _inventory().public_surface(extraction)
|
|
1319
|
+
|
|
1320
|
+
|
|
946
1321
|
# --------------------------------------------------------------------------
|
|
947
1322
|
# Rename candidates (update-skill detect-changes Category C)
|
|
948
1323
|
# --------------------------------------------------------------------------
|
|
@@ -1150,6 +1525,8 @@ REPLACE_WAIT_SECONDS = 5.0 # how long a Windows rename onto a file held open is
|
|
|
1150
1525
|
# The update block apply sets at the top level of the map, in this order.
|
|
1151
1526
|
UPDATE_BLOCK_KEYS = ("last_update", "update_type", "test_report_run_id", "files_changed", "exports_affected",
|
|
1152
1527
|
"confidence_tier", "manual_sections_preserved")
|
|
1528
|
+
# The parts of a signature a gap-driven repair writes from a verification record, else from merge's records.
|
|
1529
|
+
SIGNATURE_PARTS = ("export_type", "params", "return_type")
|
|
1153
1530
|
|
|
1154
1531
|
|
|
1155
1532
|
def _labels(method: object, node: object) -> dict:
|
|
@@ -1192,6 +1569,7 @@ class _Fresh:
|
|
|
1192
1569
|
self.runner = {}
|
|
1193
1570
|
self.details = {}
|
|
1194
1571
|
self.worker = {}
|
|
1572
|
+
self.surface, self.surface_problem = public_surface(extraction)
|
|
1195
1573
|
for record in _exports_of(extraction, "--extraction file"):
|
|
1196
1574
|
key = (_name_of(record), _file_of(record))
|
|
1197
1575
|
if all(key):
|
|
@@ -1215,6 +1593,17 @@ class _Fresh:
|
|
|
1215
1593
|
def all_keys(self) -> list[tuple]:
|
|
1216
1594
|
return sorted({k for source in (self.runner, self.details, self.worker) for k in source})
|
|
1217
1595
|
|
|
1596
|
+
def public(self, name: str, path: str) -> bool | None:
|
|
1597
|
+
"""The `public` mark `records` gave the export, else the public-surface rule over --extraction; None when
|
|
1598
|
+
neither marks it."""
|
|
1599
|
+
mark = (self.worker.get((name, path)) or {}).get("public")
|
|
1600
|
+
if isinstance(mark, bool):
|
|
1601
|
+
return mark
|
|
1602
|
+
if self.surface is None:
|
|
1603
|
+
return None
|
|
1604
|
+
record = self.runner.get((name, path)) or self.details.get((name, path)) or {}
|
|
1605
|
+
return self.surface.public(name, path, record.get("language"))
|
|
1606
|
+
|
|
1218
1607
|
def record(self, name: str, path: str) -> dict | None:
|
|
1219
1608
|
"""The entry fields of an export's fresh record, or None when no source holds it."""
|
|
1220
1609
|
key = (name, path)
|
|
@@ -1265,16 +1654,17 @@ def _without_none(fields: dict, keep: tuple = ()) -> dict:
|
|
|
1265
1654
|
|
|
1266
1655
|
def _new_entry(fields: dict, library: str) -> dict:
|
|
1267
1656
|
"""An entry in create-skill's key order, from the fields a record gives."""
|
|
1268
|
-
order = ("export_name", "export_type", "source_library", "params", "return_type", "source_file",
|
|
1657
|
+
order = ("export_name", "reexported_as", "export_type", "source_library", "params", "return_type", "source_file",
|
|
1269
1658
|
"source_line", "confidence", "extraction_method", "ast_node_type", "signature_source")
|
|
1270
1659
|
merged = {**fields, "source_library": fields.get("source_library") or library}
|
|
1271
1660
|
out = {k: merged[k] for k in order if k in merged and (merged[k] is not None or k == "ast_node_type")}
|
|
1272
1661
|
return out
|
|
1273
1662
|
|
|
1274
1663
|
|
|
1275
|
-
def _rewrite(entry: dict, fields: dict) -> dict:
|
|
1276
|
-
"""The entry with `fields` set in place; keys the update does not set keep
|
|
1277
|
-
|
|
1664
|
+
def _rewrite(entry: dict, fields: dict, drop: tuple = ()) -> dict:
|
|
1665
|
+
"""The entry with `fields` set in place and the keys `drop` names removed; keys the update does not set keep
|
|
1666
|
+
their values."""
|
|
1667
|
+
out = {k: v for k, v in entry.items() if k not in drop}
|
|
1278
1668
|
for key, value in fields.items():
|
|
1279
1669
|
if value is None and key not in ("ast_node_type",):
|
|
1280
1670
|
continue
|
|
@@ -1298,18 +1688,87 @@ class _Map:
|
|
|
1298
1688
|
"provenance_version": "2.0", "skill_name": skill_name, "skill_type": "single", "entries": []}
|
|
1299
1689
|
if not isinstance(self.doc.get("entries"), list):
|
|
1300
1690
|
raise ValueError("provenance `entries` must be an array")
|
|
1301
|
-
libraries
|
|
1302
|
-
|
|
1303
|
-
self.
|
|
1691
|
+
# the map's libraries in entry order, and each file's, as the map stood before the run (#808)
|
|
1692
|
+
self.libraries: list[str] = []
|
|
1693
|
+
self.libraries_by_file: dict[str, set[str]] = {}
|
|
1694
|
+
for entry in self.doc["entries"]:
|
|
1695
|
+
library = entry.get("source_library") if isinstance(entry, dict) else None
|
|
1696
|
+
if not (isinstance(library, str) and library):
|
|
1697
|
+
continue
|
|
1698
|
+
if library not in self.libraries:
|
|
1699
|
+
self.libraries.append(library)
|
|
1700
|
+
if _file_of(entry):
|
|
1701
|
+
self.libraries_by_file.setdefault(_file_of(entry), set()).add(library)
|
|
1702
|
+
self.library = self.libraries[0] if self.libraries else skill_name
|
|
1703
|
+
# each folder's libraries (the entries under it, "" the root) and _library_at per path, built on first use
|
|
1704
|
+
self._libraries_under: dict[str, set[str]] | None = None
|
|
1705
|
+
self._library_of: dict[str, str | None] = {}
|
|
1706
|
+
# (name, file) of each new entry library_for gave the first library for want of a better one
|
|
1707
|
+
self.library_unresolved: list[tuple[str | None, str | None]] = []
|
|
1304
1708
|
self.added: list[str] = []
|
|
1305
1709
|
self.updated: list[str] = []
|
|
1306
1710
|
self.removed: list[str] = []
|
|
1711
|
+
self.renamed: list[tuple[str, str]] = [] # (old name, new name)
|
|
1307
1712
|
self.touched_files: set[str] = set()
|
|
1308
1713
|
|
|
1309
|
-
def
|
|
1310
|
-
"""The
|
|
1311
|
-
|
|
1312
|
-
|
|
1714
|
+
def library_for(self, path: str | None, name: str | None = None, *, old: str | None = None) -> str:
|
|
1715
|
+
"""The source_library of a new entry for `name` in `path` (the record of a MOVED file, or of an export
|
|
1716
|
+
whose change names an old_file, passes that old path as `old`, tried second). A map naming one library (or
|
|
1717
|
+
none: --skill-name) gives it; in one naming several, the library of the file, else of the nearest folder
|
|
1718
|
+
(see _library_at). With neither, or no path, the map's first library, and the entry goes to
|
|
1719
|
+
`library_unresolved` for the summary's warning."""
|
|
1720
|
+
if len(self.libraries) <= 1:
|
|
1721
|
+
return self.library
|
|
1722
|
+
for candidate in (path, old):
|
|
1723
|
+
library = self._library_at(_norm_path(candidate))
|
|
1724
|
+
if library is not None:
|
|
1725
|
+
return library
|
|
1726
|
+
self.library_unresolved.append((name, _norm_path(path) or _norm_path(old)))
|
|
1727
|
+
return self.library
|
|
1728
|
+
|
|
1729
|
+
@staticmethod
|
|
1730
|
+
def _root_library(libraries: set[str]) -> str | None:
|
|
1731
|
+
"""The one library of the set, or of several the one that is a path prefix of all the others (`storybook`
|
|
1732
|
+
beside `storybook/test`); None when no library is."""
|
|
1733
|
+
return next((library for library in sorted(libraries)
|
|
1734
|
+
if all(other == library or other.startswith(library + "/") for other in libraries)), None)
|
|
1735
|
+
|
|
1736
|
+
def _library_at(self, path: str | None) -> str | None:
|
|
1737
|
+
"""The library the map's entries of `path` settle (see _root_library), else the one the entries under the
|
|
1738
|
+
nearest folder above it that holds entries settle; None when neither does. Memoized per path."""
|
|
1739
|
+
if not path:
|
|
1740
|
+
return None
|
|
1741
|
+
if path in self._library_of:
|
|
1742
|
+
return self._library_of[path]
|
|
1743
|
+
if self._libraries_under is None:
|
|
1744
|
+
self._libraries_under = {}
|
|
1745
|
+
for file, libraries in self.libraries_by_file.items():
|
|
1746
|
+
folders = file.split("/")[:-1]
|
|
1747
|
+
for depth in range(len(folders) + 1):
|
|
1748
|
+
prefix = "/".join(folders[:depth]) + "/" if depth else ""
|
|
1749
|
+
self._libraries_under.setdefault(prefix, set()).update(libraries)
|
|
1750
|
+
library = self._root_library(self.libraries_by_file.get(path, set()))
|
|
1751
|
+
if library is None:
|
|
1752
|
+
folders = path.split("/")[:-1]
|
|
1753
|
+
for depth in range(len(folders), -1, -1):
|
|
1754
|
+
under = self._libraries_under.get("/".join(folders[:depth]) + "/" if depth else "")
|
|
1755
|
+
if under:
|
|
1756
|
+
library = self._root_library(under)
|
|
1757
|
+
break
|
|
1758
|
+
self._library_of[path] = library
|
|
1759
|
+
return library
|
|
1760
|
+
|
|
1761
|
+
def find(self, name: str, path: str | None, line: int | None = None, *, reexport: bool = False) -> int | None:
|
|
1762
|
+
"""The index of the entry with this name (and file, and line when several share both); with `reexport`,
|
|
1763
|
+
when no entry has the name, of the entry whose reexported_as is the name (a gap names a renamed re-export
|
|
1764
|
+
by its public name)."""
|
|
1765
|
+
def hits_of(key) -> list[int]:
|
|
1766
|
+
return [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and key(e) == name
|
|
1767
|
+
and (path is None or _file_of(e) == path)]
|
|
1768
|
+
|
|
1769
|
+
hits = hits_of(_name_of)
|
|
1770
|
+
if not hits and reexport:
|
|
1771
|
+
hits = hits_of(lambda e: e.get("reexported_as"))
|
|
1313
1772
|
if len(hits) > 1 and line is not None:
|
|
1314
1773
|
hits = [i for i in hits if _line_of(self.doc["entries"][i].get("source_line")) == line]
|
|
1315
1774
|
return hits[0] if len(hits) == 1 else None
|
|
@@ -1323,12 +1782,14 @@ class _Map:
|
|
|
1323
1782
|
if entry.get("source_file"):
|
|
1324
1783
|
self.touched_files.add(_norm_path(entry["source_file"]))
|
|
1325
1784
|
|
|
1326
|
-
def update(self, index: int, fields: dict) -> None:
|
|
1785
|
+
def update(self, index: int, fields: dict, drop: tuple = ()) -> None:
|
|
1327
1786
|
before = self.doc["entries"][index]
|
|
1328
|
-
after = _rewrite(before, fields)
|
|
1787
|
+
after = _rewrite(before, fields, drop)
|
|
1329
1788
|
if after != before:
|
|
1330
1789
|
self.doc["entries"][index] = after
|
|
1331
1790
|
self.updated.append(_name_of(after))
|
|
1791
|
+
if _name_of(before) and _name_of(after) and _name_of(before) != _name_of(after):
|
|
1792
|
+
self.renamed.append((_name_of(before), _name_of(after)))
|
|
1332
1793
|
for path in (_file_of(before), _file_of(after)):
|
|
1333
1794
|
if path:
|
|
1334
1795
|
self.touched_files.add(path)
|
|
@@ -1342,19 +1803,39 @@ class _Map:
|
|
|
1342
1803
|
|
|
1343
1804
|
|
|
1344
1805
|
def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fresh: _Fresh,
|
|
1345
|
-
warnings: list[str]) -> tuple[int, int]:
|
|
1346
|
-
"""Normal mode: the manifest's file and export changes. Returns (files, exports) for the update block
|
|
1806
|
+
warnings: list[str], not_public: list[dict]) -> tuple[int, int]:
|
|
1807
|
+
"""Normal mode: the manifest's file and export changes. Returns (files, exports) for the update block; each
|
|
1808
|
+
name it leaves out as off a public-api skill's public surface goes to `not_public`."""
|
|
1347
1809
|
per_file = [f for f in _get_list(manifest or {}, "per_file") if isinstance(f, dict)]
|
|
1348
|
-
|
|
1349
|
-
|
|
1810
|
+
# the names of the entries this run removes (a DELETED_EXPORT, a DELETED file's, a MOVED file's old path's):
|
|
1811
|
+
# one that comes back in another file is re-added whatever its mark, so no held export is lost
|
|
1812
|
+
removed: set[str] = set()
|
|
1813
|
+
for item in per_file:
|
|
1814
|
+
path = _norm_path(item.get("old_path") if item.get("status") == "MOVED" else item.get("file_path"))
|
|
1815
|
+
if item.get("status") in ("DELETED", "MOVED") and path:
|
|
1816
|
+
removed.update(_name_of(pmap.doc["entries"][i]) for i in pmap.indexes_of(path))
|
|
1817
|
+
removed.update(_name_of(c) for c in item.get("exports_affected") or []
|
|
1818
|
+
if isinstance(c, dict) and c.get("change_type") == "DELETED_EXPORT")
|
|
1819
|
+
listed: set[tuple[str, str]] = set()
|
|
1820
|
+
|
|
1821
|
+
def off_surface(name: str, path: str) -> bool:
|
|
1822
|
+
"""True for a name the run would add that is off the public surface, which it lists."""
|
|
1823
|
+
if name in removed or fresh.public(name, path) is not False:
|
|
1824
|
+
return False
|
|
1825
|
+
if (name, path) not in listed:
|
|
1826
|
+
listed.add((name, path))
|
|
1827
|
+
not_public.append({"name": name, "file": path})
|
|
1828
|
+
return True
|
|
1829
|
+
|
|
1830
|
+
def refresh(index: int | None, name: str, path: str, *, add: bool, old: str | None = None) -> None:
|
|
1350
1831
|
record = fresh.record(name, path)
|
|
1351
1832
|
if record is None:
|
|
1352
1833
|
warnings.append(f"provenance: {name} in {path}: no extraction record, entry "
|
|
1353
1834
|
f"{'kept as it was' if index is not None else 'not added'}")
|
|
1354
1835
|
return
|
|
1355
1836
|
if index is None:
|
|
1356
|
-
if add:
|
|
1357
|
-
pmap.add(_new_entry(record, pmap.
|
|
1837
|
+
if add and not off_surface(name, path):
|
|
1838
|
+
pmap.add(_new_entry(record, pmap.library_for(path, name, old=old)))
|
|
1358
1839
|
return
|
|
1359
1840
|
pmap.update(index, _without_none(record, keep=("ast_node_type",)))
|
|
1360
1841
|
|
|
@@ -1373,8 +1854,10 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
|
|
|
1373
1854
|
for name, key_path in fresh.keys_of(path):
|
|
1374
1855
|
record = fresh.record(name, key_path)
|
|
1375
1856
|
carried = old_entries.get(name)
|
|
1857
|
+
if carried is None and off_surface(name, key_path):
|
|
1858
|
+
continue
|
|
1376
1859
|
entry = _rewrite(carried, _without_none(record, keep=("ast_node_type",))) if carried \
|
|
1377
|
-
else _new_entry(record, pmap.
|
|
1860
|
+
else _new_entry(record, pmap.library_for(key_path, name, old=old))
|
|
1378
1861
|
pmap.add(entry)
|
|
1379
1862
|
continue
|
|
1380
1863
|
if status == "ADDED":
|
|
@@ -1386,7 +1869,8 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
|
|
|
1386
1869
|
if not isinstance(change, dict) or not _name_of(change):
|
|
1387
1870
|
continue
|
|
1388
1871
|
name, kind = _name_of(change), change.get("change_type")
|
|
1389
|
-
|
|
1872
|
+
# a move across files (a MOVED_EXPORT, or a MODIFIED_EXPORT that also moved) names the file its entry is at
|
|
1873
|
+
index = pmap.find(name, _norm_path(change.get("old_file")) or path, _line_of(change.get("old_line")))
|
|
1390
1874
|
if kind == "DELETED_EXPORT":
|
|
1391
1875
|
if index is not None:
|
|
1392
1876
|
pmap.remove([index])
|
|
@@ -1395,9 +1879,24 @@ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fr
|
|
|
1395
1879
|
line = record["source_line"] if record and record.get("source_line") is not None \
|
|
1396
1880
|
else _line_of(change.get("new_line"))
|
|
1397
1881
|
if index is not None and line is not None:
|
|
1398
|
-
|
|
1882
|
+
fields = {"source_file": path, "source_line": line}
|
|
1883
|
+
if record is not None and record["extraction_method"] == "ast-grep":
|
|
1884
|
+
# the runner matched it: its labels, never its export_type, params or return_type
|
|
1885
|
+
fields.update(confidence=record["confidence"], extraction_method=record["extraction_method"])
|
|
1886
|
+
if record["ast_node_type"] is not None:
|
|
1887
|
+
fields["ast_node_type"] = record["ast_node_type"]
|
|
1888
|
+
entry = pmap.doc["entries"][index]
|
|
1889
|
+
# its signature_source only when the record holds each signature part the entry does
|
|
1890
|
+
if all(record.get(part) is not None for part in ("params", "return_type")
|
|
1891
|
+
if entry.get(part) is not None):
|
|
1892
|
+
fields["signature_source"] = record["signature_source"]
|
|
1893
|
+
elif _tier(entry.get("signature_source")) is None:
|
|
1894
|
+
# the signature it keeps was read by eye: set that, or the backfill would derive T1
|
|
1895
|
+
# from the ast-grep label set here
|
|
1896
|
+
fields["signature_source"] = "T1-low"
|
|
1897
|
+
pmap.update(index, fields)
|
|
1399
1898
|
elif kind in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1400
|
-
refresh(index, name, path, add=True)
|
|
1899
|
+
refresh(index, name, path, add=True, old=_norm_path(change.get("old_file")))
|
|
1401
1900
|
for rename in _get_list(categories or {}, "category_c", "renamed_exports"):
|
|
1402
1901
|
if not isinstance(rename, dict):
|
|
1403
1902
|
continue
|
|
@@ -1434,7 +1933,7 @@ def _apply_docs_only(pmap: _Map, records: dict) -> tuple[int, int]:
|
|
|
1434
1933
|
"confidence": "T3", "signature_source": "T3"}
|
|
1435
1934
|
carried = old.get(_name_of(record))
|
|
1436
1935
|
entry = _rewrite(carried, _without_none(fields)) if carried else _new_entry(
|
|
1437
|
-
_without_none(fields), pmap.
|
|
1936
|
+
_without_none(fields), pmap.library_for(url, _name_of(record)))
|
|
1438
1937
|
pmap.add(entry)
|
|
1439
1938
|
return len(changed), len(exports)
|
|
1440
1939
|
|
|
@@ -1444,8 +1943,9 @@ def _drift_note(drift: tuple | None) -> str:
|
|
|
1444
1943
|
|
|
1445
1944
|
|
|
1446
1945
|
def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, drift: tuple | None,
|
|
1447
|
-
warnings: list[str]) -> tuple[int, int] | list[dict]:
|
|
1448
|
-
"""Gap-driven mode: one write per verification record. A list return is the refusal
|
|
1946
|
+
warnings: list[str], relabeled: list[dict]) -> tuple[int, int] | list[dict]:
|
|
1947
|
+
"""Gap-driven mode: one write per verification record. A list return is the refusal; each entry whose
|
|
1948
|
+
signature_source the repair changes goes to `relabeled`."""
|
|
1449
1949
|
verification = [r for r in records.get("verification") or [] if isinstance(r, dict) and _name_of(r)]
|
|
1450
1950
|
reextracted = _Fresh(records.get("files") or [], None, None)
|
|
1451
1951
|
merged = {}
|
|
@@ -1456,18 +1956,108 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
|
|
|
1456
1956
|
def in_map(r: dict) -> bool:
|
|
1457
1957
|
return bool(r.get("in_map")) or bool(r.get("map_entry"))
|
|
1458
1958
|
|
|
1959
|
+
def signature(r: dict) -> tuple[dict, tuple]:
|
|
1960
|
+
"""(the signature parts to set, the parts to remove) of an export: with the record §4a's extraction gave at
|
|
1961
|
+
its line (its verification record has an extraction_method), every part from it, a null params or
|
|
1962
|
+
return_type removed; with none, merge's."""
|
|
1963
|
+
if r.get("extraction_method") is not None:
|
|
1964
|
+
return _without_none({part: r.get(part) for part in SIGNATURE_PARTS}), \
|
|
1965
|
+
tuple(part for part in ("params", "return_type") if r.get(part) is None)
|
|
1966
|
+
extra = merged.get(_name_of(r), {})
|
|
1967
|
+
return _without_none({part: extra.get(part) for part in SIGNATURE_PARTS}), ()
|
|
1968
|
+
|
|
1969
|
+
def resign(r: dict, index: int, fields: dict) -> None:
|
|
1970
|
+
"""Write an entry the map holds with `fields` and, when §4a found the export's record, its signature; an
|
|
1971
|
+
ast-grep entry takes the record's signature_source, and its node kind when the record's export_type is
|
|
1972
|
+
another (a source-read entry keeps T1-low and no kind); confidence and extraction_method stay."""
|
|
1973
|
+
entry = pmap.doc["entries"][index]
|
|
1974
|
+
drop: tuple = ()
|
|
1975
|
+
method = r.get("extraction_method")
|
|
1976
|
+
if r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT") and method is not None \
|
|
1977
|
+
and r.get("reachability") != "internal-unreachable":
|
|
1978
|
+
parts, gone = signature(r)
|
|
1979
|
+
fields = {**fields, **parts}
|
|
1980
|
+
drop = tuple(part for part in gone if entry.get(part) is not None)
|
|
1981
|
+
if entry.get("extraction_method") in ("ast-grep", "ast_bridge"):
|
|
1982
|
+
fields["signature_source"] = _labels(method, None)["signature_source"]
|
|
1983
|
+
if parts.get("export_type") not in (None, entry.get("export_type")):
|
|
1984
|
+
fields["ast_node_type"] = r.get("ast_node_type")
|
|
1985
|
+
old, new = entry.get("signature_source"), fields["signature_source"]
|
|
1986
|
+
if old is not None and old != new:
|
|
1987
|
+
path = _file_of(entry)
|
|
1988
|
+
relabeled.append({"name": _name_of(entry), "file": path, "from": old, "to": new})
|
|
1989
|
+
warnings.append(f"provenance: {_name_of(entry)} in {path}: signature_source {old} relabeled "
|
|
1990
|
+
f"{new}, from the {method} record of its signature")
|
|
1991
|
+
pmap.update(index, fields, drop)
|
|
1992
|
+
|
|
1993
|
+
note = _drift_note(drift)
|
|
1994
|
+
|
|
1995
|
+
def why(r: dict) -> str:
|
|
1996
|
+
"""The cause of an `unknown` outcome its record gives, `<unknown_reason>: <words>`, else ''."""
|
|
1997
|
+
reason = r.get("unknown_reason")
|
|
1998
|
+
return f"{reason}: {UNKNOWN_REASONS[reason]}" if r.get("verification") == "unknown" \
|
|
1999
|
+
and reason in UNKNOWN_REASONS else ""
|
|
2000
|
+
|
|
2001
|
+
def unchecked(r: dict) -> str | None:
|
|
2002
|
+
"""The warning for a blocking NEW_EXPORT or MODIFIED_EXPORT the map holds that its spot-check could not
|
|
2003
|
+
locate (missing, or unknown for a cause its unknown_reason names), which stays unchanged: the gap, the
|
|
2004
|
+
export, the outcome, its cause and the one fix a re-run acts on; None for any other record."""
|
|
2005
|
+
outcome = r.get("verification")
|
|
2006
|
+
if outcome not in ("missing", "unknown") or not in_map(r) or not _blocking(r.get("severity")) \
|
|
2007
|
+
or r.get("gap_category") not in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
2008
|
+
return None
|
|
2009
|
+
name, gap = _name_of(r), r.get("gap_id")
|
|
2010
|
+
path = _location(r.get("map_entry"))[0]
|
|
2011
|
+
cause = f"its map entry's file{f' {path}' if path else ''} is gone" if outcome == "missing" else why(r)
|
|
2012
|
+
detail = "; ".join(part for part in (cause, note) if part)
|
|
2013
|
+
if r.get("unknown_reason") == "ambiguous": # a Source at the definition lets the re-run's lookup pick one
|
|
2014
|
+
fix = ("its entries are left unchanged. Fix its entries by hand, or give the gap a `file:line` Source at "
|
|
2015
|
+
"its definition when that file holds only one of them, then re-run update-skill")
|
|
2016
|
+
else: # the re-run checks the same map entry again: only a hand fix changes it
|
|
2017
|
+
fix = ("its entry is left unchanged. Fix the entry by hand (set its source_file and source_line to the "
|
|
2018
|
+
"line that defines it), then re-run update-skill")
|
|
2019
|
+
return f"provenance: {name}{f' ({gap})' if gap else ''}: {outcome}{f' ({detail})' if detail else ''}; {fix}"
|
|
2020
|
+
|
|
2021
|
+
def elsewhere(r: dict) -> str | None:
|
|
2022
|
+
"""The warning for a NEW_EXPORT or MODIFIED_EXPORT whose gap cites another file than the map's entries of its
|
|
2023
|
+
name (unknown, cited-elsewhere), whatever its severity: the gap, the export, the cited file and its check's
|
|
2024
|
+
cause, the record's map_locations and the fix for each side that may be wrong; None for any other record."""
|
|
2025
|
+
if r.get("verification") != "unknown" or r.get("unknown_reason") != "cited-elsewhere":
|
|
2026
|
+
return None
|
|
2027
|
+
name, gap = _name_of(r), r.get("gap_id")
|
|
2028
|
+
cited_file, cited_line = _location(r.get("source_citation"))
|
|
2029
|
+
cited = _where(cited_file, cited_line) or cited_file
|
|
2030
|
+
check = r.get("citation_check") if isinstance(r.get("citation_check"), str) else None
|
|
2031
|
+
words = "no such file" if check == "missing" else UNKNOWN_REASONS.get(check or "")
|
|
2032
|
+
where = [w for w in r.get("map_locations") or [] if isinstance(w, str) and w]
|
|
2033
|
+
detail = "; ".join(part for part in (
|
|
2034
|
+
why(r), f"it cites {cited}" if cited else "",
|
|
2035
|
+
f"its check there: {check}{f' ({words})' if words else ''}" if check else "",
|
|
2036
|
+
f"the map holds it at {', '.join(where)}" if where else "", note) if part)
|
|
2037
|
+
several = len(where) > 1
|
|
2038
|
+
kept = "its entries are left unchanged" if several else "its entry is left unchanged"
|
|
2039
|
+
head = f"provenance: {name}{f' ({gap})' if gap else ''}: unknown ({detail}); {kept}, and nothing is " \
|
|
2040
|
+
"re-extracted for it."
|
|
2041
|
+
if check in UNREAD: # nothing was read at the cited file: make the check read it first
|
|
2042
|
+
return f"{head} Nothing was read at the cited file: {UNREAD[check]}, then re-run update-skill"
|
|
2043
|
+
moved = "when an export moved, fix by hand whichever of those entries moved" if several \
|
|
2044
|
+
else "when the export moved, fix the map entry by hand"
|
|
2045
|
+
scope = "" if check == "missing" else (", and if the gap comes back, exclude the cited file in the brief's "
|
|
2046
|
+
"scope when the skill does not cover it")
|
|
2047
|
+
return (f"{head} When the map is right, re-run Test Skill, since the test compared another file's "
|
|
2048
|
+
f"declaration{scope}; {moved} (set its source_file and source_line to the line that defines it), "
|
|
2049
|
+
f"then re-run update-skill")
|
|
2050
|
+
|
|
1459
2051
|
blocking = [{"export_name": _name_of(r), "severity": r.get("severity")} for r in verification
|
|
1460
2052
|
if r.get("verification") == "unknown" and not in_map(r)
|
|
1461
|
-
and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT")
|
|
1462
|
-
and str(r.get("severity") or "").strip().lower() not in NON_BLOCKING]
|
|
2053
|
+
and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT") and _blocking(r.get("severity"))]
|
|
1463
2054
|
if blocking:
|
|
1464
2055
|
return blocking
|
|
1465
|
-
note = _drift_note(drift)
|
|
1466
2056
|
pinned_keys: set[tuple] = set()
|
|
1467
2057
|
for r in verification:
|
|
1468
2058
|
name, outcome, category = _name_of(r), r.get("verification"), r.get("gap_category")
|
|
1469
2059
|
entry_file, entry_line = _location(r.get("map_entry"))
|
|
1470
|
-
index = pmap.find(name, entry_file, entry_line) if r.get("map_entry") else None
|
|
2060
|
+
index = pmap.find(name, entry_file, entry_line, reexport=True) if r.get("map_entry") else None
|
|
1471
2061
|
lines = r.get("pinned_definition_lines")
|
|
1472
2062
|
r5 = f"; definition lines per the test report: {lines}" if category == "MOVED_EXPORT" else ""
|
|
1473
2063
|
if outcome == "rescoped":
|
|
@@ -1476,10 +2066,10 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
|
|
|
1476
2066
|
elif outcome == "verified" and index is not None:
|
|
1477
2067
|
if drift and category == "MOVED_EXPORT":
|
|
1478
2068
|
warnings.append(f"provenance: {name}: verified at HEAD only ({note}{r5})")
|
|
2069
|
+
resign(r, index, {})
|
|
1479
2070
|
elif outcome == "moved" and index is not None:
|
|
1480
2071
|
path, line = _location(r.get("new_location"))
|
|
1481
|
-
if path and line is not None
|
|
1482
|
-
pmap.update(index, {"source_file": path, "source_line": line})
|
|
2072
|
+
resign(r, index, {"source_file": path, "source_line": line} if path and line is not None else {})
|
|
1483
2073
|
elif outcome == "re-extracted":
|
|
1484
2074
|
if r.get("reachability") == "internal-unreachable":
|
|
1485
2075
|
continue # not public API: merge queued it as a metadata update
|
|
@@ -1488,9 +2078,13 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
|
|
|
1488
2078
|
if record is None:
|
|
1489
2079
|
warnings.append(f"provenance: {name}: re-extracted, but no record of it in the run's files")
|
|
1490
2080
|
continue
|
|
1491
|
-
|
|
2081
|
+
local = (reextracted.worker.get((name, path)) or {}).get("local")
|
|
2082
|
+
if isinstance(local, str) and local and local != name:
|
|
2083
|
+
# matched by the public name a re-export gives it: the entry keeps its declared name
|
|
2084
|
+
record = {**record, "export_name": local, "reexported_as": name}
|
|
2085
|
+
existing = pmap.find(name, path, reexport=True)
|
|
1492
2086
|
if existing is None:
|
|
1493
|
-
pmap.add(_new_entry(record, pmap.
|
|
2087
|
+
pmap.add(_new_entry(record, pmap.library_for(path, name)))
|
|
1494
2088
|
else:
|
|
1495
2089
|
pmap.update(existing, _without_none(record, keep=("ast_node_type",)))
|
|
1496
2090
|
elif outcome in ("verified", "moved") and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
@@ -1498,29 +2092,28 @@ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, dri
|
|
|
1498
2092
|
continue
|
|
1499
2093
|
where = r.get("source_citation") if outcome == "verified" else r.get("new_location")
|
|
1500
2094
|
path, line = _location(where)
|
|
1501
|
-
if not path or line is None or (name, path) in pinned_keys
|
|
2095
|
+
if not path or line is None or (name, path) in pinned_keys \
|
|
2096
|
+
or pmap.find(name, path, reexport=True) is not None:
|
|
1502
2097
|
continue
|
|
1503
2098
|
pinned_keys.add((name, path))
|
|
1504
|
-
|
|
2099
|
+
# the signature its record gives, else merge's; the source-read labels whatever the record's tool
|
|
1505
2100
|
pmap.add(_new_entry(_without_none({
|
|
1506
|
-
"export_name": name,
|
|
1507
|
-
"export_type": extra.get("export_type") or r.get("export_type"),
|
|
1508
|
-
"params": extra.get("params"), "return_type": extra.get("return_type"),
|
|
2101
|
+
"export_name": name, **signature(r)[0],
|
|
1509
2102
|
"source_file": path, "source_line": line, **_labels("source-read", None)},
|
|
1510
|
-
keep=("ast_node_type",)), pmap.
|
|
2103
|
+
keep=("ast_node_type",)), pmap.library_for(path, name)))
|
|
1511
2104
|
elif outcome == "unknown" and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1512
|
-
extra = merged.get(name, {})
|
|
1513
2105
|
pmap.add(_new_entry(_without_none({
|
|
1514
|
-
"export_name": name,
|
|
1515
|
-
"
|
|
1516
|
-
**_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library))
|
|
2106
|
+
"export_name": name, **signature(r)[0],
|
|
2107
|
+
**_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library_for(None, name)))
|
|
1517
2108
|
else:
|
|
1518
2109
|
if r.get("unknown_reason") == "drift-override":
|
|
1519
2110
|
warnings.append(f"provenance: {name}: unknown ({note}; no line taken from HEAD{r5})")
|
|
2111
|
+
elif (warning := elsewhere(r) or unchecked(r)) is not None:
|
|
2112
|
+
warnings.append(warning)
|
|
1520
2113
|
elif drift:
|
|
1521
|
-
warnings.append(f"provenance: {name}: {outcome} ({note}{r5})")
|
|
2114
|
+
warnings.append(f"provenance: {name}: {outcome} ({why(r) + '; ' if why(r) else ''}{note}{r5})")
|
|
1522
2115
|
else:
|
|
1523
|
-
warnings.append(f"provenance: {name}: {outcome}{r5}")
|
|
2116
|
+
warnings.append(f"provenance: {name}: {outcome}{f' ({why(r)})' if why(r) else ''}{r5}")
|
|
1524
2117
|
return len(pmap.touched_files), len(verification)
|
|
1525
2118
|
|
|
1526
2119
|
|
|
@@ -1583,14 +2176,165 @@ def _apply_files(pmap: _Map, compare: dict | None, new_files: dict | None, promo
|
|
|
1583
2176
|
return done
|
|
1584
2177
|
|
|
1585
2178
|
|
|
2179
|
+
REFERENCE_APP = "reference-app"
|
|
2180
|
+
|
|
2181
|
+
|
|
2182
|
+
def _export_name(item: object) -> str | None:
|
|
2183
|
+
"""The name of a metadata.json exports[] item: a string, or an object's name or export_name."""
|
|
2184
|
+
if isinstance(item, str):
|
|
2185
|
+
return item or None
|
|
2186
|
+
return _name_of(item) if isinstance(item, dict) else None
|
|
2187
|
+
|
|
2188
|
+
|
|
2189
|
+
def exports_after(metadata: object, pmap: _Map, update_type: str) -> list:
|
|
2190
|
+
"""metadata.json's exports[] after the update (see "Exports" in the module docstring)."""
|
|
2191
|
+
if not isinstance(metadata, dict):
|
|
2192
|
+
raise ValueError("--metadata-in file must hold a JSON object (the skill's metadata.json)")
|
|
2193
|
+
if metadata.get("scope_type") == REFERENCE_APP:
|
|
2194
|
+
return []
|
|
2195
|
+
current = metadata.get("exports") if isinstance(metadata.get("exports"), list) else []
|
|
2196
|
+
held = {_name_of(e) for e in pmap.doc["entries"] if isinstance(e, dict)} - {None}
|
|
2197
|
+
# an entry added under its declared name, re-exported under a public one: exports[] lists the public name
|
|
2198
|
+
public = {_name_of(e): e["reexported_as"] for e in pmap.doc["entries"]
|
|
2199
|
+
if isinstance(e, dict) and isinstance(e.get("reexported_as"), str) and e["reexported_as"]}
|
|
2200
|
+
renamed = dict(pmap.renamed)
|
|
2201
|
+
gone = ({n for n in pmap.removed if n} | set(renamed)) - held
|
|
2202
|
+
if update_type == "full":
|
|
2203
|
+
gone |= {n for n in map(_export_name, current) if n and n not in held}
|
|
2204
|
+
out: list = []
|
|
2205
|
+
seen: set[str] = set()
|
|
2206
|
+
|
|
2207
|
+
def put(item: object, name: str | None) -> None:
|
|
2208
|
+
if name and name not in seen:
|
|
2209
|
+
seen.add(name)
|
|
2210
|
+
out.append(item)
|
|
2211
|
+
|
|
2212
|
+
for item in current:
|
|
2213
|
+
name = _export_name(item)
|
|
2214
|
+
if name in gone:
|
|
2215
|
+
new = renamed.get(name)
|
|
2216
|
+
put(new, new if new in held else None)
|
|
2217
|
+
else:
|
|
2218
|
+
put(item, name)
|
|
2219
|
+
for name in [public.get(n, n) for n in pmap.added] + [new for _old, new in pmap.renamed]:
|
|
2220
|
+
put(name, name if name in held or name in public.values() else None)
|
|
2221
|
+
return out
|
|
2222
|
+
|
|
2223
|
+
|
|
2224
|
+
_DETECTOR = None
|
|
2225
|
+
|
|
2226
|
+
|
|
2227
|
+
def _detector():
|
|
2228
|
+
"""skf-detect-scripts-assets.py, loaded once from this folder: the purpose a new script's or asset's row
|
|
2229
|
+
takes, read from the file as create-skill's detector reads it."""
|
|
2230
|
+
global _DETECTOR
|
|
2231
|
+
if _DETECTOR is None:
|
|
2232
|
+
path = Path(__file__).resolve().parent / "skf-detect-scripts-assets.py"
|
|
2233
|
+
spec = importlib.util.spec_from_file_location("skf_detect_scripts_assets", path)
|
|
2234
|
+
if spec is None or spec.loader is None or not path.is_file():
|
|
2235
|
+
raise ValueError(f"cannot load {path.name} beside {Path(__file__).name}: re-install SKF")
|
|
2236
|
+
module = importlib.util.module_from_spec(spec)
|
|
2237
|
+
spec.loader.exec_module(module)
|
|
2238
|
+
_DETECTOR = module
|
|
2239
|
+
return _DETECTOR
|
|
2240
|
+
|
|
2241
|
+
|
|
2242
|
+
def _purpose(source_file: str, kind: str, source_root: Path | None) -> str:
|
|
2243
|
+
"""A new script's or asset's purpose: the detector's, read from the file under `source_root`, else its name."""
|
|
2244
|
+
path = source_root / source_file if source_root is not None else None
|
|
2245
|
+
if path is None or not path.is_file():
|
|
2246
|
+
return Path(source_file).name
|
|
2247
|
+
detector = _detector()
|
|
2248
|
+
hint = detector.asset_type(path, source_root) if kind == "asset" else None
|
|
2249
|
+
return detector.extract_purpose(path, asset_type_hint=hint)
|
|
2250
|
+
|
|
2251
|
+
|
|
2252
|
+
def file_rows_after(metadata: dict, pmap: _Map, source_root: Path | None) -> dict[str, list]:
|
|
2253
|
+
"""metadata.json's scripts[] and assets[] after the update (see "Lists" in the module docstring)."""
|
|
2254
|
+
by_source: dict[str, dict] = {}
|
|
2255
|
+
by_file: dict[str, dict] = {}
|
|
2256
|
+
for key in ("scripts", "assets"):
|
|
2257
|
+
rows = metadata.get(key)
|
|
2258
|
+
for row in rows if isinstance(rows, list) else []:
|
|
2259
|
+
if not isinstance(row, dict):
|
|
2260
|
+
continue
|
|
2261
|
+
source = _norm_path(row.get("source_file"))
|
|
2262
|
+
if source:
|
|
2263
|
+
by_source.setdefault(source, row)
|
|
2264
|
+
if isinstance(row.get("file"), str):
|
|
2265
|
+
by_file.setdefault(row["file"], row)
|
|
2266
|
+
lists: dict[str, list] = {"scripts": [], "assets": []}
|
|
2267
|
+
entries = pmap.doc.get("file_entries")
|
|
2268
|
+
for entry in entries if isinstance(entries, list) else []:
|
|
2269
|
+
if not isinstance(entry, dict) or _file_type(entry) not in ("script", "asset"):
|
|
2270
|
+
continue
|
|
2271
|
+
kind = _file_type(entry)
|
|
2272
|
+
source = _norm_path(entry.get("source_file"))
|
|
2273
|
+
held = by_source.get(source) if source else None
|
|
2274
|
+
if held is None and isinstance(entry.get("file_name"), str):
|
|
2275
|
+
held = by_file.get(entry["file_name"])
|
|
2276
|
+
if held is None:
|
|
2277
|
+
held = {"file": entry.get("file_name") or f"{kind}s/{Path(source or '').name}",
|
|
2278
|
+
"purpose": _purpose(source, kind, source_root) if source else "",
|
|
2279
|
+
"source_file": source, "confidence": entry.get("confidence") or "T1-low"}
|
|
2280
|
+
lists[f"{kind}s"].append(held)
|
|
2281
|
+
return lists
|
|
2282
|
+
|
|
2283
|
+
|
|
2284
|
+
# the signature_source tiers skf-render-metadata-stats.py bins, in their spelling (it compares them case-insensitively)
|
|
2285
|
+
SIGNATURE_SOURCES = ("T1", "T1-low", "T2", "T3")
|
|
2286
|
+
WARNING_LIST_CAP = 10 # the items a warning lists before "and N more"
|
|
2287
|
+
|
|
2288
|
+
|
|
2289
|
+
def _capped(items: list[str]) -> str:
|
|
2290
|
+
"""The items joined with ", ", the first WARNING_LIST_CAP of them and "and N more" for the rest."""
|
|
2291
|
+
more = len(items) - WARNING_LIST_CAP
|
|
2292
|
+
return ", ".join(items[:WARNING_LIST_CAP] + ([f"and {more} more"] if more > 0 else []))
|
|
2293
|
+
|
|
2294
|
+
|
|
2295
|
+
def _tier(value: object) -> str | None:
|
|
2296
|
+
"""A signature_source or confidence tier in its SIGNATURE_SOURCES spelling; None for any other value."""
|
|
2297
|
+
if not isinstance(value, str):
|
|
2298
|
+
return None
|
|
2299
|
+
return next((tier for tier in SIGNATURE_SOURCES if tier.upper() == value.strip().upper()), None)
|
|
2300
|
+
|
|
2301
|
+
|
|
2302
|
+
def _backfill_signature_source(pmap: _Map) -> tuple[dict[str, int], list[tuple[object, object, str]]]:
|
|
2303
|
+
"""Give each entry whose signature_source is missing or not a tier the stats helper bins (a map written before
|
|
2304
|
+
the key existed, #802) the one its labels give: its confidence when that names T1-low, T2 or T3, else T1 for
|
|
2305
|
+
an extraction_method of ast-grep or ast_bridge (compared stripped and case-insensitively) and T1-low for any
|
|
2306
|
+
other (relabel-rule.md: T1 needs evidence an ast-grep rule matched). Returns (the fills counted by value, in
|
|
2307
|
+
tier order; (name, old value, new value) of each unrecognized value replaced, which the map no longer holds);
|
|
2308
|
+
the names are not listed as updated."""
|
|
2309
|
+
counts: dict[str, int] = {}
|
|
2310
|
+
replaced: list[tuple[object, object, str]] = []
|
|
2311
|
+
for entry in pmap.doc["entries"]:
|
|
2312
|
+
if not isinstance(entry, dict) or _tier(entry.get("signature_source")) is not None:
|
|
2313
|
+
continue
|
|
2314
|
+
confidence, method = _tier(entry.get("confidence")), entry.get("extraction_method")
|
|
2315
|
+
if confidence in ("T1-low", "T2", "T3"):
|
|
2316
|
+
value = confidence
|
|
2317
|
+
elif isinstance(method, str) and method.strip().lower() in ("ast-grep", "ast_bridge"):
|
|
2318
|
+
value = "T1"
|
|
2319
|
+
else:
|
|
2320
|
+
value = "T1-low"
|
|
2321
|
+
if entry.get("signature_source") is not None:
|
|
2322
|
+
replaced.append((entry.get("export_name"), entry["signature_source"], value))
|
|
2323
|
+
entry["signature_source"] = value
|
|
2324
|
+
counts[value] = counts.get(value, 0) + 1
|
|
2325
|
+
return {tier: counts[tier] for tier in SIGNATURE_SOURCES if tier in counts}, replaced
|
|
2326
|
+
|
|
2327
|
+
|
|
1586
2328
|
def apply_update(*, update_type: str, provenance: dict | None, skill_name: str, manifest: dict | None = None,
|
|
1587
2329
|
categories: dict | None = None, extraction: dict | None = None, details: dict | None = None,
|
|
1588
2330
|
records: dict | None = None, merge_records: dict | None = None, compare: dict | None = None,
|
|
1589
2331
|
new_files: dict | None = None, promoted: list | None = None, source_root: Path | None = None,
|
|
1590
2332
|
generation_date: str, test_report_run_id: str | None, confidence_tier: str,
|
|
1591
2333
|
manual_sections_preserved: int, source_commit: str | None = None,
|
|
1592
|
-
source_ref: str | None = None, drift: tuple | None = None
|
|
1593
|
-
|
|
2334
|
+
source_ref: str | None = None, drift: tuple | None = None,
|
|
2335
|
+
metadata: dict | None = None) -> tuple[dict, dict]:
|
|
2336
|
+
"""(new map, summary) of one update (see "Apply" in the module docstring). With `metadata` (the skill's
|
|
2337
|
+
metadata.json), the summary's `lists` holds the exports[], scripts[] and assets[] it takes after the update.
|
|
1594
2338
|
|
|
1595
2339
|
A refusal returns ({}, summary) with status "refused" and no map.
|
|
1596
2340
|
Raises ValueError for inputs it cannot read.
|
|
@@ -1602,9 +2346,11 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
|
|
|
1602
2346
|
records = records if isinstance(records, dict) else {}
|
|
1603
2347
|
pmap = _Map(provenance, skill_name)
|
|
1604
2348
|
warnings: list[str] = []
|
|
2349
|
+
not_public: list[dict] = []
|
|
2350
|
+
relabeled: list[dict] = []
|
|
1605
2351
|
mode = records.get("mode")
|
|
1606
2352
|
if update_type == "gap-driven":
|
|
1607
|
-
outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings)
|
|
2353
|
+
outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings, relabeled)
|
|
1608
2354
|
if isinstance(outcome, list):
|
|
1609
2355
|
return {}, {"status": "refused", "blocking_unresolved": outcome}
|
|
1610
2356
|
files, exports = outcome
|
|
@@ -1617,13 +2363,35 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
|
|
|
1617
2363
|
index = pmap.find(name, path)
|
|
1618
2364
|
record = fresh.record(name, path)
|
|
1619
2365
|
if index is None:
|
|
1620
|
-
pmap.add(_new_entry(record, pmap.
|
|
2366
|
+
pmap.add(_new_entry(record, pmap.library_for(path, name)))
|
|
1621
2367
|
else:
|
|
1622
2368
|
pmap.update(index, _without_none(record, keep=("ast_node_type",)))
|
|
1623
2369
|
files = len({path for _name, path in fresh.all_keys()})
|
|
1624
2370
|
exports = len(fresh.all_keys())
|
|
1625
2371
|
else:
|
|
1626
|
-
files, exports = _apply_normal(pmap, manifest, categories, fresh, warnings)
|
|
2372
|
+
files, exports = _apply_normal(pmap, manifest, categories, fresh, warnings, not_public)
|
|
2373
|
+
if fresh.surface_problem:
|
|
2374
|
+
warnings.append(f"provenance: the public surface of this public-api skill could not be applied "
|
|
2375
|
+
f"({fresh.surface_problem}): every new export was added")
|
|
2376
|
+
if not_public:
|
|
2377
|
+
warnings.append(_not_public_warning(not_public))
|
|
2378
|
+
if pmap.library_unresolved:
|
|
2379
|
+
count = len(pmap.library_unresolved)
|
|
2380
|
+
named = _capped([f"{name} in {path}" if path else f"{name} (no file)"
|
|
2381
|
+
for name, path in pmap.library_unresolved])
|
|
2382
|
+
warnings.append(f"provenance: {count} new entr{'y' if count == 1 else 'ies'} took source_library "
|
|
2383
|
+
f"{pmap.library}, the map's first, as the map names several libraries "
|
|
2384
|
+
f"({', '.join(pmap.libraries)}) and each entry's file or folder does not settle which: "
|
|
2385
|
+
f"{named}; check each entry's source_library")
|
|
2386
|
+
backfilled, replaced = _backfill_signature_source(pmap)
|
|
2387
|
+
if backfilled:
|
|
2388
|
+
count = sum(backfilled.values())
|
|
2389
|
+
listed = [f"{name}: {old if isinstance(old, str) else json.dumps(old)} -> {new}" for name, old, new in replaced]
|
|
2390
|
+
values = f"; unrecognized values replaced: {_capped(listed)}" if replaced else ""
|
|
2391
|
+
warnings.append(f"provenance: backfilled signature_source on {count} entr{'y' if count == 1 else 'ies'} "
|
|
2392
|
+
f"that had none or an unrecognized value (a map written before the key existed), from each "
|
|
2393
|
+
f"one's confidence and extraction_method: "
|
|
2394
|
+
f"{', '.join(f'{n} {value}' for value, n in backfilled.items())}{values}")
|
|
1627
2395
|
file_changes = _apply_files(pmap, compare, new_files, promoted, source_root)
|
|
1628
2396
|
doc = pmap.doc
|
|
1629
2397
|
for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
|
|
@@ -1643,12 +2411,168 @@ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str,
|
|
|
1643
2411
|
summary = {
|
|
1644
2412
|
"status": "written",
|
|
1645
2413
|
"entries": {"added": pmap.added, "updated": pmap.updated, "removed": pmap.removed},
|
|
2414
|
+
"not_public": not_public,
|
|
2415
|
+
"relabeled": relabeled,
|
|
2416
|
+
"signature_source_backfilled": backfilled,
|
|
1646
2417
|
"file_entries": file_changes,
|
|
1647
2418
|
"warnings": warnings,
|
|
1648
2419
|
}
|
|
2420
|
+
if metadata is not None:
|
|
2421
|
+
summary["lists"] = {"exports": exports_after(metadata, pmap, update_type),
|
|
2422
|
+
**file_rows_after(metadata, pmap, source_root)}
|
|
1649
2423
|
return doc, summary
|
|
1650
2424
|
|
|
1651
2425
|
|
|
2426
|
+
GENERATION_DATE_RE = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ")
|
|
2427
|
+
# the stats fields skf-render-metadata-stats.py writes, some only for a shape or a judgment
|
|
2428
|
+
STATS_HELPER_FIELDS = ("exports_documented", "exports_public_api", "exports_internal", "exports_total",
|
|
2429
|
+
"public_api_coverage", "total_coverage", "scripts_count", "assets_count",
|
|
2430
|
+
"pattern_surfaces_documented", "effective_denominator")
|
|
2431
|
+
# the other stats fields create-skill writes, by the skill's scope_type: a component library's own counts (the
|
|
2432
|
+
# Component Library stats of compile-assembly-rules.md, its stats_extra) and a reference app's notes
|
|
2433
|
+
STATS_CREATE_FIELDS = {
|
|
2434
|
+
"component-library": ("components_registered", "components_documented", "props_interfaces_extracted",
|
|
2435
|
+
"components_unique", "demo_files_excluded", "design_variants"),
|
|
2436
|
+
"reference-app": ("notes",),
|
|
2437
|
+
}
|
|
2438
|
+
# a stats key that explains effective_denominator, dropped when this run's denominator differs from the old one
|
|
2439
|
+
DENOMINATOR_NOTE_PREFIX = "effective_denominator_"
|
|
2440
|
+
SKF_NO_ROOT = ("metadata: tool_versions.skf is unknown: no --skf-root was given; pass the installed SKF's folder "
|
|
2441
|
+
"({project-root}/_bmad/skf)")
|
|
2442
|
+
AST_GREP_UNKNOWN = ("metadata: tool_versions.ast_grep is unknown: --extraction holds exports ast-grep matched but no "
|
|
2443
|
+
"ast_grep.version")
|
|
2444
|
+
_QMD_PREFIX = re.compile(r"^qmd\s+", re.IGNORECASE) # what `qmd --version` prints first
|
|
2445
|
+
|
|
2446
|
+
|
|
2447
|
+
def _tool_version(value: object) -> str | None:
|
|
2448
|
+
return value.strip() if isinstance(value, str) and value.strip() else None
|
|
2449
|
+
|
|
2450
|
+
|
|
2451
|
+
def _skf_unknown(skf_root: str | Path) -> str:
|
|
2452
|
+
return (f"metadata: tool_versions.skf is unknown: {Path(skf_root).as_posix()} holds no package.json or VERSION "
|
|
2453
|
+
f"file with a version")
|
|
2454
|
+
|
|
2455
|
+
|
|
2456
|
+
def _kept_warning(key: str) -> str:
|
|
2457
|
+
return f"metadata: stats.{key} kept from an earlier run; check it still describes this run's counts"
|
|
2458
|
+
|
|
2459
|
+
|
|
2460
|
+
def tool_versions_after(old: object, *, skf_version: str | None, skf_root: str | Path | None,
|
|
2461
|
+
ast_grep_version: str | None, qmd_version: str | None) -> tuple[dict, list[str]]:
|
|
2462
|
+
"""(the tool_versions an update leaves, its warnings) (see "Metadata" in the module docstring)."""
|
|
2463
|
+
warnings: list[str] = []
|
|
2464
|
+
if isinstance(old, dict):
|
|
2465
|
+
tools = dict(old)
|
|
2466
|
+
else:
|
|
2467
|
+
tools = {"ast_grep": None, "qmd": None}
|
|
2468
|
+
if old is not None:
|
|
2469
|
+
kind = {bool: "a boolean", str: "a string", list: "an array"}.get(type(old), "a number")
|
|
2470
|
+
warnings.append(f"metadata: tool_versions was {kind}, not an object; replaced with the versions this run "
|
|
2471
|
+
f"records")
|
|
2472
|
+
ast_grep = _tool_version(ast_grep_version)
|
|
2473
|
+
if ast_grep is not None:
|
|
2474
|
+
tools["ast_grep"] = ast_grep
|
|
2475
|
+
if ast_grep == "unknown":
|
|
2476
|
+
warnings.append(AST_GREP_UNKNOWN)
|
|
2477
|
+
qmd = _tool_version(qmd_version)
|
|
2478
|
+
if qmd is not None:
|
|
2479
|
+
qmd = _tool_version(_QMD_PREFIX.sub("", qmd, count=1))
|
|
2480
|
+
if qmd is not None:
|
|
2481
|
+
tools["qmd"] = qmd
|
|
2482
|
+
skf = _tool_version(skf_version)
|
|
2483
|
+
if skf is None or skf == "unknown":
|
|
2484
|
+
skf = "unknown"
|
|
2485
|
+
warnings.append(SKF_NO_ROOT if skf_root is None else _skf_unknown(skf_root))
|
|
2486
|
+
tools["skf"] = skf
|
|
2487
|
+
return tools, warnings
|
|
2488
|
+
|
|
2489
|
+
|
|
2490
|
+
def stats_after(old_stats: dict, new_stats: dict, scope_type: object) -> tuple[dict, list[str], list[str]]:
|
|
2491
|
+
"""(the stats an update leaves, dropped_stats, their warnings) (see "Metadata" in the module docstring)."""
|
|
2492
|
+
exempt = STATS_CREATE_FIELDS.get(scope_type, ()) if isinstance(scope_type, str) else ()
|
|
2493
|
+
denominator = new_stats.get("effective_denominator")
|
|
2494
|
+
recounted = old_stats.get("effective_denominator") != denominator
|
|
2495
|
+
kept: dict = {}
|
|
2496
|
+
dropped: list[str] = []
|
|
2497
|
+
warnings: list[str] = []
|
|
2498
|
+
for key, value in old_stats.items():
|
|
2499
|
+
if key in new_stats:
|
|
2500
|
+
continue
|
|
2501
|
+
if key in STATS_HELPER_FIELDS:
|
|
2502
|
+
dropped.append(key)
|
|
2503
|
+
warnings.append(f"metadata: stats.{key} dropped: this run's stats helper did not write it")
|
|
2504
|
+
continue
|
|
2505
|
+
if key.startswith(DENOMINATOR_NOTE_PREFIX) and recounted:
|
|
2506
|
+
dropped.append(key)
|
|
2507
|
+
warnings.append(f"metadata: stats.{key} dropped: it explained an effective_denominator this run changed "
|
|
2508
|
+
f"or removed")
|
|
2509
|
+
continue
|
|
2510
|
+
kept[key] = value
|
|
2511
|
+
# an explanation of a denominator this run left as it was still describes it
|
|
2512
|
+
if key not in exempt and not (key.startswith(DENOMINATOR_NOTE_PREFIX) and denominator is not None):
|
|
2513
|
+
warnings.append(_kept_warning(key))
|
|
2514
|
+
return {**new_stats, **kept}, dropped, warnings
|
|
2515
|
+
|
|
2516
|
+
|
|
2517
|
+
def update_metadata(metadata: object, lists: object, stats: object, *, version: str, generation_date: str,
|
|
2518
|
+
source_commit: str | None = None, source_ref: str | None = None,
|
|
2519
|
+
skf_version: str | None = None, skf_root: str | Path | None = None,
|
|
2520
|
+
ast_grep_version: str | None = None, qmd_version: str | None = None) -> tuple[dict, dict]:
|
|
2521
|
+
"""(the metadata.json an update leaves, {"dropped_stats": [...], "warnings": [...]}) (see "Metadata" in the
|
|
2522
|
+
module docstring). Raises ValueError."""
|
|
2523
|
+
if not isinstance(metadata, dict):
|
|
2524
|
+
raise ValueError("--metadata file must hold a JSON object (the skill's metadata.json)")
|
|
2525
|
+
if not isinstance(lists, dict) or not all(isinstance(lists.get(k), list) for k in ("exports", "scripts",
|
|
2526
|
+
"assets")):
|
|
2527
|
+
raise ValueError("--lists file must hold the exports, scripts and assets lists apply wrote (--lists-out)")
|
|
2528
|
+
if not isinstance(stats, dict) or not isinstance(stats.get("stats"), dict) \
|
|
2529
|
+
or not isinstance(stats.get("confidence_distribution"), dict):
|
|
2530
|
+
raise ValueError("--stats file must hold the stats helper's JSON: a stats and a confidence_distribution "
|
|
2531
|
+
"object")
|
|
2532
|
+
if not version.strip():
|
|
2533
|
+
raise ValueError("--version must not be empty")
|
|
2534
|
+
if not GENERATION_DATE_RE.fullmatch(generation_date):
|
|
2535
|
+
raise ValueError(f"--generation-date must be YYYY-MM-DDTHH:MM:SSZ, read from the clock; got "
|
|
2536
|
+
f"{generation_date!r}")
|
|
2537
|
+
for kind in ("scripts", "assets"):
|
|
2538
|
+
counted = stats["stats"].get(f"{kind}_count")
|
|
2539
|
+
if counted != len(lists[kind]):
|
|
2540
|
+
raise ValueError(f"the stats count {counted} {kind} but apply's lists hold {len(lists[kind])}: run the "
|
|
2541
|
+
f"stats helper again with the lists' {kind}")
|
|
2542
|
+
out = {k: v for k, v in metadata.items() if k not in ("last_update", "update_type")}
|
|
2543
|
+
out["version"] = version
|
|
2544
|
+
out["generation_date"] = generation_date
|
|
2545
|
+
for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
|
|
2546
|
+
if value is not None:
|
|
2547
|
+
out[key] = value or None
|
|
2548
|
+
out["exports"] = lists["exports"]
|
|
2549
|
+
for kind in ("scripts", "assets"):
|
|
2550
|
+
if lists[kind]:
|
|
2551
|
+
out[kind] = lists[kind]
|
|
2552
|
+
else:
|
|
2553
|
+
out.pop(kind, None)
|
|
2554
|
+
out["tool_versions"], warnings = tool_versions_after(
|
|
2555
|
+
metadata.get("tool_versions"), skf_version=skf_version, skf_root=skf_root,
|
|
2556
|
+
ast_grep_version=ast_grep_version, qmd_version=qmd_version)
|
|
2557
|
+
old_stats = metadata.get("stats") if isinstance(metadata.get("stats"), dict) else {}
|
|
2558
|
+
out["stats"], dropped, stats_warnings = stats_after(old_stats, stats["stats"], metadata.get("scope_type"))
|
|
2559
|
+
warnings += stats_warnings
|
|
2560
|
+
out["confidence_distribution"] = stats["confidence_distribution"]
|
|
2561
|
+
return out, {"dropped_stats": dropped, "warnings": warnings}
|
|
2562
|
+
|
|
2563
|
+
|
|
2564
|
+
NOT_PUBLIC_NAMED = 10 # the names the not_public warning spells out; the summary lists them all
|
|
2565
|
+
|
|
2566
|
+
|
|
2567
|
+
def _not_public_warning(not_public: list[dict]) -> str:
|
|
2568
|
+
"""The one warning for every name apply left out as off a public-api skill's public surface."""
|
|
2569
|
+
names = [item["name"] for item in not_public]
|
|
2570
|
+
shown = ", ".join(names[:NOT_PUBLIC_NAMED])
|
|
2571
|
+
more = f" and {len(names) - NOT_PUBLIC_NAMED} more" if len(names) > NOT_PUBLIC_NAMED else ""
|
|
2572
|
+
return (f"provenance: {len(names)} new export(s) off the public surface of this public-api skill not added "
|
|
2573
|
+
f"(the summary's not_public lists each with its file): {shown}{more}")
|
|
2574
|
+
|
|
2575
|
+
|
|
1652
2576
|
def _write_json_atomic(path: Path, value) -> None:
|
|
1653
2577
|
"""Write `value` as indented JSON through a temporary file beside `path` and one rename, retried on Windows,
|
|
1654
2578
|
where a scanner may hold the target open for a moment."""
|
|
@@ -1681,22 +2605,12 @@ def _write_json_atomic(path: Path, value) -> None:
|
|
|
1681
2605
|
# a patch sets only where the seed has none.
|
|
1682
2606
|
PATCH_FIELDS = ("signature", "members", "docstring", "qmd_evidence")
|
|
1683
2607
|
PATCH_IF_NULL = ("params", "return_type")
|
|
1684
|
-
_INVENTORY = None
|
|
1685
2608
|
|
|
1686
2609
|
|
|
1687
2610
|
def _typed_param(param, language: object) -> str | None:
|
|
1688
|
-
"""One parameter as the provenance map writes it: skf-extraction-inventory.py's typed_param,
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
if _INVENTORY is None:
|
|
1692
|
-
path = Path(__file__).resolve().parent / "skf-extraction-inventory.py"
|
|
1693
|
-
spec = importlib.util.spec_from_file_location("skf_extraction_inventory", path)
|
|
1694
|
-
if spec is None or spec.loader is None:
|
|
1695
|
-
raise ImportError(f"cannot load {path}")
|
|
1696
|
-
module = importlib.util.module_from_spec(spec)
|
|
1697
|
-
spec.loader.exec_module(module)
|
|
1698
|
-
_INVENTORY = module
|
|
1699
|
-
return _INVENTORY.typed_param(param, language if isinstance(language, str) else None)
|
|
2611
|
+
"""One parameter as the provenance map writes it: skf-extraction-inventory.py's typed_param, so create-skill
|
|
2612
|
+
and update-skill write one form."""
|
|
2613
|
+
return _inventory().typed_param(param, language if isinstance(language, str) else None)
|
|
1700
2614
|
|
|
1701
2615
|
|
|
1702
2616
|
def _typed_params(value, language: object) -> list | None:
|
|
@@ -1784,6 +2698,25 @@ def build_records(extraction: dict | None, details: dict | None, files: list | N
|
|
|
1784
2698
|
if patch.get(field) is not None:
|
|
1785
2699
|
seeded[key][field] = patch[field]
|
|
1786
2700
|
fill_if_null(key, patch)
|
|
2701
|
+
diff = extraction.get("entry_point_diff") if isinstance(extraction, dict) else None
|
|
2702
|
+
aliases: dict[tuple, set] = {}
|
|
2703
|
+
rows = [*(diff.get("public") or []), *(diff.get("other_definitions") or [])] if isinstance(diff, dict) else []
|
|
2704
|
+
for item in rows:
|
|
2705
|
+
members = item.get("members") if isinstance(item, dict) and item.get("via") == "namespace" else None
|
|
2706
|
+
for named in [item, *(members if isinstance(members, list) else [])]:
|
|
2707
|
+
if not isinstance(named, dict):
|
|
2708
|
+
continue
|
|
2709
|
+
name, local = named.get("name"), named.get("local")
|
|
2710
|
+
key = (local, _norm_path(named.get("file")))
|
|
2711
|
+
if isinstance(name, str) and name and isinstance(local, str) and local != name and key in seeded:
|
|
2712
|
+
aliases.setdefault(key, set()).add(name) # the public name the runner gives `local` in that file
|
|
2713
|
+
for key, names in aliases.items():
|
|
2714
|
+
seeded[key]["aliases"] = sorted(names)
|
|
2715
|
+
surface, _problem = public_surface(extraction) # apply warns when a public-api run gives no surface
|
|
2716
|
+
for key in seeded if surface is not None else ():
|
|
2717
|
+
mark = surface.public(key[0], key[1], languages.get(key))
|
|
2718
|
+
if mark is not None:
|
|
2719
|
+
seeded[key]["public"] = mark
|
|
1787
2720
|
paths = list(dict.fromkeys([p for p in (_norm_path(f) for f in files or []) if p]
|
|
1788
2721
|
+ sorted({key[1] for key in seeded})))
|
|
1789
2722
|
blocks = [{"file_path": path, "exports": [seeded[key] for key in sorted(seeded) if key[1] == path]}
|
|
@@ -1797,6 +2730,7 @@ def build_records(extraction: dict | None, details: dict | None, files: list | N
|
|
|
1797
2730
|
"T1-low": sum(1 for r in exports if r["confidence"] == "T1-low"),
|
|
1798
2731
|
"T2": sum(1 for r in exports if r.get("qmd_evidence") is not None),
|
|
1799
2732
|
},
|
|
2733
|
+
"marked_not_public": sum(1 for r in exports if r.get("public") is False),
|
|
1800
2734
|
"warnings": warnings,
|
|
1801
2735
|
}
|
|
1802
2736
|
return {"mode": "normal", "files": blocks}, summary
|
|
@@ -1818,6 +2752,26 @@ DRIFT_IN_MAP = "a signature from the tree"
|
|
|
1818
2752
|
REACHABILITY = ("public", "internal-unreachable")
|
|
1819
2753
|
VERIFIER_MISSING = "provenance: spot-checks not run: skf-verify-provenance-completeness.py is missing; re-install SKF"
|
|
1820
2754
|
OUTCOMES = ("verified", "moved", "missing", "re-extracted", "rescoped", "unknown")
|
|
2755
|
+
# Why an entry ends `unknown` (its unknown_reason, see "Gap records"), with the words apply's warning gives it.
|
|
2756
|
+
UNKNOWN_REASONS = {
|
|
2757
|
+
"ambiguous": "several map entries of its name",
|
|
2758
|
+
"no-definition": "no line of its file defines it, by the definition-line rules",
|
|
2759
|
+
"several-definitions": "several lines of its file define it, none the recorded one",
|
|
2760
|
+
"unchecked-language": "the definition-line rules cover no language of its file, and no line was answered as "
|
|
2761
|
+
"declaring it",
|
|
2762
|
+
"no-line": "its map entry gives no file or line to check",
|
|
2763
|
+
"spot-check-failed": "the spot-check failed",
|
|
2764
|
+
"verifier-missing": "skf-verify-provenance-completeness.py is missing",
|
|
2765
|
+
"no-source-root": "no source tree to check",
|
|
2766
|
+
"drift-override": "the drift override kept its line from being moved",
|
|
2767
|
+
"cited-elsewhere": "the gap cites another file than the provenance map's entries of its name",
|
|
2768
|
+
}
|
|
2769
|
+
# A cited-elsewhere entry's citation_check that read nothing at the cited file, with the fix that lets a re-run read it.
|
|
2770
|
+
UNREAD = {
|
|
2771
|
+
"no-source-root": "give update-skill the source tree",
|
|
2772
|
+
"verifier-missing": "re-install SKF",
|
|
2773
|
+
"spot-check-failed": "fix what its spot-check failed on, which the `spot-check failed` warning names",
|
|
2774
|
+
}
|
|
1821
2775
|
_DEFINITION_LINES = re.compile(r"definition lines?\s*\(([\d,\s]+)\)", re.IGNORECASE)
|
|
1822
2776
|
_VERIFIER = None
|
|
1823
2777
|
|
|
@@ -1869,40 +2823,48 @@ def _drift_reason(entry: dict) -> str | None:
|
|
|
1869
2823
|
return None
|
|
1870
2824
|
if entry.get("provenance_completeness"):
|
|
1871
2825
|
return DRIFT_COMPLETENESS
|
|
1872
|
-
|
|
1873
|
-
|
|
2826
|
+
match = entry.get("map_match") if isinstance(entry.get("map_match"), dict) else {}
|
|
2827
|
+
status = match.get("status")
|
|
2828
|
+
# a not-found with a candidate that has a file is a citation of another file than the map's entries: held (#824)
|
|
2829
|
+
held = status in ("found", "ambiguous") or status == "not-found" and any(
|
|
2830
|
+
isinstance(c, dict) and _norm_path(c.get("source_file")) for c in match.get("candidates") or [])
|
|
2831
|
+
return DRIFT_IN_MAP if held else DRIFT_NOT_IN_MAP
|
|
1874
2832
|
|
|
1875
2833
|
|
|
1876
2834
|
def _spot_check(root: Path | None, path: str | None, name: str, line: int | None, export_type: object,
|
|
1877
|
-
answers: dict) -> tuple[str, int | None, bool, str | None]:
|
|
2835
|
+
answers: dict) -> tuple[str, int | None, bool, str | None, str | None]:
|
|
1878
2836
|
"""(outcome, the line a `moved` export moves to, whether it waits on a `declaring_line` answer, why the
|
|
1879
|
-
check failed)."""
|
|
2837
|
+
check failed, why it is `unknown`: an UNKNOWN_REASONS key)."""
|
|
2838
|
+
if root is None:
|
|
2839
|
+
return "unknown", None, False, None, "no-source-root"
|
|
1880
2840
|
verifier = _verifier()
|
|
1881
|
-
if verifier is None
|
|
1882
|
-
return "unknown", None, False, None
|
|
2841
|
+
if verifier is None:
|
|
2842
|
+
return "unknown", None, False, None, "verifier-missing"
|
|
2843
|
+
if not path or line is None:
|
|
2844
|
+
return "unknown", None, False, None, "no-line"
|
|
1883
2845
|
try:
|
|
1884
2846
|
report = verifier.definition_lines_report(path, name, root, line,
|
|
1885
2847
|
export_type if isinstance(export_type, str) else None)
|
|
1886
2848
|
except Exception as exc: # noqa: BLE001 - a file it cannot read, or any verifier failure: this entry only
|
|
1887
|
-
return "unknown", None, False, f"{type(exc).__name__}: {exc}"
|
|
2849
|
+
return "unknown", None, False, f"{type(exc).__name__}: {exc}", "spot-check-failed"
|
|
1888
2850
|
check = report.get("line_check")
|
|
1889
2851
|
if check == "file-missing":
|
|
1890
|
-
return "missing", None, False, None
|
|
2852
|
+
return "missing", None, False, None, None
|
|
1891
2853
|
if check == "skipped-export-type":
|
|
1892
|
-
return "verified", None, False, None
|
|
2854
|
+
return "verified", None, False, None, None
|
|
1893
2855
|
if check == "skipped-language":
|
|
1894
2856
|
if "declaring_line" not in answers:
|
|
1895
|
-
return "unknown", None, True, None
|
|
2857
|
+
return "unknown", None, True, None, None
|
|
1896
2858
|
declared = answers["declaring_line"]
|
|
1897
2859
|
if declared is None:
|
|
1898
|
-
return "unknown", None, False, None
|
|
1899
|
-
return ("verified", None, False, None) if declared == line else ("moved", declared, False, None)
|
|
2860
|
+
return "unknown", None, False, None, "unchecked-language"
|
|
2861
|
+
return ("verified", None, False, None, None) if declared == line else ("moved", declared, False, None, None)
|
|
1900
2862
|
if report.get("line_is_definition"):
|
|
1901
|
-
return "verified", None, False, None
|
|
2863
|
+
return "verified", None, False, None, None
|
|
1902
2864
|
defs = report.get("definition_lines") or []
|
|
1903
2865
|
if len(defs) == 1 and defs[0] != line:
|
|
1904
|
-
return "moved", defs[0], False, None
|
|
1905
|
-
return "unknown", None, False, None
|
|
2866
|
+
return "moved", defs[0], False, None, None
|
|
2867
|
+
return "unknown", None, False, None, "several-definitions" if defs else "no-definition"
|
|
1906
2868
|
|
|
1907
2869
|
|
|
1908
2870
|
def _pinned_lines(remediation: object) -> list[int] | None:
|
|
@@ -1912,6 +2874,18 @@ def _pinned_lines(remediation: object) -> list[int] | None:
|
|
|
1912
2874
|
return lines or None
|
|
1913
2875
|
|
|
1914
2876
|
|
|
2877
|
+
def _inside(root: Path | None, path: str | None) -> bool:
|
|
2878
|
+
"""True for a relative path that stays under the source root, links resolved."""
|
|
2879
|
+
rel = Path(path) if path else None
|
|
2880
|
+
if root is None or rel is None or rel.is_absolute() or ".." in rel.parts:
|
|
2881
|
+
return False
|
|
2882
|
+
try:
|
|
2883
|
+
(root / rel).resolve().relative_to(root.resolve())
|
|
2884
|
+
except (OSError, ValueError):
|
|
2885
|
+
return False
|
|
2886
|
+
return True
|
|
2887
|
+
|
|
2888
|
+
|
|
1915
2889
|
def _where(path: object, line: object) -> str | None:
|
|
1916
2890
|
file, number = _norm_path(path), _line_of(line)
|
|
1917
2891
|
return f"{file}:{number}" if file and number is not None else None
|
|
@@ -1931,7 +2905,14 @@ class _GapEntry:
|
|
|
1931
2905
|
self.map_entry = {"source_file": found.get("source_file"), "source_line": found.get("source_line")} \
|
|
1932
2906
|
if found else None
|
|
1933
2907
|
self.map_export_type = found.get("export_type") if found else None
|
|
2908
|
+
# the map entry's own name when skf-parse-gaps matched it by its reexported_as (the gap names the public one)
|
|
2909
|
+
self.declared = _name_of(found) if found and _name_of(found) else self.name
|
|
1934
2910
|
self.in_map = bool(match.get("candidates"))
|
|
2911
|
+
# where map_match's candidates sit, `file:line` (or the file); a placeholder with no file names none
|
|
2912
|
+
self.map_locations = list(dict.fromkeys(
|
|
2913
|
+
_where(c.get("source_file"), c.get("source_line")) or _norm_path(c.get("source_file"))
|
|
2914
|
+
for c in match.get("candidates") or [] if isinstance(c, dict) and _norm_path(c.get("source_file"))))
|
|
2915
|
+
self.citation_check: str | None = None # a cited-elsewhere entry's citation check: missing, or its reason
|
|
1935
2916
|
citation = entry.get("source_citation")
|
|
1936
2917
|
self.citation = citation if isinstance(citation, dict) else None
|
|
1937
2918
|
self.recorded: tuple[str | None, int | None] = (None, None) # the file and line a check read
|
|
@@ -1939,8 +2920,9 @@ class _GapEntry:
|
|
|
1939
2920
|
self.new_line: int | None = None
|
|
1940
2921
|
self.unknown_reason: str | None = None
|
|
1941
2922
|
self.reextract = False
|
|
1942
|
-
self.record: dict | None = None # the §4a record it matched
|
|
2923
|
+
self.record: dict | None = None # the §4a record it matched, or took its signature from
|
|
1943
2924
|
self.record_path: str | None = None
|
|
2925
|
+
self.signature_at: tuple[str, int] | None = None # where a check located it: its signature's record
|
|
1944
2926
|
self.waits = False # on a declaring_line answer
|
|
1945
2927
|
self.failure: str | None = None # why its spot-check failed
|
|
1946
2928
|
|
|
@@ -1954,8 +2936,25 @@ class _GapEntry:
|
|
|
1954
2936
|
|
|
1955
2937
|
def check(self, root: Path | None, path: object, line: object, export_type: object) -> None:
|
|
1956
2938
|
self.recorded = (_norm_path(path), _line_of(line))
|
|
1957
|
-
self.outcome, self.new_line, self.waits, self.failure = _spot_check(
|
|
1958
|
-
root, self.recorded[0], self.
|
|
2939
|
+
self.outcome, self.new_line, self.waits, self.failure, self.unknown_reason = _spot_check(
|
|
2940
|
+
root, self.recorded[0], self.declared, self.recorded[1], export_type, self.answers)
|
|
2941
|
+
|
|
2942
|
+
def locate(self, root: Path | None) -> None:
|
|
2943
|
+
"""Record where a check located the export (the checked file, at the recorded line or the one it moved
|
|
2944
|
+
to), so its signature comes from §4a's record there; never a file outside the source root."""
|
|
2945
|
+
line = self.new_line if self.outcome == "moved" else self.recorded[1]
|
|
2946
|
+
if self.outcome in ("verified", "moved") and line is not None and _inside(root, self.recorded[0]):
|
|
2947
|
+
self.signature_at = (self.recorded[0], line)
|
|
2948
|
+
|
|
2949
|
+
def take_signature(self, blocks: dict[str, list[dict]]) -> None:
|
|
2950
|
+
"""Take the one record of the entry's name in the block of the file it was located in that is at its line,
|
|
2951
|
+
else the one with no line; none otherwise."""
|
|
2952
|
+
path, line = self.signature_at
|
|
2953
|
+
named = [r for r in blocks.get(path, []) if _name_of(r) in (self.name, self.declared)]
|
|
2954
|
+
at = [r for r in named if _line_of(r.get("location")) == line] \
|
|
2955
|
+
or [r for r in named if _line_of(r.get("location")) is None]
|
|
2956
|
+
if len(at) == 1:
|
|
2957
|
+
self.record, self.record_path = at[0], path
|
|
1959
2958
|
|
|
1960
2959
|
def route(self, root: Path | None, drift: bool) -> None:
|
|
1961
2960
|
"""Spot-check the entry, or send it to targeted re-extraction (see "Gap records" in the module docstring)."""
|
|
@@ -1974,14 +2973,23 @@ class _GapEntry:
|
|
|
1974
2973
|
return
|
|
1975
2974
|
if self.match_status == "found":
|
|
1976
2975
|
self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
|
|
2976
|
+
self.locate(root)
|
|
1977
2977
|
return
|
|
1978
|
-
if self.match_status == "ambiguous":
|
|
1979
|
-
self.outcome = "unknown"
|
|
2978
|
+
if self.match_status == "ambiguous": # several map entries of its name: no check, apply warns
|
|
2979
|
+
self.outcome, self.unknown_reason = "unknown", "ambiguous"
|
|
1980
2980
|
return
|
|
1981
2981
|
if self.citation:
|
|
1982
2982
|
self.check(root, self.citation.get("file"), self.citation.get("line"), None)
|
|
1983
2983
|
if self.waits or self.outcome in ("verified", "moved"):
|
|
2984
|
+
if not self.in_map: # a citation of another file than the map's entries of its name takes none
|
|
2985
|
+
self.locate(root)
|
|
1984
2986
|
return
|
|
2987
|
+
if self.match_status == "not-found" and self.map_locations:
|
|
2988
|
+
# the map holds its name at other files: the cited file's declaration may be another package's, so it is
|
|
2989
|
+
# never re-extracted there (#824); the record keeps the check's cause, and apply warns with what to fix
|
|
2990
|
+
self.citation_check = "missing" if self.outcome == "missing" else self.unknown_reason
|
|
2991
|
+
self.outcome, self.unknown_reason = "unknown", "cited-elsewhere"
|
|
2992
|
+
return
|
|
1985
2993
|
if self.entry.get("provenance_completeness") and root is not None:
|
|
1986
2994
|
self.reextract = True
|
|
1987
2995
|
elif self.blocking:
|
|
@@ -1992,14 +3000,20 @@ class _GapEntry:
|
|
|
1992
3000
|
self.outcome = "unknown"
|
|
1993
3001
|
|
|
1994
3002
|
def match(self, blocks: dict[str, list[dict]]) -> tuple[int, int]:
|
|
1995
|
-
"""Take the first export of the entry's name in the blocks of its resolved files
|
|
1996
|
-
|
|
3003
|
+
"""Take the first export of the entry's name in the blocks of its resolved files, else the first whose
|
|
3004
|
+
`aliases` hold its name (the public name the runner gives its local name in that file); (files scanned,
|
|
3005
|
+
exports found in them)."""
|
|
1997
3006
|
exports = 0
|
|
3007
|
+
aliased: tuple[dict, str] | None = None
|
|
1998
3008
|
for path in self.paths:
|
|
1999
3009
|
for record in blocks.get(path, []):
|
|
2000
3010
|
exports += 1
|
|
2001
3011
|
if self.record is None and _name_of(record) == self.name:
|
|
2002
3012
|
self.record, self.record_path = record, path
|
|
3013
|
+
elif aliased is None and isinstance(record.get("aliases"), list) and self.name in record["aliases"]:
|
|
3014
|
+
aliased = (record, path)
|
|
3015
|
+
if self.record is None and aliased is not None:
|
|
3016
|
+
self.record, self.record_path = aliased
|
|
2003
3017
|
if self.record is not None:
|
|
2004
3018
|
self.outcome = "re-extracted"
|
|
2005
3019
|
return len(self.paths), exports
|
|
@@ -2028,8 +3042,14 @@ class _GapEntry:
|
|
|
2028
3042
|
return (self.record or {}).get("location")
|
|
2029
3043
|
return None
|
|
2030
3044
|
|
|
3045
|
+
def signature(self) -> dict | None:
|
|
3046
|
+
"""The record a located export takes its signature from, or None."""
|
|
3047
|
+
return self.record if self.signature_at and not self.internal else None
|
|
3048
|
+
|
|
2031
3049
|
def verification(self) -> dict:
|
|
3050
|
+
signed = self.signature() or {}
|
|
2032
3051
|
recorded = _where(*self.recorded)
|
|
3052
|
+
elsewhere = self.outcome == "unknown" and self.unknown_reason == "cited-elsewhere"
|
|
2033
3053
|
if self.outcome == "re-extracted":
|
|
2034
3054
|
citation = self.new_location()
|
|
2035
3055
|
elif self.outcome in ("verified", "moved", "missing", "rescoped") and recorded:
|
|
@@ -2046,13 +3066,19 @@ class _GapEntry:
|
|
|
2046
3066
|
"provenance_citation": citation,
|
|
2047
3067
|
"source_citation": self.citation,
|
|
2048
3068
|
"new_location": self.new_location(),
|
|
2049
|
-
"unknown_reason": self.unknown_reason,
|
|
3069
|
+
"unknown_reason": self.unknown_reason if self.outcome == "unknown" else None,
|
|
3070
|
+
"citation_check": self.citation_check if elsewhere else None,
|
|
3071
|
+
"map_locations": self.map_locations if elsewhere else None,
|
|
2050
3072
|
"pinned_definition_lines": _pinned_lines(self.entry.get("remediation"))
|
|
2051
3073
|
if self.category == "MOVED_EXPORT" else None,
|
|
2052
3074
|
"resolution_source": "remediation-paths" if self.outcome == "re-extracted" else None,
|
|
2053
3075
|
"reachability": self.answers.get("reachability") if self.category == "NEW_EXPORT" and self.located
|
|
2054
3076
|
else None,
|
|
2055
|
-
"export_type":
|
|
3077
|
+
"export_type": signed.get("type", signed.get("export_type")),
|
|
3078
|
+
"params": signed.get("params", signed.get("parameters")),
|
|
3079
|
+
"return_type": signed.get("return_type"),
|
|
3080
|
+
"extraction_method": signed.get("extraction_method"),
|
|
3081
|
+
"ast_node_type": signed.get("ast_node_type"),
|
|
2056
3082
|
}
|
|
2057
3083
|
|
|
2058
3084
|
def label(self, provenance: list[dict]) -> str:
|
|
@@ -2065,7 +3091,7 @@ class _GapEntry:
|
|
|
2065
3091
|
elif self.map_entry is not None:
|
|
2066
3092
|
file, line = _norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"])
|
|
2067
3093
|
for entry in provenance:
|
|
2068
|
-
if (_name_of(entry) == self.
|
|
3094
|
+
if (_name_of(entry) == self.declared and _file_of(entry) == file
|
|
2069
3095
|
and _line_of(entry.get("source_line")) == line):
|
|
2070
3096
|
method = entry.get("extraction_method")
|
|
2071
3097
|
break
|
|
@@ -2114,17 +3140,22 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
|
|
|
2114
3140
|
if waiting:
|
|
2115
3141
|
return None, {"status": "needs-judgment", "needs_judgment": waiting, "warnings": warnings}
|
|
2116
3142
|
to_scan = [g for g in routed if g.reextract]
|
|
2117
|
-
|
|
3143
|
+
located = [g for g in routed if g.signature_at and not g.internal] # an internal export takes no signature
|
|
3144
|
+
files = list(dict.fromkeys([p for g in to_scan for p in g.paths] + [g.signature_at[0] for g in located]))
|
|
2118
3145
|
if plan:
|
|
2119
3146
|
return None, {"status": "planned", "reextract": {
|
|
2120
3147
|
"entries": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
|
|
2121
|
-
"resolved_paths": g.paths} for g in to_scan], "files": files
|
|
3148
|
+
"resolved_paths": g.paths} for g in to_scan], "files": files,
|
|
3149
|
+
"signatures": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "file": g.signature_at[0],
|
|
3150
|
+
"line": g.signature_at[1]} for g in located]}, "warnings": warnings}
|
|
2122
3151
|
blocks: dict[str, list[dict]] = {}
|
|
2123
3152
|
scanned = 0
|
|
3153
|
+
targeted_files = {p for g in to_scan for p in g.paths}
|
|
3154
|
+
signature_files = {g.signature_at[0] for g in located}
|
|
2124
3155
|
if files:
|
|
2125
3156
|
if remediation is None:
|
|
2126
|
-
raise ValueError("
|
|
2127
|
-
"of `records` over them")
|
|
3157
|
+
raise ValueError("gap-records has files to match (targeted re-extraction, or a located export's "
|
|
3158
|
+
"signature): pass --remediation-records, the output of `records` over them")
|
|
2128
3159
|
if not isinstance(remediation, dict) or not isinstance(remediation.get("files"), list):
|
|
2129
3160
|
raise ValueError("--remediation-records has no `files` array: is it the output of `records`?")
|
|
2130
3161
|
for block in remediation["files"]:
|
|
@@ -2146,20 +3177,36 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
|
|
|
2146
3177
|
"files_scanned": files_scanned, "exports_found_in_scan": found})
|
|
2147
3178
|
if unresolved:
|
|
2148
3179
|
return None, {"status": "unresolved", "unresolved": unresolved, "warnings": warnings}
|
|
3180
|
+
for g in located:
|
|
3181
|
+
g.take_signature(blocks)
|
|
2149
3182
|
needs = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": g.needs(),
|
|
2150
3183
|
"file": _location(g.new_location())[0] if g.outcome != "verified" else g.recorded[0],
|
|
2151
3184
|
"line": _location(g.new_location())[1] if g.outcome != "verified" else g.recorded[1]}
|
|
2152
3185
|
for g in routed if g.needs()]
|
|
2153
3186
|
if needs:
|
|
2154
3187
|
return None, {"status": "needs-judgment", "needs_judgment": needs, "warnings": warnings}
|
|
3188
|
+
warnings += [f"provenance: {g.name}: located at {g.signature_at[0]}:{g.signature_at[1]}, but no record of it "
|
|
3189
|
+
f"there; signature not repaired" for g in located if g.record is None]
|
|
2155
3190
|
kept: dict[str, list[dict]] = {}
|
|
2156
3191
|
for g in routed:
|
|
2157
|
-
if g.record is
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
3192
|
+
if g.record is None or g.internal:
|
|
3193
|
+
continue
|
|
3194
|
+
record = dict(g.record)
|
|
3195
|
+
record.pop("public", None) # a repair documents and maps every record: the mark is a normal update's
|
|
3196
|
+
record.pop("aliases", None) # the match is made: the copy names the one it took
|
|
3197
|
+
if _name_of(record) != g.name: # matched by its public name: documented under it, its own name as `local`
|
|
3198
|
+
record = {"name": g.name, "local": _name_of(record),
|
|
3199
|
+
**{k: v for k, v in record.items() if k not in ("name", "export_name", "local")}}
|
|
3200
|
+
if g.outcome == "re-extracted" and g.answers.get("docstring") is not None:
|
|
3201
|
+
record["docstring"] = g.answers["docstring"] # a located export's signature record asks none
|
|
3202
|
+
block = kept.setdefault(g.record_path, [])
|
|
3203
|
+
location = record.get("location")
|
|
3204
|
+
same = next((r for r in block if (_name_of(r) == _name_of(record) and r.get("location") == location
|
|
3205
|
+
if location is not None else r == record)), None)
|
|
3206
|
+
if same is None:
|
|
3207
|
+
block.append(record)
|
|
3208
|
+
elif same.get("docstring") is None and record.get("docstring") is not None:
|
|
3209
|
+
same["docstring"] = record["docstring"]
|
|
2163
3210
|
breakdown = {"T1": 0, "T1-low": 0, "unlabeled": 0, "T2": 0}
|
|
2164
3211
|
provenance_entries = _entries_of(provenance)
|
|
2165
3212
|
for g in routed:
|
|
@@ -2170,13 +3217,15 @@ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path |
|
|
|
2170
3217
|
"files_extracted": scanned,
|
|
2171
3218
|
"exports_extracted": len(entries),
|
|
2172
3219
|
"confidence_breakdown": breakdown,
|
|
2173
|
-
"verification": [g.verification() for g in routed],
|
|
3220
|
+
"verification": [{"gap_id": g.entry.get("gap_id"), **g.verification()} for g in routed],
|
|
2174
3221
|
"files": [{"file_path": path, "exports": exports} for path, exports in kept.items()],
|
|
2175
3222
|
}
|
|
2176
3223
|
matched = [g for g in to_scan if g.record is not None]
|
|
2177
|
-
|
|
3224
|
+
scanned_paths = set(blocks)
|
|
3225
|
+
targeted = {"resolved_count": len(matched), "files_scanned": len(scanned_paths & targeted_files),
|
|
3226
|
+
"signature_files_scanned": len(scanned_paths & signature_files),
|
|
2178
3227
|
"exports_matched": len({(g.record_path, _name_of(g.record)) for g in matched}),
|
|
2179
|
-
"tier": tier} if to_scan else None
|
|
3228
|
+
"tier": tier} if to_scan or located else None
|
|
2180
3229
|
summary = {
|
|
2181
3230
|
"status": "written",
|
|
2182
3231
|
"gap_count": len(entries),
|
|
@@ -2356,10 +3405,93 @@ def _apply_categories(args: argparse.Namespace) -> dict | None:
|
|
|
2356
3405
|
return {**(categories or {}), "category_c": cat_c}
|
|
2357
3406
|
|
|
2358
3407
|
|
|
3408
|
+
def _metadata_in(path: str | None) -> dict | None:
|
|
3409
|
+
"""apply's --metadata-in: the skill's metadata.json, an object."""
|
|
3410
|
+
if not path:
|
|
3411
|
+
return None
|
|
3412
|
+
metadata = _load_json_file(Path(path), "--metadata-in file")
|
|
3413
|
+
if not isinstance(metadata, dict):
|
|
3414
|
+
raise ValueError("--metadata-in file must hold a JSON object (the skill's metadata.json)")
|
|
3415
|
+
return metadata
|
|
3416
|
+
|
|
3417
|
+
|
|
3418
|
+
_QUICK_METADATA = None
|
|
3419
|
+
|
|
3420
|
+
|
|
3421
|
+
def _quick_metadata():
|
|
3422
|
+
"""skf-render-quick-metadata.py, loaded once from this folder as skf-extraction-inventory.py loads it: its
|
|
3423
|
+
probe_skf_version reads the installed SKF's version, so create-skill and update-skill read it one way. Raises
|
|
3424
|
+
ValueError, one line, when it cannot be loaded."""
|
|
3425
|
+
global _QUICK_METADATA
|
|
3426
|
+
if _QUICK_METADATA is None:
|
|
3427
|
+
path = Path(__file__).resolve().parent / "skf-render-quick-metadata.py"
|
|
3428
|
+
cannot = f"cannot load {path.name} beside {Path(__file__).name}"
|
|
3429
|
+
if not path.is_file():
|
|
3430
|
+
raise ValueError(f"{cannot}: it is missing; re-install SKF")
|
|
3431
|
+
spec = importlib.util.spec_from_file_location("skf_render_quick_metadata", path)
|
|
3432
|
+
if spec is None or spec.loader is None:
|
|
3433
|
+
raise ValueError(f"{cannot}: re-install SKF")
|
|
3434
|
+
module = importlib.util.module_from_spec(spec)
|
|
3435
|
+
try:
|
|
3436
|
+
spec.loader.exec_module(module)
|
|
3437
|
+
except (ImportError, SyntaxError, OSError) as exc:
|
|
3438
|
+
raise ValueError(f"{cannot} ({type(exc).__name__}: {' '.join(str(exc).split())}): re-install "
|
|
3439
|
+
f"SKF") from exc
|
|
3440
|
+
if not callable(getattr(module, "probe_skf_version", None)):
|
|
3441
|
+
raise ValueError(f"{cannot}: it has no probe_skf_version; re-install SKF")
|
|
3442
|
+
_QUICK_METADATA = module
|
|
3443
|
+
return _QUICK_METADATA
|
|
3444
|
+
|
|
3445
|
+
|
|
3446
|
+
def _runner_ast_grep(path: str | None) -> str | None:
|
|
3447
|
+
"""The ast-grep version the recipe runner's output (--extraction) records: "unknown" when it records none but
|
|
3448
|
+
holds an export ast-grep matched, and None when ast-grep did not run (no export it matched, as in the
|
|
3449
|
+
{"exports": []} step 2 writes when the runner cannot run)."""
|
|
3450
|
+
if not path:
|
|
3451
|
+
return None
|
|
3452
|
+
extraction = _load_json_file(Path(path), "--extraction file")
|
|
3453
|
+
if not isinstance(extraction, dict):
|
|
3454
|
+
raise ValueError("--extraction file must hold a JSON object (the recipe runner's output)")
|
|
3455
|
+
ast_grep = extraction.get("ast_grep")
|
|
3456
|
+
version = _tool_version(ast_grep.get("version")) if isinstance(ast_grep, dict) else None
|
|
3457
|
+
if version is not None:
|
|
3458
|
+
return version
|
|
3459
|
+
exports = extraction.get("exports") if isinstance(extraction.get("exports"), list) else []
|
|
3460
|
+
matched = any(isinstance(e, dict) and e.get("extraction_method") == "ast-grep" for e in exports)
|
|
3461
|
+
return "unknown" if matched else None
|
|
3462
|
+
|
|
3463
|
+
|
|
3464
|
+
def _cmd_metadata(args: argparse.Namespace) -> int:
|
|
3465
|
+
target = Path(args.metadata)
|
|
3466
|
+
try:
|
|
3467
|
+
skf_version = _quick_metadata().probe_skf_version(args.skf_root) if args.skf_root else None
|
|
3468
|
+
metadata, summary = update_metadata(
|
|
3469
|
+
_load_json_file(target, "--metadata file"), _load_json_file(Path(args.lists), "--lists file"),
|
|
3470
|
+
_load_json_file(Path(args.stats), "--stats file"), version=args.version,
|
|
3471
|
+
generation_date=args.generation_date, source_commit=args.source_commit, source_ref=args.source_ref,
|
|
3472
|
+
skf_version=skf_version, skf_root=args.skf_root, ast_grep_version=_runner_ast_grep(args.extraction),
|
|
3473
|
+
qmd_version=args.qmd_version)
|
|
3474
|
+
_write_json_atomic(target, metadata)
|
|
3475
|
+
except ValueError as exc:
|
|
3476
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
3477
|
+
return 1
|
|
3478
|
+
except OSError as exc:
|
|
3479
|
+
print(f"error: cannot write {args.metadata}: {exc}", file=sys.stderr)
|
|
3480
|
+
return 1
|
|
3481
|
+
print(json.dumps({"status": "written", "metadata": target.as_posix(), "version": metadata["version"],
|
|
3482
|
+
"exports": len(metadata["exports"]), "scripts": len(metadata.get("scripts") or []),
|
|
3483
|
+
"assets": len(metadata.get("assets") or []), "tool_versions": metadata["tool_versions"],
|
|
3484
|
+
**summary}))
|
|
3485
|
+
return 0
|
|
3486
|
+
|
|
3487
|
+
|
|
2359
3488
|
def _cmd_apply(args: argparse.Namespace) -> int:
|
|
2360
3489
|
if bool(args.drift_head) != bool(args.drift_pinned):
|
|
2361
3490
|
print("error: --drift-head and --drift-pinned go together", file=sys.stderr)
|
|
2362
3491
|
return 1
|
|
3492
|
+
if bool(args.metadata_in) != bool(args.lists_out):
|
|
3493
|
+
print("error: --metadata-in and --lists-out go together", file=sys.stderr)
|
|
3494
|
+
return 1
|
|
2363
3495
|
try:
|
|
2364
3496
|
promoted = _optional_json(args.promoted_docs, "--promoted-docs file")
|
|
2365
3497
|
if promoted is not None and not isinstance(promoted, list):
|
|
@@ -2385,6 +3517,7 @@ def _cmd_apply(args: argparse.Namespace) -> int:
|
|
|
2385
3517
|
source_commit=args.source_commit,
|
|
2386
3518
|
source_ref=args.source_ref,
|
|
2387
3519
|
drift=(args.drift_head, args.drift_pinned) if args.drift_head else None,
|
|
3520
|
+
metadata=_metadata_in(args.metadata_in),
|
|
2388
3521
|
)
|
|
2389
3522
|
except ValueError as exc:
|
|
2390
3523
|
print(f"error: {exc}", file=sys.stderr)
|
|
@@ -2392,11 +3525,15 @@ def _cmd_apply(args: argparse.Namespace) -> int:
|
|
|
2392
3525
|
if summary["status"] == "refused":
|
|
2393
3526
|
print(json.dumps(summary))
|
|
2394
3527
|
return EXIT_REFUSED
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
3528
|
+
lists = summary.pop("lists", None)
|
|
3529
|
+
for path, value in ((args.not_public_out, summary["not_public"]), (args.lists_out, lists),
|
|
3530
|
+
(args.output, doc)):
|
|
3531
|
+
try:
|
|
3532
|
+
if path:
|
|
3533
|
+
_write_json_atomic(Path(path), value)
|
|
3534
|
+
except OSError as exc:
|
|
3535
|
+
print(f"error: cannot write {path}: {exc}", file=sys.stderr)
|
|
3536
|
+
return 1
|
|
2400
3537
|
print(json.dumps({**summary, "map": args.output}))
|
|
2401
3538
|
return 0
|
|
2402
3539
|
|
|
@@ -2464,6 +3601,76 @@ def _cmd_gap_records(args: argparse.Namespace) -> int:
|
|
|
2464
3601
|
return 0
|
|
2465
3602
|
|
|
2466
3603
|
|
|
3604
|
+
# --------------------------------------------------------------------------
|
|
3605
|
+
# Baseline gaps (update-skill detect-changes Category B)
|
|
3606
|
+
# --------------------------------------------------------------------------
|
|
3607
|
+
|
|
3608
|
+
|
|
3609
|
+
VERIFIER_FILE = "skf-verify-provenance-completeness.py"
|
|
3610
|
+
|
|
3611
|
+
|
|
3612
|
+
def _baseline_verifier():
|
|
3613
|
+
"""The verifier beside this script, for its `baseline_gaps`. Unlike `_verifier()`, which lets a gap-driven
|
|
3614
|
+
spot-check go on as unknown, one that cannot be loaded (or has no `baseline_gaps`) is a ValueError: with no
|
|
3615
|
+
verifier no gap is listed, and every name the runner leaves out would read as a deleted export."""
|
|
3616
|
+
path = Path(__file__).resolve().parent / VERIFIER_FILE
|
|
3617
|
+
where = f"cannot load {VERIFIER_FILE} beside {Path(__file__).name}"
|
|
3618
|
+
try:
|
|
3619
|
+
spec = importlib.util.spec_from_file_location("skf_verify_provenance_completeness_baseline", path)
|
|
3620
|
+
if spec is None or spec.loader is None or not path.is_file():
|
|
3621
|
+
raise ImportError(f"no file at {path}")
|
|
3622
|
+
module = importlib.util.module_from_spec(spec)
|
|
3623
|
+
spec.loader.exec_module(module)
|
|
3624
|
+
except Exception as exc: # whatever the script raises, the call reports it
|
|
3625
|
+
raise ValueError(f"{where}: {exc}; re-install SKF") from exc
|
|
3626
|
+
if not callable(getattr(module, "baseline_gaps", None)):
|
|
3627
|
+
raise ValueError(f"{where}: it has no baseline_gaps; re-install SKF")
|
|
3628
|
+
return module
|
|
3629
|
+
|
|
3630
|
+
|
|
3631
|
+
def baseline_gaps(provenance: dict, extraction: dict, files: list, source_root: Path, verifier) -> dict:
|
|
3632
|
+
"""{"gaps", "unchecked"}: the map entries of `files` the runner's `extraction` lacks whose file still declares
|
|
3633
|
+
them (see "Baseline gaps" in the module docstring)."""
|
|
3634
|
+
if not isinstance(extraction, dict):
|
|
3635
|
+
raise ValueError("--extraction file has no `exports` array: is it the helper's output?")
|
|
3636
|
+
held = {(name, file) for e in _exports_of(extraction, "--extraction file")
|
|
3637
|
+
if (name := _name_of(e)) is not None and (file := _file_of(e)) is not None}
|
|
3638
|
+
diff = extraction.get("entry_point_diff")
|
|
3639
|
+
listed = {(item["name"], _norm_path(item["file"]))
|
|
3640
|
+
for item in (diff.get("extraction_gaps") or [] if isinstance(diff, dict) else [])
|
|
3641
|
+
if isinstance(item, dict) and isinstance(item.get("name"), str) and _norm_path(item.get("file"))}
|
|
3642
|
+
if provenance.get("entries") is not None and not isinstance(provenance["entries"], list):
|
|
3643
|
+
raise ValueError("provenance `entries` must be an array") # missing or null: no entry (#684)
|
|
3644
|
+
wanted = {f for f in (_norm_path(p) for p in files) if f is not None}
|
|
3645
|
+
gaps, unchecked = verifier.baseline_gaps(provenance, source_root, held, wanted, listed)
|
|
3646
|
+
return {"gaps": gaps, "unchecked": unchecked}
|
|
3647
|
+
|
|
3648
|
+
|
|
3649
|
+
def _cmd_baseline_gaps(args: argparse.Namespace) -> int:
|
|
3650
|
+
try:
|
|
3651
|
+
source_root = Path(args.source_root)
|
|
3652
|
+
if not source_root.is_dir():
|
|
3653
|
+
raise ValueError(f"--source-root {args.source_root} is not a folder")
|
|
3654
|
+
files = _load_json_file(Path(args.files), "--files file")
|
|
3655
|
+
if not (isinstance(files, list) and all(isinstance(f, str) for f in files)):
|
|
3656
|
+
raise ValueError(f"--files file {args.files} must hold a JSON array of paths")
|
|
3657
|
+
provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file")
|
|
3658
|
+
if not isinstance(provenance, dict):
|
|
3659
|
+
raise ValueError(f"--provenance-map file {args.provenance_map} must hold a JSON object")
|
|
3660
|
+
extraction = _load_json_file(Path(args.extraction), "--extraction file")
|
|
3661
|
+
result = baseline_gaps(provenance, extraction, files, source_root, _baseline_verifier())
|
|
3662
|
+
_write_json_atomic(Path(args.output), result)
|
|
3663
|
+
except ValueError as exc:
|
|
3664
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
3665
|
+
return 1
|
|
3666
|
+
except OSError as exc:
|
|
3667
|
+
print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
|
|
3668
|
+
return 1
|
|
3669
|
+
print(json.dumps({"status": "written", "output": args.output, "gaps": len(result["gaps"]),
|
|
3670
|
+
"unchecked": len(result["unchecked"])}))
|
|
3671
|
+
return 0
|
|
3672
|
+
|
|
3673
|
+
|
|
2467
3674
|
def _build_parser() -> argparse.ArgumentParser:
|
|
2468
3675
|
parser = argparse.ArgumentParser(
|
|
2469
3676
|
prog="skf-build-change-manifest",
|
|
@@ -2537,9 +3744,50 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
2537
3744
|
p_apply.add_argument("--source-ref", default=None, help="the map's source_ref (empty: null)")
|
|
2538
3745
|
p_apply.add_argument("--drift-head", default=None, help="under the drift override: HEAD's short SHA")
|
|
2539
3746
|
p_apply.add_argument("--drift-pinned", default=None, help="under the drift override: the pinned short SHA")
|
|
3747
|
+
p_apply.add_argument("--not-public-out", metavar="FILE",
|
|
3748
|
+
help="also write the summary's not_public list here (a JSON array)")
|
|
3749
|
+
p_apply.add_argument("--metadata-in", metavar="FILE",
|
|
3750
|
+
help="the skill's metadata.json, whose exports[], scripts[] and assets[] the update "
|
|
3751
|
+
"starts from")
|
|
3752
|
+
p_apply.add_argument("--lists-out", metavar="FILE",
|
|
3753
|
+
help="write the exports[], scripts[] and assets[] metadata.json takes after the update "
|
|
3754
|
+
"here (a JSON object)")
|
|
2540
3755
|
p_apply.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the new map")
|
|
2541
3756
|
p_apply.set_defaults(func=_cmd_apply)
|
|
2542
3757
|
|
|
3758
|
+
p_meta = sub.add_parser(
|
|
3759
|
+
"metadata", help="write the metadata.json an update leaves",
|
|
3760
|
+
description="Rewrite the skill's metadata.json in place. It sets version and generation_date, "
|
|
3761
|
+
"source_commit and source_ref when given, exports, scripts and assets from --lists (a script "
|
|
3762
|
+
"or asset list left out when empty), confidence_distribution from --stats and the stats "
|
|
3763
|
+
"helper's stats fields, dropping one an earlier run wrote that --stats lacks; tool_versions.skf "
|
|
3764
|
+
"from --skf-root (unknown, with a warning, when it gives no version), and tool_versions.ast_grep "
|
|
3765
|
+
"and tool_versions.qmd from --extraction and --qmd-version when this run used them, else as "
|
|
3766
|
+
"they stand. When --stats' effective_denominator differs from the old one (or one is absent), it "
|
|
3767
|
+
"drops each old stats key named effective_denominator_*; each dropped key is in dropped_stats, "
|
|
3768
|
+
"with a warning. It keeps every other old stats key, with a warning for each one create-skill "
|
|
3769
|
+
"does not write for the skill's scope_type. It removes last_update and "
|
|
3770
|
+
"update_type and keeps every other field, a patched one included. It refuses a --stats whose "
|
|
3771
|
+
"scripts_count or assets_count is not the length of --lists' scripts or assets.")
|
|
3772
|
+
p_meta.add_argument("--metadata", required=True, metavar="FILE",
|
|
3773
|
+
help="the skill's metadata.json, rewritten in place")
|
|
3774
|
+
p_meta.add_argument("--lists", required=True, metavar="FILE", help="the lists apply wrote (--lists-out)")
|
|
3775
|
+
p_meta.add_argument("--stats", required=True, metavar="FILE",
|
|
3776
|
+
help="the JSON skf-render-metadata-stats.py wrote (its -o file)")
|
|
3777
|
+
p_meta.add_argument("--version", required=True, help="the version this update writes")
|
|
3778
|
+
p_meta.add_argument("--generation-date", required=True, help="the clock's UTC time, YYYY-MM-DDTHH:MM:SSZ")
|
|
3779
|
+
p_meta.add_argument("--source-commit", default=None, help="the commit this update read (an empty one: null)")
|
|
3780
|
+
p_meta.add_argument("--source-ref", default=None, help="the ref this update read (an empty one: null)")
|
|
3781
|
+
p_meta.add_argument("--skf-root", metavar="DIR",
|
|
3782
|
+
help="the installed SKF ({project-root}/_bmad/skf): tool_versions.skf, from its package.json "
|
|
3783
|
+
"or VERSION file")
|
|
3784
|
+
p_meta.add_argument("--extraction", metavar="FILE",
|
|
3785
|
+
help="the recipe runner's output this run wrote: tool_versions.ast_grep, its ast_grep.version")
|
|
3786
|
+
p_meta.add_argument("--qmd-version", default=None,
|
|
3787
|
+
help="what qmd --version printed, when this run queried QMD: tool_versions.qmd (a leading "
|
|
3788
|
+
"\"qmd \" removed)")
|
|
3789
|
+
p_meta.set_defaults(func=_cmd_metadata)
|
|
3790
|
+
|
|
2543
3791
|
p_records = sub.add_parser("records", help="write step 3's re-extraction records from the runner's and the "
|
|
2544
3792
|
"workers' files")
|
|
2545
3793
|
p_records.add_argument("--extraction", metavar="FILE", help="the recipe runner's output")
|
|
@@ -2559,9 +3807,9 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
2559
3807
|
help="the workspace drift check's status; overridden runs the drift gate")
|
|
2560
3808
|
p_gap.add_argument("--judgments", metavar="FILE", help="the answers to its needs_judgment[], keyed by gap id")
|
|
2561
3809
|
p_gap.add_argument("--remediation-records", metavar="FILE",
|
|
2562
|
-
help="`records` output over the files
|
|
3810
|
+
help="`records` output over the files the plan listed")
|
|
2563
3811
|
p_gap.add_argument("--plan", action="store_true",
|
|
2564
|
-
help="print the answers it needs and the files
|
|
3812
|
+
help="print the answers it needs and the files §4a's extraction scans; write no records")
|
|
2565
3813
|
p_gap.add_argument("--files-out", metavar="FILE", help="with --plan: write those files here, a JSON array")
|
|
2566
3814
|
p_gap.add_argument("--evidence", metavar="FILE",
|
|
2567
3815
|
help="append the targeted_reextraction record to this JSON-lines file when it ran")
|
|
@@ -2569,6 +3817,17 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
2569
3817
|
p_gap.add_argument("-o", "--output", metavar="FILE", help="where to write the records (without --plan)")
|
|
2570
3818
|
p_gap.set_defaults(func=_cmd_gap_records)
|
|
2571
3819
|
|
|
3820
|
+
p_base = sub.add_parser("baseline-gaps", help="list the map entries the runner left out that their modified "
|
|
3821
|
+
"file still declares, for Category B's step 2 to read")
|
|
3822
|
+
p_base.add_argument("--provenance-map", required=True, metavar="FILE", help="the map the update starts from")
|
|
3823
|
+
p_base.add_argument("--extraction", required=True, metavar="FILE",
|
|
3824
|
+
help="the recipe runner's output ({\"exports\": []} when it could not run)")
|
|
3825
|
+
p_base.add_argument("--files", required=True, metavar="FILE",
|
|
3826
|
+
help="the files whose entries are checked: Category A's modified-files.json")
|
|
3827
|
+
p_base.add_argument("--source-root", required=True, metavar="DIR", help="the source tree the files are read in")
|
|
3828
|
+
p_base.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the gaps")
|
|
3829
|
+
p_base.set_defaults(func=_cmd_baseline_gaps)
|
|
3830
|
+
|
|
2572
3831
|
for p in (p_build, p_ratio):
|
|
2573
3832
|
p.add_argument(
|
|
2574
3833
|
"--input", default=None, help="path to JSON input (default: stdin)"
|