bmad-module-skill-forge 2.2.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +49 -49
- package/docs/404.md +10 -8
- package/docs/_data/pinned.yaml +22 -13
- package/docs/_internal/STABILITY.md +58 -35
- package/docs/agents.md +39 -53
- package/docs/architecture.md +65 -51
- package/docs/bmad-synergy.md +53 -51
- package/docs/campaign.md +127 -75
- package/docs/concepts.md +34 -28
- package/docs/examples.md +70 -68
- package/docs/forge-auto.md +34 -31
- package/docs/getting-started.md +118 -57
- package/docs/how-it-works.md +25 -21
- package/docs/index.md +15 -15
- package/docs/skill-model.md +70 -59
- package/docs/troubleshooting.md +143 -18
- package/docs/verifying-a-skill.md +87 -57
- package/docs/why-skf.md +24 -22
- package/docs/workflows.md +274 -110
- package/package.json +18 -10
- package/src/forger/preferences.yaml +8 -1
- package/src/knowledge/agentskills-spec.md +2 -0
- package/src/knowledge/ccc-bridge.md +40 -24
- package/src/knowledge/confidence-tiers.md +9 -5
- package/src/knowledge/overview.md +3 -3
- package/src/knowledge/progressive-capability.md +8 -8
- package/src/knowledge/provenance-tracking.md +9 -5
- package/src/knowledge/qmd-registry.md +3 -3
- package/src/knowledge/skf-knowledge-index.csv +2 -2
- package/src/knowledge/skill-lifecycle.md +18 -11
- package/src/knowledge/split-body-strategy.md +0 -3
- package/src/knowledge/tool-resolution.md +4 -2
- package/src/knowledge/version-paths.md +101 -15
- package/src/module-help.csv +3 -3
- package/src/module.yaml +10 -0
- package/src/shared/data/ast-grep-recipes.yaml +608 -0
- package/src/shared/data/ide-context-files.json +30 -0
- package/src/shared/health-check.md +31 -59
- package/src/shared/references/description-guard-protocol.md +12 -9
- package/src/shared/references/feasibility-report-schema.md +19 -13
- package/src/shared/references/headless-gate-convention.md +55 -3
- package/src/shared/references/output-contract-schema.md +19 -8
- package/src/shared/references/pipeline-contracts.md +94 -29
- package/src/shared/references/tessl-review.md +77 -0
- package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
- package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
- package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
- package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
- package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
- package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
- package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
- package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
- package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
- package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
- package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
- package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
- package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +233 -64
- package/src/shared/scripts/skf-build-change-manifest.py +2753 -28
- package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
- package/src/shared/scripts/skf-check-unit-records.py +184 -0
- package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
- package/src/shared/scripts/skf-classify-changed-files.py +772 -0
- package/src/shared/scripts/skf-comention-pairs.py +1333 -123
- package/src/shared/scripts/skf-compare-file-hashes.py +267 -105
- package/src/shared/scripts/skf-count-imports.py +1867 -0
- package/src/shared/scripts/skf-count-tokens.py +1 -1
- package/src/shared/scripts/skf-derive-assembly-shape.py +21 -3
- package/src/shared/scripts/skf-description-guard.py +99 -11
- package/src/shared/scripts/skf-detect-docs.py +694 -2
- package/src/shared/scripts/skf-detect-language.py +486 -107
- package/src/shared/scripts/skf-detect-registry.py +887 -0
- package/src/shared/scripts/skf-detect-scripts-assets.py +197 -55
- package/src/shared/scripts/skf-detect-tools.py +209 -27
- package/src/shared/scripts/skf-detect-workspaces.py +506 -30
- package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
- package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
- package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
- package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
- package/src/shared/scripts/skf-extract-public-api.py +5247 -96
- package/src/shared/scripts/skf-extraction-inventory.py +1280 -0
- package/src/shared/scripts/skf-extraction-snapshot.py +664 -0
- package/src/shared/scripts/skf-fetch-temporal.py +674 -0
- package/src/shared/scripts/skf-find-cycles.py +97 -29
- package/src/shared/scripts/skf-find-test-report.py +476 -0
- package/src/shared/scripts/skf-forge-tier-rw.py +469 -205
- package/src/shared/scripts/skf-github-fetch.py +365 -0
- package/src/shared/scripts/skf-github-probe.py +700 -0
- package/src/shared/scripts/skf-hash-content.py +180 -20
- package/src/shared/scripts/skf-language-corpora.py +1 -1
- package/src/shared/scripts/skf-load-provenance.py +161 -27
- package/src/shared/scripts/skf-manifest-ops.py +1 -1
- package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
- package/src/shared/scripts/skf-merge-doc-urls.py +143 -20
- package/src/shared/scripts/skf-names-present.py +221 -0
- package/src/shared/scripts/skf-pair-intersect.py +119 -28
- package/src/shared/scripts/skf-parse-gaps.py +1384 -0
- package/src/shared/scripts/skf-preapply.py +1 -1
- package/src/shared/scripts/skf-preflight.py +176 -28
- package/src/shared/scripts/skf-promote-staged.py +430 -0
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +64 -8
- package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
- package/src/shared/scripts/skf-quick-batch.py +774 -0
- package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
- package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
- package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
- package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
- package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
- package/src/shared/scripts/skf-resolve-authoritative-files.py +155 -14
- package/src/shared/scripts/skf-resolve-package.py +523 -106
- package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
- package/src/shared/scripts/skf-run-lock.py +577 -0
- package/src/shared/scripts/skf-scan-manifests.py +998 -57
- package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
- package/src/shared/scripts/skf-severity-classify.py +712 -108
- package/src/shared/scripts/skf-shape-detect.py +298 -204
- package/src/shared/scripts/skf-shard-body.py +9 -8
- package/src/shared/scripts/skf-skill-inventory.py +2412 -89
- package/src/shared/scripts/skf-skills-module.py +472 -0
- package/src/shared/scripts/skf-source-tree.py +1796 -0
- package/src/shared/scripts/skf-structural-diff.py +787 -121
- package/src/shared/scripts/skf-tessl-review.py +706 -0
- package/src/shared/scripts/skf-update-active-symlink.py +120 -67
- package/src/shared/scripts/skf-update-run-state.py +624 -0
- package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
- package/src/shared/scripts/skf-validate-brief-schema.py +23 -2
- package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
- package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
- package/src/shared/scripts/skf-validate-output.py +712 -64
- package/src/shared/scripts/skf-validate-pins.py +1 -1
- package/src/shared/scripts/skf-verify-no-trace.py +116 -10
- package/src/shared/scripts/skf-verify-provenance-completeness.py +3945 -152
- package/src/shared/scripts/skf-write-skill-brief.py +406 -25
- package/src/shared/tool-requirements.yaml +201 -0
- package/src/skf-analyze-source/SKILL.md +41 -36
- package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
- package/src/skf-analyze-source/customize.toml +36 -24
- package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
- package/src/skf-analyze-source/references/continue.md +2 -13
- package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
- package/src/skf-analyze-source/references/generate-briefs.md +121 -55
- package/src/skf-analyze-source/references/headless-contract.md +42 -15
- package/src/skf-analyze-source/references/identify-units.md +96 -110
- package/src/skf-analyze-source/references/init.md +73 -71
- package/src/skf-analyze-source/references/map-and-detect.md +123 -105
- package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
- package/src/skf-analyze-source/references/recommend.md +51 -70
- package/src/skf-analyze-source/references/scan-project.md +60 -84
- package/src/skf-analyze-source/references/scan-root.md +39 -0
- package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
- package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
- package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
- package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
- package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
- package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
- package/src/skf-audit-skill/SKILL.md +43 -51
- package/src/skf-audit-skill/assets/drift-report-template.md +28 -3
- package/src/skf-audit-skill/customize.toml +41 -20
- package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
- package/src/skf-audit-skill/references/doc-drift.md +141 -0
- package/src/skf-audit-skill/references/headless-contract.md +29 -0
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +149 -210
- package/src/skf-audit-skill/references/re-index.md +103 -110
- package/src/skf-audit-skill/references/report.md +96 -33
- package/src/skf-audit-skill/references/semantic-diff.md +34 -25
- package/src/skf-audit-skill/references/severity-classify.md +79 -65
- package/src/skf-audit-skill/references/severity-rules.md +15 -6
- package/src/skf-audit-skill/references/structural-diff.md +80 -90
- package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
- package/src/skf-audit-skill/scripts/render-drift-tables.py +712 -0
- package/src/skf-brief-skill/SKILL.md +37 -26
- package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +18 -25
- package/src/skf-brief-skill/assets/skill-brief-schema.md +30 -23
- package/src/skf-brief-skill/customize.toml +48 -20
- package/src/skf-brief-skill/references/analyze-target.md +144 -127
- package/src/skf-brief-skill/references/confirm-brief.md +11 -9
- package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
- package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
- package/src/skf-brief-skill/references/gather-intent.md +87 -182
- package/src/skf-brief-skill/references/headless-args.md +84 -16
- package/src/skf-brief-skill/references/health-check.md +1 -1
- package/src/skf-brief-skill/references/invocation-contract.md +18 -13
- package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
- package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
- package/src/skf-brief-skill/references/scope-definition.md +123 -47
- package/src/skf-brief-skill/references/step-auto-brief.md +26 -100
- package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
- package/src/skf-brief-skill/references/version-resolution.md +7 -10
- package/src/skf-brief-skill/references/write-brief.md +66 -57
- package/src/skf-campaign/SKILL.md +31 -34
- package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
- package/src/skf-campaign/customize.toml +47 -16
- package/src/skf-campaign/references/campaign-contracts.md +39 -29
- package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
- package/src/skf-campaign/references/health-check.md +7 -16
- package/src/skf-campaign/references/step-01-setup.md +46 -84
- package/src/skf-campaign/references/step-02-strategy.md +31 -30
- package/src/skf-campaign/references/step-03-pins.md +26 -18
- package/src/skf-campaign/references/step-04-provenance.md +24 -23
- package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
- package/src/skf-campaign/references/step-06-batch.md +41 -34
- package/src/skf-campaign/references/step-07-capstone.md +40 -25
- package/src/skf-campaign/references/step-08-verify.md +28 -29
- package/src/skf-campaign/references/step-09-refine.md +29 -30
- package/src/skf-campaign/references/step-10-export.md +48 -36
- package/src/skf-campaign/references/step-11-maintenance.md +31 -34
- package/src/skf-campaign/references/step-resume.md +37 -68
- package/src/skf-campaign/scripts/campaign-deps.py +57 -18
- package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
- package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
- package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
- package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
- package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
- package/src/skf-campaign/scripts/campaign-report.py +204 -37
- package/src/skf-campaign/scripts/campaign-state.py +1128 -0
- package/src/skf-campaign/scripts/campaign-status.py +10 -74
- package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
- package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
- package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
- package/src/skf-campaign/templates/campaign-report-template.md +5 -1
- package/src/skf-campaign/templates/kickoff-template.md +0 -14
- package/src/skf-create-skill/SKILL.md +27 -20
- package/src/skf-create-skill/assets/compile-assembly-rules.md +11 -8
- package/src/skf-create-skill/assets/skill-sections.md +21 -11
- package/src/skf-create-skill/customize.toml +20 -8
- package/src/skf-create-skill/references/authoritative-files-protocol.md +36 -58
- package/src/skf-create-skill/references/batch-mode.md +81 -0
- package/src/skf-create-skill/references/ccc-index-check.md +27 -0
- package/src/skf-create-skill/references/compile.md +119 -70
- package/src/skf-create-skill/references/component-extraction.md +113 -91
- package/src/skf-create-skill/references/enrich.md +41 -23
- package/src/skf-create-skill/references/entry-points-by-hand.md +37 -0
- package/src/skf-create-skill/references/extract.md +204 -162
- package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +11 -11
- package/src/skf-create-skill/references/extraction-patterns.md +588 -201
- package/src/skf-create-skill/references/generate-artifacts.md +145 -93
- package/src/skf-create-skill/references/health-check.md +1 -1
- package/src/skf-create-skill/references/load-brief.md +53 -36
- package/src/skf-create-skill/references/relabel-rule.md +12 -0
- package/src/skf-create-skill/references/report.md +37 -57
- package/src/skf-create-skill/references/source-resolution-protocols.md +58 -169
- package/src/skf-create-skill/references/step-auto-shard.md +20 -93
- package/src/skf-create-skill/references/step-doc-rot.md +44 -60
- package/src/skf-create-skill/references/step-doc-sources.md +28 -19
- package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
- package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
- package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
- package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
- package/src/skf-create-skill/references/validate.md +173 -99
- package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
- package/src/skf-create-stack-skill/SKILL.md +29 -56
- package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
- package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
- package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
- package/src/skf-create-stack-skill/customize.toml +23 -12
- package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
- package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
- package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
- package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
- package/src/skf-create-stack-skill/references/generate-output.md +138 -128
- package/src/skf-create-stack-skill/references/init.md +90 -32
- package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
- package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
- package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
- package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
- package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
- package/src/skf-create-stack-skill/references/report.md +39 -33
- package/src/skf-create-stack-skill/references/validate.md +20 -39
- package/src/skf-drop-skill/SKILL.md +28 -51
- package/src/skf-drop-skill/customize.toml +44 -30
- package/src/skf-drop-skill/references/execute.md +83 -146
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +64 -0
- package/src/skf-drop-skill/references/report.md +46 -30
- package/src/skf-drop-skill/references/select.md +122 -124
- package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
- package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
- package/src/skf-export-skill/SKILL.md +40 -62
- package/src/skf-export-skill/assets/managed-section-format.md +14 -50
- package/src/skf-export-skill/assets/snippet-format.md +3 -1
- package/src/skf-export-skill/customize.toml +42 -13
- package/src/skf-export-skill/references/generate-snippet.md +54 -46
- package/src/skf-export-skill/references/health-check.md +2 -0
- package/src/skf-export-skill/references/invocation-contract.md +15 -0
- package/src/skf-export-skill/references/load-skill.md +88 -62
- package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
- package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
- package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
- package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
- package/src/skf-export-skill/references/package.md +15 -73
- package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
- package/src/skf-export-skill/references/result-envelope.md +31 -16
- package/src/skf-export-skill/references/summary.md +41 -17
- package/src/skf-export-skill/references/token-report.md +4 -12
- package/src/skf-export-skill/references/update-context.md +118 -237
- package/src/skf-forger/SKILL.md +61 -37
- package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
- package/src/skf-forger/customize.toml +43 -0
- package/src/skf-forger/references/pipeline-mode.md +73 -18
- package/src/skf-forger/scripts/forge-status.py +299 -0
- package/src/skf-forger/scripts/parse-pipeline.py +322 -54
- package/src/skf-forger/scripts/pipeline-gate.py +428 -0
- package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
- package/src/skf-quick-skill/SKILL.md +54 -53
- package/src/skf-quick-skill/assets/skill-template.md +4 -2
- package/src/skf-quick-skill/customize.toml +51 -20
- package/src/skf-quick-skill/references/batch-mode.md +74 -21
- package/src/skf-quick-skill/references/compile.md +30 -31
- package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
- package/src/skf-quick-skill/references/finalize.md +69 -18
- package/src/skf-quick-skill/references/halt-contract.md +41 -29
- package/src/skf-quick-skill/references/health-check.md +5 -3
- package/src/skf-quick-skill/references/quick-extract.md +114 -45
- package/src/skf-quick-skill/references/registry-resolution.md +9 -64
- package/src/skf-quick-skill/references/resolve-target.md +116 -66
- package/src/skf-quick-skill/references/write-and-validate.md +79 -40
- package/src/skf-refine-architecture/SKILL.md +43 -53
- package/src/skf-refine-architecture/customize.toml +49 -20
- package/src/skf-refine-architecture/references/compile.md +97 -54
- package/src/skf-refine-architecture/references/exit-codes.md +30 -0
- package/src/skf-refine-architecture/references/finding-storage.md +21 -0
- package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
- package/src/skf-refine-architecture/references/improvements.md +14 -20
- package/src/skf-refine-architecture/references/init.md +98 -57
- package/src/skf-refine-architecture/references/issue-detection.md +32 -39
- package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
- package/src/skf-refine-architecture/references/report.md +65 -54
- package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
- package/src/skf-rename-skill/SKILL.md +19 -54
- package/src/skf-rename-skill/customize.toml +32 -26
- package/src/skf-rename-skill/references/execute.md +230 -184
- package/src/skf-rename-skill/references/exit-codes.md +7 -7
- package/src/skf-rename-skill/references/health-check.md +1 -7
- package/src/skf-rename-skill/references/invocation-contract.md +25 -0
- package/src/skf-rename-skill/references/report.md +63 -52
- package/src/skf-rename-skill/references/select.md +197 -115
- package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
- package/src/skf-setup/SKILL.md +26 -26
- package/src/skf-setup/customize.toml +14 -10
- package/src/skf-setup/references/auto-index.md +50 -63
- package/src/skf-setup/references/ccc-index.md +29 -90
- package/src/skf-setup/references/detect-and-tier.md +58 -67
- package/src/skf-setup/references/health-check.md +2 -0
- package/src/skf-setup/references/invocation-contract.md +14 -0
- package/src/skf-setup/references/report.md +43 -166
- package/src/skf-setup/references/tier-rules.md +5 -5
- package/src/skf-setup/references/write-config.md +34 -57
- package/src/skf-test-skill/SKILL.md +17 -57
- package/src/skf-test-skill/assets/output-section-formats.md +33 -34
- package/src/skf-test-skill/customize.toml +27 -13
- package/src/skf-test-skill/references/coherence-check.md +134 -62
- package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
- package/src/skf-test-skill/references/coverage-check.md +209 -264
- package/src/skf-test-skill/references/detect-mode.md +5 -5
- package/src/skf-test-skill/references/external-validators.md +94 -82
- package/src/skf-test-skill/references/health-check.md +6 -7
- package/src/skf-test-skill/references/init.md +164 -91
- package/src/skf-test-skill/references/invocation-contract.md +66 -0
- package/src/skf-test-skill/references/migration-section-rules.md +5 -2
- package/src/skf-test-skill/references/report.md +164 -142
- package/src/skf-test-skill/references/score.md +80 -107
- package/src/skf-test-skill/references/scoring-rules.md +42 -155
- package/src/skf-test-skill/references/source-access-protocol.md +18 -94
- package/src/skf-test-skill/references/step-hard-gate.md +51 -33
- package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
- package/src/skf-test-skill/scripts/build-result-context.py +380 -0
- package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
- package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
- package/src/skf-test-skill/scripts/compute-score.py +223 -45
- package/src/skf-test-skill/scripts/gap-ledger.py +1666 -0
- package/src/skf-test-skill/scripts/hard-gate.py +203 -0
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +1198 -0
- package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
- package/src/skf-test-skill/scripts/reconcile-coverage.py +547 -79
- package/src/skf-test-skill/scripts/score-signatures.py +479 -0
- package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
- package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +133 -56
- package/src/skf-test-skill/templates/test-report-template.md +21 -15
- package/src/skf-update-skill/SKILL.md +38 -25
- package/src/skf-update-skill/customize.toml +27 -12
- package/src/skf-update-skill/references/detect-changes.md +253 -223
- package/src/skf-update-skill/references/gap-driven.md +336 -0
- package/src/skf-update-skill/references/health-check.md +31 -5
- package/src/skf-update-skill/references/init.md +254 -84
- package/src/skf-update-skill/references/invocation-contract.md +12 -0
- package/src/skf-update-skill/references/manual-section-rules.md +6 -23
- package/src/skf-update-skill/references/merge.md +182 -49
- package/src/skf-update-skill/references/re-extract.md +53 -262
- package/src/skf-update-skill/references/report.md +102 -35
- package/src/skf-update-skill/references/write.md +357 -155
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +183 -46
- package/src/skf-verify-stack/SKILL.md +23 -42
- package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
- package/src/skf-verify-stack/customize.toml +55 -22
- package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
- package/src/skf-verify-stack/references/coverage.md +67 -64
- package/src/skf-verify-stack/references/exit-codes.md +21 -8
- package/src/skf-verify-stack/references/init.md +107 -62
- package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
- package/src/skf-verify-stack/references/integrations.md +106 -65
- package/src/skf-verify-stack/references/report.md +59 -42
- package/src/skf-verify-stack/references/requirements.md +32 -9
- package/src/skf-verify-stack/references/synthesize.md +53 -39
- package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
- package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
- package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
- package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
- package/tools/cli/commands/install.js +11 -4
- package/tools/cli/commands/status.js +15 -3
- package/tools/cli/commands/update.js +11 -4
- package/tools/cli/lib/tool-check.js +320 -0
- package/tools/cli/lib/version-check.js +22 -9
- package/tools/cli/skf-cli.js +4 -1
- package/.gitattributes +0 -16
- package/.nvmrc +0 -1
- package/docs/_internal/RELEASING.md +0 -679
- package/release-audits/v1.0.0-launch-audit.md +0 -728
- package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
- package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
- package/src/skf-campaign/manifest.yaml +0 -15
- package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
- package/src/skf-create-skill/references/ecosystem-check.md +0 -74
- package/src/skf-drop-skill/references/headless-contract.md +0 -18
- package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
- package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
- package/src/skf-update-skill/references/validate.md +0 -124
- package/tools/validate-docs-drift.js +0 -233
- package/tools/validate-docs-links.js +0 -665
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# /// script
|
|
2
|
-
# requires-python = ">=3.
|
|
2
|
+
# requires-python = ">=3.11"
|
|
3
3
|
# dependencies = []
|
|
4
4
|
# ///
|
|
5
5
|
"""SKF Build Change Manifest — aggregate category A/B/C/D detection results.
|
|
@@ -26,15 +26,131 @@ schema, plus the provenance map for the denominator.
|
|
|
26
26
|
|
|
27
27
|
Subcommands:
|
|
28
28
|
|
|
29
|
-
build [--input <file>] [--
|
|
29
|
+
build [--input <file>] [--category-a <file>] [--category-b-diff <file>]
|
|
30
|
+
[--category-c <file>] [--ccc-pairs <file>]
|
|
31
|
+
[--file-compare <file> --provenance-map <file>] [--new-files <file>]
|
|
30
32
|
Reads category JSON from stdin or `--input <file>`, emits the
|
|
31
|
-
unified manifest envelope.
|
|
33
|
+
unified manifest envelope (see "Manifest" below).
|
|
34
|
+
|
|
35
|
+
build --doc-hashes <file>
|
|
36
|
+
A docs-only skill's manifest, from skf-detect-docs.py
|
|
37
|
+
compare-hashes' output and nothing else (see "Docs-only" below).
|
|
32
38
|
|
|
33
39
|
deletion-ratio --provenance-map <file> [--input <file>]
|
|
40
|
+
[--category-a <file>] [--category-b-diff <file>]
|
|
41
|
+
[--category-c <file>] [--ccc-pairs <file>]
|
|
34
42
|
Same category JSON input. Reads provenance entries[] from
|
|
35
43
|
<file>, computes the §2.2 trigger envelope. Auto-skips when
|
|
36
44
|
degraded_mode or update_mode==gap-driven is set in the input.
|
|
37
45
|
|
|
46
|
+
rename-candidates --category-a <file> --provenance-map <file>
|
|
47
|
+
--tier Quick|Forge|Forge+|Deep [--category-b-diff <file>]
|
|
48
|
+
[--extraction <file>] [--export-details <file>]
|
|
49
|
+
[--source-root <dir> [--sizes-commit <commit>]] [-o <file>]
|
|
50
|
+
Category C by fixed rules (see "Rename candidates" below): pairs
|
|
51
|
+
Category A's deleted files with its added ones, and the diff's
|
|
52
|
+
removed exports with its added ones and the added files' exports.
|
|
53
|
+
Prints {"category_c": {...}, "evidence": {...}, "unpaired": {...}}.
|
|
54
|
+
|
|
55
|
+
apply --update-type incremental|gap-driven|full --skill-name <name>
|
|
56
|
+
--generation-date <iso> --confidence-tier <tier>
|
|
57
|
+
--manual-sections-preserved <n> -o <file> [inputs...]
|
|
58
|
+
[--not-public-out <file>] [--metadata-in <file> --lists-out <file>]
|
|
59
|
+
The provenance map an update writes (see "Apply" below): the old
|
|
60
|
+
map with this run's changes, written to -o through a temporary file
|
|
61
|
+
and a rename, the summary's not_public list to --not-public-out and
|
|
62
|
+
metadata.json's exports[], scripts[] and assets[] to --lists-out
|
|
63
|
+
when given. Prints a summary with the warnings for a person.
|
|
64
|
+
|
|
65
|
+
metadata --metadata <metadata.json> --lists <file> --stats <file>
|
|
66
|
+
--version <version> --generation-date <iso>
|
|
67
|
+
[--source-commit <sha>] [--source-ref <ref>]
|
|
68
|
+
The skill's metadata.json an update leaves (see "Metadata" below),
|
|
69
|
+
written over --metadata through a temporary file and a rename.
|
|
70
|
+
Prints {"status": "written", "metadata": <path>, "version",
|
|
71
|
+
"exports": N, "scripts": N, "assets": N}.
|
|
72
|
+
|
|
73
|
+
records [--extraction <file>] [--export-details <file>]
|
|
74
|
+
[--files-from <file>] [--patches <dir>] -o <file>
|
|
75
|
+
Step 3's re-extraction records (see "Records" below), written to -o
|
|
76
|
+
through a temporary file and a rename. Prints {"status": "written",
|
|
77
|
+
"output", "files_extracted", "exports_extracted",
|
|
78
|
+
"confidence_breakdown": {"T1", "T1-low", "T2"}, "marked_not_public"
|
|
79
|
+
(the exports marked public: false), "warnings"}.
|
|
80
|
+
|
|
81
|
+
gap-records --manifest <file> --provenance-map <file> [--source-root <dir>]
|
|
82
|
+
[--drift-status ok|skipped|overridden] [--judgments <file>]
|
|
83
|
+
[--remediation-records <file>] [--evidence <file> --tier <tier>]
|
|
84
|
+
(--plan [--files-out <file>] | -o <file>)
|
|
85
|
+
A gap-driven repair's spot-checks, routing and verification records
|
|
86
|
+
(see "Gap records" below): --plan prints what to answer and which
|
|
87
|
+
files targeted re-extraction scans; without it, the records are
|
|
88
|
+
written to -o through a temporary file and a rename.
|
|
89
|
+
|
|
90
|
+
baseline-gaps --provenance-map <file> --extraction <file> --files <file>
|
|
91
|
+
--source-root <dir> -o <file>
|
|
92
|
+
Category B's baseline gaps (see "Baseline gaps" below): each map
|
|
93
|
+
entry of the files --files lists that the runner did not report and
|
|
94
|
+
its file still declares, written to -o through a temporary file and
|
|
95
|
+
a rename. Prints {"status": "written", "output": <-o>, "gaps": N,
|
|
96
|
+
"unchecked": N}.
|
|
97
|
+
|
|
98
|
+
Helper files in place of typed slices (update-skill detect-changes §2.1):
|
|
99
|
+
|
|
100
|
+
--category-a <file> the output of skf-classify-changed-files.py
|
|
101
|
+
classify: its category_a is the Category A
|
|
102
|
+
slice, and each of its moved_files (a
|
|
103
|
+
same-content move) is a category_c.renamed_files
|
|
104
|
+
item, ahead of the input's own
|
|
105
|
+
--category-b-diff <file> the output of skf-structural-diff.py over the
|
|
106
|
+
modified files: the Category B slice, mapped
|
|
107
|
+
removed[] -> deleted_exports {name, file, old_line}
|
|
108
|
+
added[] -> new_exports {name, file, line}
|
|
109
|
+
changed[] -> modified_exports {name, file,
|
|
110
|
+
old_line, new_line} for an export
|
|
111
|
+
with a field other than `line`, or
|
|
112
|
+
one signature_unverified[] lists
|
|
113
|
+
(its lines null when no field
|
|
114
|
+
changed); moved_exports for an
|
|
115
|
+
export whose only field is `line`
|
|
116
|
+
moved[] -> moved_exports {name, file:
|
|
117
|
+
current_file, old_file:
|
|
118
|
+
previous_file (only when it
|
|
119
|
+
differs), old_line:
|
|
120
|
+
previous_line, new_line: line},
|
|
121
|
+
unless it is a modified export,
|
|
122
|
+
which then takes old_file and
|
|
123
|
+
old_line: previous_line
|
|
124
|
+
label_changes[] is not drift, and a name
|
|
125
|
+
ambiguous_names[] lists stays removed and added
|
|
126
|
+
--category-c <file> the output of rename-candidates: its category_c
|
|
127
|
+
is the Category C slice
|
|
128
|
+
--ccc-pairs <file> the pairs the update's CCC check added to what
|
|
129
|
+
rename-candidates left unpaired,
|
|
130
|
+
{"renamed_files": [...], "renamed_exports":
|
|
131
|
+
[...]}, appended to the Category C slice
|
|
132
|
+
--file-compare <file> build only: the output of skf-hash-content.py
|
|
133
|
+
compare, with --provenance-map (the map it
|
|
134
|
+
compared): each MODIFIED_FILE and DELETED_FILE
|
|
135
|
+
row goes to the {type}s_modified or
|
|
136
|
+
{type}s_deleted list of category_d, by the
|
|
137
|
+
file_type of the file_entries[] row of its
|
|
138
|
+
source_file (script, asset or doc; a row with
|
|
139
|
+
none takes the type its file_name's first
|
|
140
|
+
folder names, scripts/, assets/ or docs/, else
|
|
141
|
+
asset)
|
|
142
|
+
--new-files <file> build only: the output of skf-new-file-diff.py:
|
|
143
|
+
each new_files[] item goes to scripts_added or
|
|
144
|
+
assets_added by its kind
|
|
145
|
+
Either Category D file makes the two files the category_d slice. A given
|
|
146
|
+
file replaces the input's own slice. Either way, every category_c
|
|
147
|
+
rename then takes its pair out of the lists it was found in: a renamed
|
|
148
|
+
file's old_path out of category_a.deleted and new_path out of
|
|
149
|
+
category_a.added; a renamed export's old_name out of
|
|
150
|
+
category_b.deleted_exports (the one in its `old_file` when the item gives
|
|
151
|
+
one, else preferably the one in its `file`) and its new_name out of
|
|
152
|
+
category_b.new_exports (preferably the one in its `file`).
|
|
153
|
+
|
|
38
154
|
Input JSON shape (object on stdin or in --input file):
|
|
39
155
|
|
|
40
156
|
{
|
|
@@ -44,10 +160,10 @@ Input JSON shape (object on stdin or in --input file):
|
|
|
44
160
|
"deleted": ["path3", ...]
|
|
45
161
|
},
|
|
46
162
|
"category_b": {
|
|
47
|
-
"modified_exports": [{name, file, old_line, new_line}, ...],
|
|
163
|
+
"modified_exports": [{name, file, old_line, new_line, old_file?}, ...],
|
|
48
164
|
"new_exports": [{name, file, line}, ...],
|
|
49
165
|
"deleted_exports": [{name, file, old_line}, ...],
|
|
50
|
-
"moved_exports": [{name, file, old_line, new_line}, ...]
|
|
166
|
+
"moved_exports": [{name, file, old_line, new_line, old_file?}, ...]
|
|
51
167
|
},
|
|
52
168
|
"category_c": {
|
|
53
169
|
"renamed_files": [{old_path, new_path}, ...],
|
|
@@ -55,27 +171,417 @@ Input JSON shape (object on stdin or in --input file):
|
|
|
55
171
|
},
|
|
56
172
|
"category_d": {
|
|
57
173
|
"scripts_modified": ["path", ...], "scripts_added": [...], "scripts_deleted": [...],
|
|
58
|
-
"assets_modified": [...], "assets_added": [...], "assets_deleted": [...]
|
|
174
|
+
"assets_modified": [...], "assets_added": [...], "assets_deleted": [...],
|
|
175
|
+
"docs_modified": [...], "docs_deleted": [...]
|
|
59
176
|
},
|
|
60
177
|
"degraded_mode": false,
|
|
61
178
|
"update_mode": "normal" | "gap-driven" // optional; influences deletion-ratio only
|
|
62
179
|
}
|
|
63
180
|
|
|
181
|
+
Manifest (build's output):
|
|
182
|
+
|
|
183
|
+
{"no_changes": <every count is 0>, "degraded_mode": <bool>,
|
|
184
|
+
"counts": {files_changed, files_added, files_deleted, files_moved,
|
|
185
|
+
exports_modified, exports_new, exports_deleted,
|
|
186
|
+
exports_renamed, exports_moved, and one count per
|
|
187
|
+
category_d list},
|
|
188
|
+
"total_export_changes": N, "per_file": [...],
|
|
189
|
+
"category_d": {the eight category_d lists, each path once}}
|
|
190
|
+
|
|
191
|
+
Each per_file item's exports_affected holds {name, change_type,
|
|
192
|
+
old_line, new_line}, and old_file for an export that moved across files
|
|
193
|
+
(a MOVED_EXPORT, or a MODIFIED_EXPORT that also changed).
|
|
194
|
+
|
|
195
|
+
A tracked document (a doc row of file_entries[]) has no other detector:
|
|
196
|
+
update-skill's Category A never lists it, so a document-only change still
|
|
197
|
+
counts, and the update records its new hash.
|
|
198
|
+
|
|
64
199
|
Every category and sub-key is optional — missing keys default to
|
|
65
200
|
empty lists. This lets gap-driven runs (which produce no Category D
|
|
66
201
|
results) and degraded runs (which skip Category B) feed the same
|
|
67
202
|
script without sentinel values.
|
|
68
203
|
|
|
204
|
+
Docs-only (update-skill detect-changes §1): build --doc-hashes reads the
|
|
205
|
+
compare-hashes output and prints {"mode": "docs-only", "no_changes":
|
|
206
|
+
<changed[] is empty>, "changed_urls": [each changed[].url],
|
|
207
|
+
"fetch_failed": [each fetch_failed[].url], "counts": {"docs_changed",
|
|
208
|
+
"docs_fetch_failed"}}. It takes no other input.
|
|
209
|
+
|
|
210
|
+
Rename candidates (update-skill detect-changes Category C):
|
|
211
|
+
|
|
212
|
+
Content similarity above 80% (fixed, not configurable) is a rename. A
|
|
213
|
+
file moved with its content unchanged is Category A's moved_files and is
|
|
214
|
+
left out here. For the rest:
|
|
215
|
+
files, Quick tier the added file's size is within 20% of the deleted
|
|
216
|
+
file's (the deleted size is `git cat-file -s` of
|
|
217
|
+
<commit>:<path> under --source-root; with no
|
|
218
|
+
--sizes-commit, a local source whose file is gone,
|
|
219
|
+
the size test is skipped) and the export names
|
|
220
|
+
overlap above 70% (shared names over all names of
|
|
221
|
+
the two files)
|
|
222
|
+
files, Forge+ above 80% of the exports match: an export of the
|
|
223
|
+
added file with the deleted file's export name and
|
|
224
|
+
signature (export_type, params and return_type),
|
|
225
|
+
over the larger file's export count
|
|
226
|
+
exports, Quick the names are above 80% alike (difflib ratio) and
|
|
227
|
+
their export types agree
|
|
228
|
+
exports, Forge+ the signatures are equal (export_type, params and
|
|
229
|
+
return_type, a parameter list or a return type
|
|
230
|
+
given) and the names differ
|
|
231
|
+
A deleted file's exports come from the provenance map, an added file's
|
|
232
|
+
from --extraction and --export-details. A pair is kept only when it is
|
|
233
|
+
each side's best match: the highest similarity wins, and a tie between
|
|
234
|
+
two candidates pairs neither. category_c.renamed_files holds {old_path,
|
|
235
|
+
new_path}; renamed_exports holds {old_name, new_name, file, old_file?},
|
|
236
|
+
with old_file only when it differs from file. evidence lists each pair
|
|
237
|
+
with its rule and similarity, and unpaired the deleted and added files
|
|
238
|
+
and the removed and added exports left over, for the judgment step.
|
|
239
|
+
|
|
240
|
+
Apply (update-skill write.md §3):
|
|
241
|
+
|
|
242
|
+
Reads the provenance map the update started from (--provenance-map;
|
|
243
|
+
without it, a degraded run's new map) and prints nothing but a summary:
|
|
244
|
+
the new map goes to -o. Every entry the run leaves alone keeps its exact
|
|
245
|
+
value; every entry it adds or rewrites carries signature_source, set
|
|
246
|
+
from the tool that produced the signature: ast-grep gives T1 (with
|
|
247
|
+
confidence T1 and the recipe's ast_node_type), a signature read by eye
|
|
248
|
+
(source-read) gives T1-low (confidence T1-low, ast_node_type null).
|
|
249
|
+
A rewritten entry keeps any key the update does not set (notes,
|
|
250
|
+
deprecated). New entries take source_library from the map's entries,
|
|
251
|
+
else --skill-name.
|
|
252
|
+
|
|
253
|
+
incremental (normal mode) reads --manifest (build's output), the renamed
|
|
254
|
+
exports of --category-c and --ccc-pairs (or of --input, the category
|
|
255
|
+
JSON's category_c) and the fresh
|
|
256
|
+
records: --reextract-records ({"mode": "normal", "files": [{file_path,
|
|
257
|
+
exports: [...]}]}, the records `records` wrote), --extraction (the
|
|
258
|
+
recipe runner's exports) and --export-details. A fresh record of an
|
|
259
|
+
export takes its line, export_type and ast labels from the runner, then
|
|
260
|
+
the details, then step 3's record; its params and return_type from the
|
|
261
|
+
details, then step 3's record (the runner's in the map's typed form).
|
|
262
|
+
Per file of the manifest:
|
|
263
|
+
MODIFIED NEW_EXPORT adds an entry, MODIFIED_EXPORT rewrites the
|
|
264
|
+
export's fields (file included) from its fresh record,
|
|
265
|
+
MOVED_EXPORT moves the entry to the file and the fresh
|
|
266
|
+
record's line, else new_line, DELETED_EXPORT removes it;
|
|
267
|
+
a change with an old_file finds its entry in that file
|
|
268
|
+
ADDED every fresh record of the file becomes an entry
|
|
269
|
+
DELETED every entry of the file is removed
|
|
270
|
+
MOVED the old path's entries are replaced by the new path's
|
|
271
|
+
fresh records, each keeping the keys of the entry of its
|
|
272
|
+
name it replaces
|
|
273
|
+
and each renamed export takes its new name and fresh fields. A
|
|
274
|
+
MOVED_EXPORT whose fresh record is the runner's (ast-grep) also takes
|
|
275
|
+
its confidence, extraction_method and ast_node_type (the entry's when
|
|
276
|
+
the record's is null), and its signature_source when the record holds
|
|
277
|
+
each of params and return_type the entry holds (else the entry's),
|
|
278
|
+
never its export_type, params or return_type; one read by eye moves
|
|
279
|
+
the line and file only.
|
|
280
|
+
|
|
281
|
+
Public-api skills (incremental only): a NEW_EXPORT or MODIFIED_EXPORT
|
|
282
|
+
the map does not hold, an ADDED file's record and a MOVED file's record
|
|
283
|
+
whose name no old entry carries become entries only when on the public
|
|
284
|
+
surface: the record's `public` mark (see "Records"), else, for a record
|
|
285
|
+
--reextract-records does not mark, the same rule over --extraction (its
|
|
286
|
+
scope.type). A name whose entry this run removes (a DELETED_EXPORT, a
|
|
287
|
+
DELETED file's or a MOVED file's old path's entry) is added whatever
|
|
288
|
+
its mark. Each one left out is listed in the summary's not_public,
|
|
289
|
+
{name, file}, with one warning for them all; an entry the map holds is
|
|
290
|
+
updated whatever its mark, and no entry is removed for it. A public-api
|
|
291
|
+
--extraction the rule cannot read (no entry_point_diff, an incomplete
|
|
292
|
+
run or one with errors, an unresolved entry point) adds every name, with
|
|
293
|
+
one warning. A docs-only
|
|
294
|
+
run's records ({"mode": "docs-only", "changed_urls": [...], "exports":
|
|
295
|
+
[{name, type, params, return_type, url}]}) replace the entries of each
|
|
296
|
+
changed URL (source_file is the URL; confidence and signature_source
|
|
297
|
+
T3).
|
|
298
|
+
|
|
299
|
+
gap-driven reads --reextract-records ({"mode": "gap-driven",
|
|
300
|
+
"verification": [...], "files": [...]}) and --merge-records ({"exports":
|
|
301
|
+
[{export_name, export_type, params, return_type}]}, merge's output for
|
|
302
|
+
an export no fresh record holds). Each verification record names
|
|
303
|
+
export_name, gap_category, severity, verification, in_map (the map holds
|
|
304
|
+
an entry of that name), map_entry ({source_file, source_line} of the one
|
|
305
|
+
entry the spot-check took, else null), source_citation, new_location,
|
|
306
|
+
unknown_reason, pinned_definition_lines and reachability:
|
|
307
|
+
verified (map_entry) unchanged
|
|
308
|
+
moved (map_entry) source_line (and source_file) from new_location
|
|
309
|
+
re-extracted the entry from its `files` record (gap-driven.md §4a);
|
|
310
|
+
none when its reachability is internal-unreachable
|
|
311
|
+
verified/moved, not in_map, a NEW_EXPORT or MODIFIED_EXPORT whose
|
|
312
|
+
reachability is not internal-unreachable
|
|
313
|
+
one source-read entry at the citation's line
|
|
314
|
+
(verified) or new_location (moved), at most one
|
|
315
|
+
per name and file, export_type and the
|
|
316
|
+
signature from --merge-records
|
|
317
|
+
unknown, not in_map, a NEW_EXPORT or MODIFIED_EXPORT, a Medium, Low or
|
|
318
|
+
Info severity an entry from --merge-records with no
|
|
319
|
+
source_file or source_line
|
|
320
|
+
unknown, not in_map, any other severity (a missing one included)
|
|
321
|
+
refused: exit 3, nothing written
|
|
322
|
+
missing, unknown in_map, unknown on a MOVED_EXPORT
|
|
323
|
+
unchanged, with a warning for a person
|
|
324
|
+
rescoped the entry removed
|
|
325
|
+
Under the drift override (--drift-head and --drift-pinned) each warning
|
|
326
|
+
names the drift.
|
|
327
|
+
|
|
328
|
+
full (degraded mode) makes every fresh record an entry.
|
|
329
|
+
|
|
330
|
+
Every mode then applies the file changes: --file-compare (skf-hash-
|
|
331
|
+
content.py compare's output) sets the content_hash of a MODIFIED_FILE
|
|
332
|
+
row and removes a DELETED_FILE row; --new-files (skf-new-file-diff.py's
|
|
333
|
+
output) adds a row per new script or asset ({kind}s/<file name>, hashed
|
|
334
|
+
under --source-root, extraction_method file-copy); --promoted-docs (the
|
|
335
|
+
documents step 2 promoted, with their hashes) adds a doc row each
|
|
336
|
+
(docs/authoritative/<path>, promoted-authoritative). A row whose
|
|
337
|
+
source_file the map already holds is never added twice.
|
|
338
|
+
|
|
339
|
+
Last it sets the top-level source_commit and source_ref (when given)
|
|
340
|
+
and the update block, replacing an earlier update's values in place:
|
|
341
|
+
last_update (--generation-date), update_type, test_report_run_id,
|
|
342
|
+
files_changed (the manifest's files, or in gap-driven mode the files of
|
|
343
|
+
the entries this run changed), exports_affected (the manifest's export
|
|
344
|
+
changes, or the verification records), confidence_tier and
|
|
345
|
+
manual_sections_preserved. An older update-history key
|
|
346
|
+
(update_operations, update_metadata) stays as it is.
|
|
347
|
+
|
|
348
|
+
Lists. With --metadata-in (the skill's metadata.json as the update
|
|
349
|
+
found it) and --lists-out, it also writes the lists metadata.json takes,
|
|
350
|
+
{"exports": [...], "scripts": [...], "assets": [...]}. exports: the
|
|
351
|
+
--metadata-in exports[] less each name whose entry this run removed (a
|
|
352
|
+
DELETED_EXPORT, a DELETED file's or a MOVED file's old path's, a
|
|
353
|
+
rescoped one, a changed URL's) or renamed, unless an entry of the new
|
|
354
|
+
map still holds it, then each name it added an entry for or renamed an
|
|
355
|
+
entry to, each once; a renamed name takes its old name's place. A name
|
|
356
|
+
left out as off the public surface (the summary's not_public) has no
|
|
357
|
+
entry, so it is not added. A full update, which rebuilds the map, also
|
|
358
|
+
drops each name the new map does not hold, and a reference app
|
|
359
|
+
(scope_type reference-app) gets []. scripts and assets: one row per
|
|
360
|
+
file_entries[] row of the new map whose file_type is script or asset
|
|
361
|
+
(a row with none takes the type its file_name's folder names), in the
|
|
362
|
+
map's order: the --metadata-in scripts[] or assets[] row of its
|
|
363
|
+
source_file (else of its file_name) as it stands, else a new row {file:
|
|
364
|
+
its file_name, purpose, source_file, confidence (default T1-low)},
|
|
365
|
+
whose purpose skf-detect-scripts-assets.py reads from the file under
|
|
366
|
+
--source-root (its file name when no --source-root is given or the
|
|
367
|
+
file is not there).
|
|
368
|
+
|
|
369
|
+
Metadata (update-skill write.md §2):
|
|
370
|
+
|
|
371
|
+
metadata.json as --metadata holds it, with what the update sets: version
|
|
372
|
+
(--version) and generation_date (--generation-date, YYYY-MM-DDTHH:MM:SSZ);
|
|
373
|
+
source_commit and source_ref when --source-commit or --source-ref is
|
|
374
|
+
given (an empty one sets null), else as they stand; exports, scripts and
|
|
375
|
+
assets from --lists (apply's --lists-out), scripts or assets left out
|
|
376
|
+
when its list is empty; confidence_distribution from --stats (the JSON
|
|
377
|
+
skf-render-metadata-stats.py writes with -o), and stats: the stats
|
|
378
|
+
helper's fields, then each other field --metadata's stats holds (a
|
|
379
|
+
component library's own counts, a reference app's notes, a field a
|
|
380
|
+
gap-driven patch added), the stats helper's fields an earlier run wrote
|
|
381
|
+
that this run's does not (effective_denominator,
|
|
382
|
+
pattern_surfaces_documented) dropped. It removes last_update and
|
|
383
|
+
update_type, which the provenance map's update block records, and keeps
|
|
384
|
+
every other field. It refuses (exit 1, nothing written) a --stats whose
|
|
385
|
+
scripts_count or assets_count is not the length of --lists' scripts or
|
|
386
|
+
assets, a --generation-date of another form, and a file it cannot read.
|
|
387
|
+
|
|
388
|
+
Summary on stdout: {"status": "written", "map": <-o>, "entries":
|
|
389
|
+
{"added", "updated", "removed"}, "not_public": [{name, file}],
|
|
390
|
+
"file_entries": {"added", "updated", "removed"}, "warnings": [...]},
|
|
391
|
+
or {"status": "refused",
|
|
392
|
+
"blocking_unresolved": [{export_name, severity}]} with exit 3.
|
|
393
|
+
|
|
394
|
+
Records (update-skill re-extract.md §4, and gap-driven.md §4a for the files
|
|
395
|
+
it scans):
|
|
396
|
+
|
|
397
|
+
Writes {"mode": "normal", "files": [{file_path, exports: [...]}]}, one
|
|
398
|
+
block per file --files-from names (and per other file an export names),
|
|
399
|
+
so the model never types a record a tool already wrote. Each export the
|
|
400
|
+
recipe runner found (--extraction) is seeded with its name, type,
|
|
401
|
+
signature, location (file:line), confidence T1, extraction_method
|
|
402
|
+
ast-grep, ast_node_type and ast_recipe, return_type and params, each
|
|
403
|
+
parameter in the provenance map's typed form (skf-extraction-
|
|
404
|
+
inventory.py typed_param: `name: type`, `name?: type`, ` = default`).
|
|
405
|
+
Where the runner left params or return_type null, --export-details
|
|
406
|
+
fills them. An export only --export-details holds (one read by eye,
|
|
407
|
+
with an export_type) is seeded from it as T1-low, source-read, with a
|
|
408
|
+
null ast_node_type and ast_recipe.
|
|
409
|
+
|
|
410
|
+
Then each worker patch in --patches (every *.json there, in name order:
|
|
411
|
+
one block {file_path, exports: [{name, ...}]} or a list of them) is
|
|
412
|
+
merged into the export of its name and file: signature, members,
|
|
413
|
+
docstring and qmd_evidence replace the seed's, and params and
|
|
414
|
+
return_type are set only where the seed has none. A patch for an export
|
|
415
|
+
no seed holds is not added, with a warning. confidence_breakdown counts
|
|
416
|
+
the exports by confidence, and T2 the exports with a qmd_evidence.
|
|
417
|
+
|
|
418
|
+
Public surface. When --extraction's scope.type is public-api (in any
|
|
419
|
+
case) and it has an entry_point_diff, each export of a language family
|
|
420
|
+
whose entry_points.by_language status is `barrel` is marked `public`:
|
|
421
|
+
true when its name and file are a pair of the runner's entry_point_diff
|
|
422
|
+
`public` (its name, or the name `local` gives it in that file) or
|
|
423
|
+
`extraction_gaps`, or when a `public` item of its name and family has
|
|
424
|
+
no file; false otherwise. A `public` item with `via` namespace and a
|
|
425
|
+
`members` list (what the submodule passes on, which users reach as
|
|
426
|
+
`pkg.sub.name`, and which counts.exports_public_api counts in its place)
|
|
427
|
+
gives its members' pairs (each member's name, or its `local` one, and
|
|
428
|
+
its file; a member with no file its name) in place of its own; one
|
|
429
|
+
whose `members` is null (a module the runner could not read), or that
|
|
430
|
+
has no `members` key (an older runner's), gives its own pair, as any
|
|
431
|
+
item does.
|
|
432
|
+
The rule is skf-extraction-inventory.py's public_surface, loaded from
|
|
433
|
+
this folder, by which create-skill marks its extraction inventory too.
|
|
434
|
+
In a normal update merge documents no export marked false that the map
|
|
435
|
+
does not hold, and apply adds none; gap-records drops the mark, and a
|
|
436
|
+
full (degraded) apply adds every record. Any other scope type, a family
|
|
437
|
+
with no barrel, and a run with no entry_point_diff, an incomplete one,
|
|
438
|
+
one with errors or one whose entry_points.unresolved is not empty mark
|
|
439
|
+
nothing.
|
|
440
|
+
|
|
441
|
+
Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
442
|
+
|
|
443
|
+
Reads the change manifest skf-parse-gaps.py translate wrote. Its
|
|
444
|
+
STRUCTURAL_FIX and metadata update entries are forwarded to merge as
|
|
445
|
+
they are (`forwarded`); every other entry is export-bearing. A severity
|
|
446
|
+
is blocking unless it is Medium, Low or Info (NON_BLOCKING), compared
|
|
447
|
+
case-insensitively, so a missing one is blocking. In order:
|
|
448
|
+
|
|
449
|
+
1. Drift gate (--drift-status overridden): each export-bearing entry
|
|
450
|
+
but a MOVED_EXPORT needs the tree, with the first reason that fits:
|
|
451
|
+
a DELETED_EXPORT "a public API recount from the tree (rule R1)", a
|
|
452
|
+
provenance_completeness entry "a line from the tree (rule R3)", one
|
|
453
|
+
whose map_match is not-found "a line and a signature from the tree",
|
|
454
|
+
any other "a signature from the tree". Any such entry: {"status":
|
|
455
|
+
"drift-blocked", "drift_blocked": [{gap_id, name, change_category,
|
|
456
|
+
severity, reason}]}, and no spot-check runs.
|
|
457
|
+
2. A DELETED_EXPORT without a `rescope` that holds its `amendment` and
|
|
458
|
+
`exclude`: {"status": "blocked", "rescope_without_amendment": [names]}.
|
|
459
|
+
3. Spot-checks, by skf-verify-provenance-completeness.py's definition-
|
|
460
|
+
lines rules, loaded from this folder, under --source-root:
|
|
461
|
+
DELETED_EXPORT rescoped, no check
|
|
462
|
+
MOVED_EXPORT checked at its map entry when map_match is
|
|
463
|
+
found (unknown otherwise); under the
|
|
464
|
+
override a `moved` becomes unknown with
|
|
465
|
+
unknown_reason drift-override; its
|
|
466
|
+
pinned_definition_lines are the numbers of
|
|
467
|
+
its remediation's "definition line(s) (...)"
|
|
468
|
+
NEW_EXPORT, MODIFIED_EXPORT
|
|
469
|
+
map_match found checked at the map entry (its export_type)
|
|
470
|
+
ambiguous unknown
|
|
471
|
+
not-found checked at its source_citation (no export
|
|
472
|
+
type) when it has one; when that pins no
|
|
473
|
+
line (unknown or missing), or with no
|
|
474
|
+
citation: targeted re-extraction (§4a) for a
|
|
475
|
+
provenance_completeness entry with a source
|
|
476
|
+
root, for a blocking one whatever its
|
|
477
|
+
resolved_paths, and for a missing-export or
|
|
478
|
+
missing-type one with resolved_paths; else
|
|
479
|
+
unknown
|
|
480
|
+
A check gives `verified` (the line is a definition line, or a module
|
|
481
|
+
or package entry whose file exists), `moved` (one definition line,
|
|
482
|
+
not the recorded one: new_location), `missing` (no such file) or
|
|
483
|
+
`unknown` (several definition lines, none the recorded one, or
|
|
484
|
+
none). A file the rules cover no language of
|
|
485
|
+
takes the answered `declaring_line`: the recorded line is verified,
|
|
486
|
+
another is moved, null is unknown. With no verifier every check is
|
|
487
|
+
unknown, with a `provenance: spot-checks not run` warning; a check
|
|
488
|
+
the verifier fails on (a file it cannot read, or any error) is
|
|
489
|
+
unknown, with a `provenance: spot-check failed for {name}: {error}`
|
|
490
|
+
warning.
|
|
491
|
+
--plan stops here: {"status": "planned", "reextract": {"entries":
|
|
492
|
+
[{gap_id, name, severity, resolved_paths}], "files": [the union of
|
|
493
|
+
their resolved_paths]}, "warnings"}, the files also written to
|
|
494
|
+
--files-out as a JSON array.
|
|
495
|
+
4. Targeted re-extraction (--remediation-records, `records`' output
|
|
496
|
+
over those files): each routed entry with resolved_paths takes the
|
|
497
|
+
first export of its name in the blocks of those files, in their
|
|
498
|
+
order: `re-extracted`, new_location its record's location,
|
|
499
|
+
resolution_source remediation-paths, and the record copied into
|
|
500
|
+
`files` as `records` wrote it, less its `public` mark (a normal
|
|
501
|
+
update's), with the answered `docstring` added.
|
|
502
|
+
An entry with no match, or no path to scan, is unresolved, except a
|
|
503
|
+
missing-export or missing-type one that is not blocking, which is
|
|
504
|
+
unknown. Any unresolved entry: {"status": "unresolved",
|
|
505
|
+
"unresolved": [{gap_id, name, severity, remediation_paths,
|
|
506
|
+
rejected_paths, files_scanned, exports_found_in_scan}]}.
|
|
507
|
+
5. The public-reachability gate: every NEW_EXPORT that is verified,
|
|
508
|
+
moved or re-extracted takes the answered `reachability`, `public` or
|
|
509
|
+
`internal-unreachable`. An internal one keeps its outcome in its
|
|
510
|
+
verification record, its record is not in `files`, `counts` tallies
|
|
511
|
+
it under `reclassified` instead of its outcome, and it is listed in
|
|
512
|
+
`reclassified` (merge queues it as a metadata update; apply writes no
|
|
513
|
+
entry for it).
|
|
514
|
+
|
|
515
|
+
Any answer missing in step 3 or 5 (a `docstring` too, for each
|
|
516
|
+
re-extracted entry): {"status": "needs-judgment", "needs_judgment":
|
|
517
|
+
[{gap_id, name, needs: [...], file?, line?}]}. --judgments gives them,
|
|
518
|
+
keyed by gap id (the answers file translate reads; its keys are left).
|
|
519
|
+
Then it writes {"mode": "gap-driven", "files_extracted" (the blocks of
|
|
520
|
+
--remediation-records, 0 when it scanned none), "exports_extracted" (the
|
|
521
|
+
manifest's entries), "confidence_breakdown": {"T1", "T1-low",
|
|
522
|
+
"unlabeled", "T2": 0} (each manifest entry by the extraction_method of
|
|
523
|
+
its re-extracted record or, in the map, of its map entry; a pinned cited
|
|
524
|
+
export the map lacks and the gate passed is T1-low), "verification":
|
|
525
|
+
[one record per export-bearing entry: export_name, gap_category,
|
|
526
|
+
severity, verification, in_map, map_entry, provenance_citation,
|
|
527
|
+
source_citation, new_location, unknown_reason, pinned_definition_lines,
|
|
528
|
+
resolution_source, reachability, export_type (always null: for a cited
|
|
529
|
+
export the map lacks, apply takes it from merge.md's
|
|
530
|
+
merge-records.json)], "files": [...]}, and prints {"status": "written",
|
|
531
|
+
"output", "gap_count", "files_extracted", "exports_extracted",
|
|
532
|
+
"confidence_breakdown", "counts" (per verification outcome, and
|
|
533
|
+
`reclassified`), "reclassified", "forwarded", "targeted_reextraction",
|
|
534
|
+
"warnings"}. When targeted re-extraction ran, `targeted_reextraction`
|
|
535
|
+
is {resolved_count, files_scanned, exports_matched, tier}, also appended
|
|
536
|
+
to --evidence as one JSON line keyed `targeted_reextraction`.
|
|
537
|
+
|
|
538
|
+
Baseline gaps (update-skill detect-changes Category B, step 1):
|
|
539
|
+
|
|
540
|
+
The recipe runner never reports a module, an alias that renames, a
|
|
541
|
+
dunder such as `__version__` or another underscore name, so the diff
|
|
542
|
+
would read each such map entry as a deleted export. baseline-gaps lists,
|
|
543
|
+
for step 2 to read by eye, each entry of a file --files lists (the
|
|
544
|
+
modified files, Category A's modified-files.json) whose name and file
|
|
545
|
+
no export of --extraction (the runner's JSON, or {"exports": []} when
|
|
546
|
+
it could not run) holds, under its name or its re-export target, and
|
|
547
|
+
whose file still declares it. The selection and the declaration rule
|
|
548
|
+
are skf-verify-provenance-completeness.py's `baseline_gaps` (the rule
|
|
549
|
+
audit-skill's extraction snapshot applies), loaded from this folder: a
|
|
550
|
+
verifier that cannot be loaded is an error, never an empty list. A
|
|
551
|
+
name the runner's own `entry_point_diff.extraction_gaps` lists at that
|
|
552
|
+
file is left to that list. A map whose `entries` is missing or null
|
|
553
|
+
has none to check. Writes {"gaps": [{name, file, line, entry: null,
|
|
554
|
+
export_type?}], "unchecked": [{name, file, export_type}]}: a gap's line
|
|
555
|
+
declares the name (line 1 of the module or package so named for a
|
|
556
|
+
`module` or `package` entry), and `unchecked` holds each entry no rule
|
|
557
|
+
can check, to read by eye: its file neither Python nor TS/JS, a file
|
|
558
|
+
the lookup will not read (outside the source root, a test file or a
|
|
559
|
+
minified bundle, a symlink on its path), or a dotted `module` or
|
|
560
|
+
`package` name.
|
|
561
|
+
|
|
69
562
|
Exit codes:
|
|
70
|
-
0
|
|
71
|
-
1
|
|
563
|
+
0 operation succeeded (for gap-records, whatever its `status`)
|
|
564
|
+
1 user error (malformed JSON, bad path, malformed provenance file, a
|
|
565
|
+
helper file that is not the output it names, a gap-records answer of
|
|
566
|
+
the wrong type, targeted re-extraction to match with no
|
|
567
|
+
--remediation-records, or, for baseline-gaps, a source root that is
|
|
568
|
+
no folder or a verifier that cannot be loaded)
|
|
569
|
+
3 apply refused: a blocking gap reached the provenance write with no
|
|
570
|
+
source line (nothing written)
|
|
72
571
|
"""
|
|
73
572
|
|
|
74
573
|
from __future__ import annotations
|
|
75
574
|
|
|
76
575
|
import argparse
|
|
576
|
+
import difflib
|
|
577
|
+
import hashlib
|
|
578
|
+
import importlib.util
|
|
77
579
|
import json
|
|
580
|
+
import os
|
|
581
|
+
import re
|
|
582
|
+
import subprocess
|
|
78
583
|
import sys
|
|
584
|
+
import time
|
|
79
585
|
from pathlib import Path
|
|
80
586
|
|
|
81
587
|
|
|
@@ -99,7 +605,10 @@ def _get_list(d: dict, *path: str) -> list:
|
|
|
99
605
|
def _load_input(input_path: Path | None) -> dict:
|
|
100
606
|
"""Read JSON either from --input path or from stdin."""
|
|
101
607
|
if input_path is not None:
|
|
102
|
-
|
|
608
|
+
try:
|
|
609
|
+
text = input_path.read_text(encoding="utf-8")
|
|
610
|
+
except OSError as exc:
|
|
611
|
+
raise ValueError(f"cannot read input {input_path}: {exc}") from exc
|
|
103
612
|
else:
|
|
104
613
|
text = sys.stdin.read()
|
|
105
614
|
try:
|
|
@@ -111,6 +620,139 @@ def _load_input(input_path: Path | None) -> dict:
|
|
|
111
620
|
return data
|
|
112
621
|
|
|
113
622
|
|
|
623
|
+
def _load_helper_file(path: Path, what: str, key: str) -> dict:
|
|
624
|
+
"""A helper's JSON output: an object holding `key`. Raises ValueError."""
|
|
625
|
+
try:
|
|
626
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
627
|
+
except OSError as exc:
|
|
628
|
+
raise ValueError(f"cannot read {what} {path}: {exc}") from exc
|
|
629
|
+
except json.JSONDecodeError as exc:
|
|
630
|
+
raise ValueError(f"{what} {path} is not valid JSON: {exc}") from exc
|
|
631
|
+
if not isinstance(data, dict) or key not in data:
|
|
632
|
+
raise ValueError(f"{what} {path} has no `{key}`: is it the helper's output?")
|
|
633
|
+
return data
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
# --------------------------------------------------------------------------
|
|
637
|
+
# Helper files (update-skill detect-changes §2.1)
|
|
638
|
+
# --------------------------------------------------------------------------
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def category_b_from_diff(diff: dict) -> dict:
|
|
642
|
+
"""Category B's four lists from a skf-structural-diff.py diff of the
|
|
643
|
+
modified files (see the module docstring for the mapping)."""
|
|
644
|
+
lists: dict[str, list] = {
|
|
645
|
+
"modified_exports": [], "new_exports": [], "deleted_exports": [], "moved_exports": [],
|
|
646
|
+
}
|
|
647
|
+
for item in _get_list(diff, "removed"):
|
|
648
|
+
if isinstance(item, dict):
|
|
649
|
+
lists["deleted_exports"].append(
|
|
650
|
+
{"name": item.get("name"), "file": item.get("file"), "old_line": item.get("line")})
|
|
651
|
+
for item in _get_list(diff, "added"):
|
|
652
|
+
if isinstance(item, dict):
|
|
653
|
+
lists["new_exports"].append({"name": item.get("name"), "file": item.get("file"), "line": item.get("line")})
|
|
654
|
+
changed: dict[tuple, dict] = {}
|
|
655
|
+
for item in _get_list(diff, "changed"):
|
|
656
|
+
if not isinstance(item, dict):
|
|
657
|
+
continue
|
|
658
|
+
record = changed.setdefault(
|
|
659
|
+
(item.get("name"), item.get("file")), {"fields": set(), "line": item.get("line"), "old_line": None})
|
|
660
|
+
record["fields"].add(item.get("field"))
|
|
661
|
+
if item.get("field") == "line":
|
|
662
|
+
record["old_line"] = item.get("baseline_value")
|
|
663
|
+
unverified = list(dict.fromkeys(
|
|
664
|
+
(item.get("name"), item.get("file")) for item in _get_list(diff, "signature_unverified")
|
|
665
|
+
if isinstance(item, dict)
|
|
666
|
+
))
|
|
667
|
+
modified: set[tuple] = set()
|
|
668
|
+
for key, record in changed.items():
|
|
669
|
+
old_line = record["old_line"] if record["old_line"] is not None else record["line"]
|
|
670
|
+
entry = {"name": key[0], "file": key[1], "old_line": old_line, "new_line": record["line"]}
|
|
671
|
+
if record["fields"] - {"line"} or key in unverified:
|
|
672
|
+
lists["modified_exports"].append(entry)
|
|
673
|
+
modified.add(key)
|
|
674
|
+
else:
|
|
675
|
+
lists["moved_exports"].append(entry)
|
|
676
|
+
for key in unverified:
|
|
677
|
+
if key not in modified:
|
|
678
|
+
lists["modified_exports"].append({"name": key[0], "file": key[1], "old_line": None, "new_line": None})
|
|
679
|
+
modified.add(key)
|
|
680
|
+
for item in _get_list(diff, "moved"):
|
|
681
|
+
if not isinstance(item, dict):
|
|
682
|
+
continue
|
|
683
|
+
key, old_file = (item.get("name"), item.get("current_file")), _norm_path(item.get("previous_file"))
|
|
684
|
+
across = old_file not in (None, _norm_path(item.get("current_file"))) # apply finds the entry at old_file
|
|
685
|
+
if key in modified:
|
|
686
|
+
if across: # moved across files and changed: a MODIFIED_EXPORT that names the file it left
|
|
687
|
+
for entry in lists["modified_exports"]:
|
|
688
|
+
if (entry["name"], entry["file"]) == key:
|
|
689
|
+
entry.update(old_line=item.get("previous_line"), old_file=old_file)
|
|
690
|
+
continue
|
|
691
|
+
move = {"name": item.get("name"), "file": item.get("current_file")}
|
|
692
|
+
if across:
|
|
693
|
+
move["old_file"] = old_file
|
|
694
|
+
lists["moved_exports"].append({**move, "old_line": item.get("previous_line"), "new_line": item.get("line")})
|
|
695
|
+
return lists
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def _take_out(entries: list, name: object, files: tuple) -> None:
|
|
699
|
+
"""Remove the first entry named `name`, preferring one in the first of
|
|
700
|
+
`files` that holds one (a None in `files` stands for any file)."""
|
|
701
|
+
for wanted in files:
|
|
702
|
+
for i, entry in enumerate(entries):
|
|
703
|
+
if isinstance(entry, dict) and entry.get("name") == name and wanted in (None, entry.get("file")):
|
|
704
|
+
del entries[i]
|
|
705
|
+
return
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
def apply_renames(payload: dict) -> dict:
|
|
709
|
+
"""Take each Category C rename's pair out of the Category A and B lists
|
|
710
|
+
it was found in (see the module docstring). payload holds category_a
|
|
711
|
+
and category_b as objects, as assemble leaves them."""
|
|
712
|
+
cat_a, cat_b = payload["category_a"], payload["category_b"]
|
|
713
|
+
cat_c = payload.get("category_c", {}) or {}
|
|
714
|
+
for move in _get_list(cat_c, "renamed_files"):
|
|
715
|
+
if not isinstance(move, dict):
|
|
716
|
+
continue
|
|
717
|
+
for key, path in (("deleted", move.get("old_path")), ("added", move.get("new_path"))):
|
|
718
|
+
listed = _get_list(cat_a, key)
|
|
719
|
+
if path in listed:
|
|
720
|
+
cat_a[key] = [p for p in listed if p != path]
|
|
721
|
+
for rename in _get_list(cat_c, "renamed_exports"):
|
|
722
|
+
if not isinstance(rename, dict):
|
|
723
|
+
continue
|
|
724
|
+
deleted = list(_get_list(cat_b, "deleted_exports"))
|
|
725
|
+
new = list(_get_list(cat_b, "new_exports"))
|
|
726
|
+
old_files = (rename["old_file"],) if rename.get("old_file") else (rename.get("file"), None)
|
|
727
|
+
_take_out(deleted, rename.get("old_name"), old_files)
|
|
728
|
+
_take_out(new, rename.get("new_name"), (rename.get("file"), None))
|
|
729
|
+
cat_b["deleted_exports"], cat_b["new_exports"] = deleted, new
|
|
730
|
+
return payload
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def assemble(payload: dict, *, category_a_doc: dict | None = None, diff: dict | None = None) -> dict:
|
|
734
|
+
"""The category JSON build and deletion-ratio read: the input, with the
|
|
735
|
+
helper files' slices in place of its own, and Category C's renames taken
|
|
736
|
+
out of the A and B lists."""
|
|
737
|
+
out = dict(payload)
|
|
738
|
+
cat_c = dict(payload.get("category_c") or {})
|
|
739
|
+
if category_a_doc is not None:
|
|
740
|
+
out["category_a"] = {key: list(_get_list(category_a_doc, "category_a", key))
|
|
741
|
+
for key in ("modified", "added", "deleted")}
|
|
742
|
+
moves = [m for m in _get_list(category_a_doc, "moved_files") if isinstance(m, dict)]
|
|
743
|
+
cat_c["renamed_files"] = moves + list(_get_list(cat_c, "renamed_files"))
|
|
744
|
+
else:
|
|
745
|
+
out["category_a"] = {key: list(_get_list(payload, "category_a", key))
|
|
746
|
+
for key in ("modified", "added", "deleted")}
|
|
747
|
+
if diff is not None:
|
|
748
|
+
out["category_b"] = category_b_from_diff(diff)
|
|
749
|
+
else:
|
|
750
|
+
out["category_b"] = {key: list(_get_list(payload, "category_b", key))
|
|
751
|
+
for key in ("modified_exports", "new_exports", "deleted_exports", "moved_exports")}
|
|
752
|
+
out["category_c"] = cat_c
|
|
753
|
+
return apply_renames(out)
|
|
754
|
+
|
|
755
|
+
|
|
114
756
|
# --------------------------------------------------------------------------
|
|
115
757
|
# Build manifest
|
|
116
758
|
# --------------------------------------------------------------------------
|
|
@@ -135,14 +777,8 @@ def build_manifest(payload: dict) -> dict:
|
|
|
135
777
|
c_renamed_files = _get_list(cat_c, "renamed_files")
|
|
136
778
|
c_renamed_exports = _get_list(cat_c, "renamed_exports")
|
|
137
779
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
"scripts_added": len(_get_list(cat_d, "scripts_added")),
|
|
141
|
-
"scripts_deleted": len(_get_list(cat_d, "scripts_deleted")),
|
|
142
|
-
"assets_modified": len(_get_list(cat_d, "assets_modified")),
|
|
143
|
-
"assets_added": len(_get_list(cat_d, "assets_added")),
|
|
144
|
-
"assets_deleted": len(_get_list(cat_d, "assets_deleted")),
|
|
145
|
-
}
|
|
780
|
+
d_lists = {key: list(dict.fromkeys(p for p in _get_list(cat_d, key) if isinstance(p, str))) for key in D_KEYS}
|
|
781
|
+
d = {key: len(paths) for key, paths in d_lists.items()}
|
|
146
782
|
|
|
147
783
|
counts = {
|
|
148
784
|
"files_changed": len(a_modified),
|
|
@@ -186,9 +822,51 @@ def build_manifest(payload: dict) -> dict:
|
|
|
186
822
|
"counts": counts,
|
|
187
823
|
"total_export_changes": total_export_changes,
|
|
188
824
|
"per_file": per_file,
|
|
825
|
+
"category_d": d_lists,
|
|
189
826
|
}
|
|
190
827
|
|
|
191
828
|
|
|
829
|
+
# Category D's lists, in the order build counts them.
|
|
830
|
+
D_KEYS = ("scripts_modified", "scripts_added", "scripts_deleted", "assets_modified", "assets_added",
|
|
831
|
+
"assets_deleted", "docs_modified", "docs_deleted")
|
|
832
|
+
FILE_TYPES = ("script", "asset", "doc")
|
|
833
|
+
_FILE_TYPE_OF_FOLDER = {"scripts": "script", "assets": "asset", "docs": "doc"}
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def _file_type(row: dict) -> str:
|
|
837
|
+
"""A file_entries[] row's file_type, else the one its file_name's first folder names, else asset."""
|
|
838
|
+
kind = row.get("file_type")
|
|
839
|
+
if kind in FILE_TYPES:
|
|
840
|
+
return kind
|
|
841
|
+
name = row.get("file_name")
|
|
842
|
+
folder = name.strip().replace("\\", "/").split("/", 1)[0] if isinstance(name, str) else ""
|
|
843
|
+
return _FILE_TYPE_OF_FOLDER.get(folder, "asset")
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def category_d_from_files(compare: dict | None, new_files: dict | None, provenance: dict | None) -> dict:
|
|
847
|
+
"""Category D's lists from skf-hash-content.py compare's rows, each typed by the file_entries[] row of its
|
|
848
|
+
source_file in `provenance`, and from skf-new-file-diff.py's new files, by kind. Raises ValueError."""
|
|
849
|
+
rows = (provenance or {}).get("file_entries") or []
|
|
850
|
+
if not isinstance(rows, list):
|
|
851
|
+
raise ValueError("provenance `file_entries` must be an array")
|
|
852
|
+
types: dict[str, str] = {}
|
|
853
|
+
for row in rows:
|
|
854
|
+
if isinstance(row, dict) and _norm_path(row.get("source_file")):
|
|
855
|
+
types.setdefault(_norm_path(row["source_file"]), _file_type(row))
|
|
856
|
+
lists: dict[str, list] = {key: [] for key in D_KEYS}
|
|
857
|
+
changes = {"MODIFIED_FILE": "modified", "DELETED_FILE": "deleted"}
|
|
858
|
+
for row in _get_list(compare or {}, "comparisons"):
|
|
859
|
+
if not isinstance(row, dict) or row.get("classification") not in changes:
|
|
860
|
+
continue
|
|
861
|
+
path = _norm_path(row.get("source_file"))
|
|
862
|
+
if path:
|
|
863
|
+
lists[f"{types.get(path, 'asset')}s_{changes[row['classification']]}"].append(path)
|
|
864
|
+
for item in _get_list(new_files or {}, "new_files"):
|
|
865
|
+
if isinstance(item, dict) and item.get("kind") in ("script", "asset") and _norm_path(item.get("source_file")):
|
|
866
|
+
lists[f"{item['kind']}s_added"].append(_norm_path(item["source_file"]))
|
|
867
|
+
return lists
|
|
868
|
+
|
|
869
|
+
|
|
192
870
|
def _build_per_file(
|
|
193
871
|
*,
|
|
194
872
|
a_modified: list,
|
|
@@ -209,12 +887,15 @@ def _build_per_file(
|
|
|
209
887
|
path = entry.get("file")
|
|
210
888
|
if not path:
|
|
211
889
|
return
|
|
212
|
-
|
|
890
|
+
change = {
|
|
213
891
|
"name": entry.get("name"),
|
|
214
892
|
"change_type": change_type,
|
|
215
893
|
"old_line": entry.get("old_line"),
|
|
216
894
|
"new_line": entry.get("new_line") or entry.get("line"),
|
|
217
|
-
}
|
|
895
|
+
}
|
|
896
|
+
if entry.get("old_file"):
|
|
897
|
+
change["old_file"] = entry["old_file"] # a move across files: a MOVED_EXPORT, or a MODIFIED_EXPORT
|
|
898
|
+
exports_by_file.setdefault(path, []).append(change)
|
|
218
899
|
|
|
219
900
|
for e in b_modified:
|
|
220
901
|
_record(e, "MODIFIED_EXPORT")
|
|
@@ -337,18 +1018,1665 @@ def _ratio_skip(*, skip_reason: str) -> dict:
|
|
|
337
1018
|
}
|
|
338
1019
|
|
|
339
1020
|
|
|
1021
|
+
# --------------------------------------------------------------------------
|
|
1022
|
+
# Shared readers (rename-candidates and apply)
|
|
1023
|
+
# --------------------------------------------------------------------------
|
|
1024
|
+
|
|
1025
|
+
|
|
1026
|
+
def _norm_path(path: object) -> str | None:
|
|
1027
|
+
"""A file path compared as structural-diff compares it: `/` separators, no leading `./`."""
|
|
1028
|
+
if not isinstance(path, str) or not path.strip():
|
|
1029
|
+
return None
|
|
1030
|
+
text = path.strip().replace("\\", "/")
|
|
1031
|
+
while text.startswith("./"):
|
|
1032
|
+
text = text[2:]
|
|
1033
|
+
return text
|
|
1034
|
+
|
|
1035
|
+
|
|
1036
|
+
def _load_json_file(path: Path, what: str):
|
|
1037
|
+
try:
|
|
1038
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
1039
|
+
except OSError as exc:
|
|
1040
|
+
raise ValueError(f"cannot read {what} {path}: {exc}") from exc
|
|
1041
|
+
except json.JSONDecodeError as exc:
|
|
1042
|
+
raise ValueError(f"{what} {path} is not valid JSON: {exc}") from exc
|
|
1043
|
+
|
|
1044
|
+
|
|
1045
|
+
def _exports_of(doc, what: str) -> list[dict]:
|
|
1046
|
+
"""The export records of an extraction or export-details file ({"exports": [...]})."""
|
|
1047
|
+
if doc is None:
|
|
1048
|
+
return []
|
|
1049
|
+
if not isinstance(doc, dict) or not isinstance(doc.get("exports"), list):
|
|
1050
|
+
raise ValueError(f"{what} has no `exports` array: is it the helper's output?")
|
|
1051
|
+
return [e for e in doc["exports"] if isinstance(e, dict)]
|
|
1052
|
+
|
|
1053
|
+
|
|
1054
|
+
def _entries_of(provenance) -> list[dict]:
|
|
1055
|
+
if provenance is None:
|
|
1056
|
+
return []
|
|
1057
|
+
if not isinstance(provenance, dict):
|
|
1058
|
+
raise ValueError("the provenance map must be a JSON object")
|
|
1059
|
+
entries = provenance.get("entries", [])
|
|
1060
|
+
if not isinstance(entries, list):
|
|
1061
|
+
raise ValueError("provenance `entries` must be an array")
|
|
1062
|
+
return [e for e in entries if isinstance(e, dict)]
|
|
1063
|
+
|
|
1064
|
+
|
|
1065
|
+
def _params(value) -> tuple | None:
|
|
1066
|
+
"""A parameter list compared by its text, whitespace removed."""
|
|
1067
|
+
if value is None:
|
|
1068
|
+
return None
|
|
1069
|
+
if not isinstance(value, list):
|
|
1070
|
+
return (str(value),)
|
|
1071
|
+
out = []
|
|
1072
|
+
for item in value:
|
|
1073
|
+
if isinstance(item, dict):
|
|
1074
|
+
name, kind = item.get("name"), item.get("type")
|
|
1075
|
+
item = f"{name}: {kind}" if kind else str(name)
|
|
1076
|
+
out.append("".join(str(item).split()))
|
|
1077
|
+
return tuple(out)
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
def _signature(record: dict) -> tuple:
|
|
1081
|
+
"""(export_type, params, return_type) of an export record, the parts a signature compares."""
|
|
1082
|
+
kind = record.get("export_type", record.get("type"))
|
|
1083
|
+
params = record.get("params", record.get("parameters"))
|
|
1084
|
+
ret = record.get("return_type")
|
|
1085
|
+
return (kind if isinstance(kind, str) else None, _params(params),
|
|
1086
|
+
"".join(ret.split()) if isinstance(ret, str) else None)
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
def _name_of(record: dict) -> str | None:
|
|
1090
|
+
name = record.get("export_name", record.get("name"))
|
|
1091
|
+
return name if isinstance(name, str) and name else None
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def _file_of(record: dict) -> str | None:
|
|
1095
|
+
return _norm_path(record.get("source_file", record.get("file", record.get("file_path"))))
|
|
1096
|
+
|
|
1097
|
+
|
|
1098
|
+
_INVENTORY = None
|
|
1099
|
+
|
|
1100
|
+
|
|
1101
|
+
def _inventory():
|
|
1102
|
+
"""skf-extraction-inventory.py, loaded once from this folder: its typed_param and its public-surface rule, so
|
|
1103
|
+
create-skill and update-skill write one parameter form and keep one public surface."""
|
|
1104
|
+
global _INVENTORY
|
|
1105
|
+
if _INVENTORY is None:
|
|
1106
|
+
path = Path(__file__).resolve().parent / "skf-extraction-inventory.py"
|
|
1107
|
+
spec = importlib.util.spec_from_file_location("skf_extraction_inventory", path)
|
|
1108
|
+
if spec is None or spec.loader is None:
|
|
1109
|
+
raise ImportError(f"cannot load {path}")
|
|
1110
|
+
module = importlib.util.module_from_spec(spec)
|
|
1111
|
+
spec.loader.exec_module(module)
|
|
1112
|
+
_INVENTORY = module
|
|
1113
|
+
return _INVENTORY
|
|
1114
|
+
|
|
1115
|
+
|
|
1116
|
+
def public_surface(extraction: object) -> tuple[object | None, str | None]:
|
|
1117
|
+
"""skf-extraction-inventory.py's public_surface over the recipe runner's output (see "Records" in the module
|
|
1118
|
+
docstring): (the public surface, None) for a public-api run, (None, why) when its surface cannot be read, and
|
|
1119
|
+
(None, None) for any other scope type. An input that is no object (a gap-driven or docs-only run gives none)
|
|
1120
|
+
is (None, None) without loading the inventory helper."""
|
|
1121
|
+
if not isinstance(extraction, dict):
|
|
1122
|
+
return None, None
|
|
1123
|
+
return _inventory().public_surface(extraction)
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
# --------------------------------------------------------------------------
|
|
1127
|
+
# Rename candidates (update-skill detect-changes Category C)
|
|
1128
|
+
# --------------------------------------------------------------------------
|
|
1129
|
+
|
|
1130
|
+
|
|
1131
|
+
RENAME_SIMILARITY = 0.80 # content similarity above this is a rename (fixed)
|
|
1132
|
+
SIZE_TOLERANCE = 0.20 # Quick tier: the added file's size within 20% of the deleted one's
|
|
1133
|
+
NAME_OVERLAP = 0.70 # Quick tier: shared export names above 70%
|
|
1134
|
+
TIERS = ("Quick", "Forge", "Forge+", "Deep")
|
|
1135
|
+
|
|
1136
|
+
|
|
1137
|
+
def _git_size(source_root: Path, commit: str, path: str) -> int | None:
|
|
1138
|
+
"""The size of `path` at `commit` (git cat-file -s), or None when git cannot tell."""
|
|
1139
|
+
try:
|
|
1140
|
+
proc = subprocess.run(["git", "-C", str(source_root), "cat-file", "-s", f"{commit}:{path}"],
|
|
1141
|
+
capture_output=True, text=True, timeout=30, check=False)
|
|
1142
|
+
except (OSError, subprocess.SubprocessError):
|
|
1143
|
+
return None
|
|
1144
|
+
if proc.returncode != 0:
|
|
1145
|
+
return None
|
|
1146
|
+
try:
|
|
1147
|
+
return int(proc.stdout.strip())
|
|
1148
|
+
except ValueError:
|
|
1149
|
+
return None
|
|
1150
|
+
|
|
1151
|
+
|
|
1152
|
+
def _file_size(source_root: Path | None, path: str) -> int | None:
|
|
1153
|
+
if source_root is None:
|
|
1154
|
+
return None
|
|
1155
|
+
try:
|
|
1156
|
+
return (source_root / path).stat().st_size
|
|
1157
|
+
except OSError:
|
|
1158
|
+
return None
|
|
1159
|
+
|
|
1160
|
+
|
|
1161
|
+
def _best_pairs(candidates: list[tuple]) -> list[tuple]:
|
|
1162
|
+
"""Each side's best match only: (score, left, right, rule) tuples, highest score first.
|
|
1163
|
+
|
|
1164
|
+
A pair is kept when neither side is taken yet and no other candidate of
|
|
1165
|
+
either side ties its score, so a tie pairs neither.
|
|
1166
|
+
"""
|
|
1167
|
+
out, used_left, used_right = [], set(), set()
|
|
1168
|
+
ordered = sorted(candidates, key=lambda c: (-c[0], c[1], c[2]))
|
|
1169
|
+
for score, left, right, rule in ordered:
|
|
1170
|
+
if left in used_left or right in used_right:
|
|
1171
|
+
continue
|
|
1172
|
+
rivals = [c for c in ordered if c[0] == score and (c[1] == left) != (c[2] == right)
|
|
1173
|
+
and c[1] not in used_left and c[2] not in used_right]
|
|
1174
|
+
if rivals:
|
|
1175
|
+
used_left.add(left)
|
|
1176
|
+
used_right.add(right)
|
|
1177
|
+
for c in rivals:
|
|
1178
|
+
used_left.add(c[1])
|
|
1179
|
+
used_right.add(c[2])
|
|
1180
|
+
continue
|
|
1181
|
+
used_left.add(left)
|
|
1182
|
+
used_right.add(right)
|
|
1183
|
+
out.append((score, left, right, rule))
|
|
1184
|
+
return out
|
|
1185
|
+
|
|
1186
|
+
|
|
1187
|
+
def rename_candidates(category_a_doc: dict, diff: dict | None, provenance: dict, extraction: dict | None,
|
|
1188
|
+
details: dict | None, tier: str, *, source_root: Path | None = None,
|
|
1189
|
+
sizes_commit: str | None = None) -> dict:
|
|
1190
|
+
"""Category C by the fixed rules of the module docstring."""
|
|
1191
|
+
if tier not in TIERS:
|
|
1192
|
+
raise ValueError(f"--tier must be one of {', '.join(TIERS)}, got {tier!r}")
|
|
1193
|
+
forge = tier != "Quick"
|
|
1194
|
+
moves = [m for m in _get_list(category_a_doc, "moved_files") if isinstance(m, dict)]
|
|
1195
|
+
moved_old = {_norm_path(m.get("old_path")) for m in moves}
|
|
1196
|
+
moved_new = {_norm_path(m.get("new_path")) for m in moves}
|
|
1197
|
+
deleted = sorted({p for p in map(_norm_path, _get_list(category_a_doc, "category_a", "deleted"))
|
|
1198
|
+
if p and p not in moved_old})
|
|
1199
|
+
added = sorted({p for p in map(_norm_path, _get_list(category_a_doc, "category_a", "added"))
|
|
1200
|
+
if p and p not in moved_new})
|
|
1201
|
+
|
|
1202
|
+
old_by_file: dict[str, dict[str, dict]] = {}
|
|
1203
|
+
for entry in _entries_of(provenance):
|
|
1204
|
+
name, path = _name_of(entry), _file_of(entry)
|
|
1205
|
+
if name and path:
|
|
1206
|
+
old_by_file.setdefault(path, {}).setdefault(name, entry)
|
|
1207
|
+
new_by_file: dict[str, dict[str, dict]] = {}
|
|
1208
|
+
for record in _exports_of(extraction, "--extraction file") + _exports_of(details, "--export-details file"):
|
|
1209
|
+
name, path = _name_of(record), _file_of(record)
|
|
1210
|
+
if name and path:
|
|
1211
|
+
merged = new_by_file.setdefault(path, {}).setdefault(name, {})
|
|
1212
|
+
for key, value in record.items():
|
|
1213
|
+
if merged.get(key) is None:
|
|
1214
|
+
merged[key] = value
|
|
1215
|
+
|
|
1216
|
+
file_candidates = []
|
|
1217
|
+
for old in deleted:
|
|
1218
|
+
old_exports = old_by_file.get(old, {})
|
|
1219
|
+
for new in added:
|
|
1220
|
+
new_exports = new_by_file.get(new, {})
|
|
1221
|
+
if not old_exports or not new_exports:
|
|
1222
|
+
continue
|
|
1223
|
+
if forge:
|
|
1224
|
+
matched = sum(1 for name, e in old_exports.items()
|
|
1225
|
+
if name in new_exports and _signature(e) == _signature(new_exports[name]))
|
|
1226
|
+
score = matched / max(len(old_exports), len(new_exports))
|
|
1227
|
+
if score > RENAME_SIMILARITY:
|
|
1228
|
+
file_candidates.append((round(score, 4), old, new, "signatures"))
|
|
1229
|
+
continue
|
|
1230
|
+
names_old, names_new = set(old_exports), set(new_exports)
|
|
1231
|
+
overlap = len(names_old & names_new) / len(names_old | names_new)
|
|
1232
|
+
if overlap <= NAME_OVERLAP:
|
|
1233
|
+
continue
|
|
1234
|
+
old_size = _git_size(source_root, sizes_commit, old) if (source_root and sizes_commit) else None
|
|
1235
|
+
new_size = _file_size(source_root, new)
|
|
1236
|
+
if old_size is not None:
|
|
1237
|
+
if new_size is None or max(old_size, new_size) == 0:
|
|
1238
|
+
continue
|
|
1239
|
+
if min(old_size, new_size) / max(old_size, new_size) < 1 - SIZE_TOLERANCE:
|
|
1240
|
+
continue
|
|
1241
|
+
rule = "size-and-names"
|
|
1242
|
+
else:
|
|
1243
|
+
rule = "names (size not checked)"
|
|
1244
|
+
file_candidates.append((round(overlap, 4), old, new, rule))
|
|
1245
|
+
file_pairs = _best_pairs(file_candidates)
|
|
1246
|
+
paired_old = {p[1] for p in file_pairs}
|
|
1247
|
+
paired_new = {p[2] for p in file_pairs}
|
|
1248
|
+
|
|
1249
|
+
removed = []
|
|
1250
|
+
for item in _get_list(diff or {}, "removed"):
|
|
1251
|
+
if isinstance(item, dict) and _name_of(item) and _file_of(item):
|
|
1252
|
+
name, path = _name_of(item), _file_of(item)
|
|
1253
|
+
removed.append({**old_by_file.get(path, {}).get(name, {}), **{k: v for k, v in item.items()
|
|
1254
|
+
if v is not None},
|
|
1255
|
+
"_key": (name, path)})
|
|
1256
|
+
candidates_new = []
|
|
1257
|
+
for item in _get_list(diff or {}, "added"):
|
|
1258
|
+
if isinstance(item, dict) and _name_of(item) and _file_of(item):
|
|
1259
|
+
name, path = _name_of(item), _file_of(item)
|
|
1260
|
+
candidates_new.append({**new_by_file.get(path, {}).get(name, {}),
|
|
1261
|
+
**{k: v for k, v in item.items() if v is not None}, "_key": (name, path)})
|
|
1262
|
+
seen_new = {c["_key"] for c in candidates_new}
|
|
1263
|
+
for path in added:
|
|
1264
|
+
if path in paired_new:
|
|
1265
|
+
continue
|
|
1266
|
+
for name, record in new_by_file.get(path, {}).items():
|
|
1267
|
+
if (name, path) not in seen_new:
|
|
1268
|
+
candidates_new.append({**record, "_key": (name, path)})
|
|
1269
|
+
|
|
1270
|
+
export_candidates = []
|
|
1271
|
+
for old in removed:
|
|
1272
|
+
old_sig = _signature(old)
|
|
1273
|
+
for new in candidates_new:
|
|
1274
|
+
if old["_key"][0] == new["_key"][0]:
|
|
1275
|
+
continue
|
|
1276
|
+
new_sig = _signature(new)
|
|
1277
|
+
if forge:
|
|
1278
|
+
if old_sig[1] is None and old_sig[2] is None:
|
|
1279
|
+
continue
|
|
1280
|
+
if old_sig == new_sig:
|
|
1281
|
+
export_candidates.append((1.0, old["_key"], new["_key"], "signature-equal"))
|
|
1282
|
+
continue
|
|
1283
|
+
if old_sig[0] and new_sig[0] and old_sig[0] != new_sig[0]:
|
|
1284
|
+
continue
|
|
1285
|
+
ratio = difflib.SequenceMatcher(None, old["_key"][0], new["_key"][0]).ratio()
|
|
1286
|
+
if ratio > RENAME_SIMILARITY:
|
|
1287
|
+
export_candidates.append((round(ratio, 4), old["_key"], new["_key"], "name-similarity"))
|
|
1288
|
+
export_pairs = _best_pairs(export_candidates)
|
|
1289
|
+
|
|
1290
|
+
renamed_exports = []
|
|
1291
|
+
for _score, (old_name, old_file), (new_name, new_file), _rule in export_pairs:
|
|
1292
|
+
item = {"old_name": old_name, "new_name": new_name, "file": new_file}
|
|
1293
|
+
if old_file != new_file:
|
|
1294
|
+
item["old_file"] = old_file
|
|
1295
|
+
renamed_exports.append(item)
|
|
1296
|
+
paired_removed = {p[1] for p in export_pairs}
|
|
1297
|
+
paired_added = {p[2] for p in export_pairs}
|
|
1298
|
+
return {
|
|
1299
|
+
"category_c": {
|
|
1300
|
+
"renamed_files": [{"old_path": old, "new_path": new} for _s, old, new, _r in file_pairs],
|
|
1301
|
+
"renamed_exports": renamed_exports,
|
|
1302
|
+
},
|
|
1303
|
+
"evidence": {
|
|
1304
|
+
"files": [{"old_path": old, "new_path": new, "rule": rule, "similarity": score}
|
|
1305
|
+
for score, old, new, rule in file_pairs],
|
|
1306
|
+
"exports": [{"old_name": o[0], "new_name": n[0], "file": n[1], "rule": rule, "similarity": score}
|
|
1307
|
+
for score, o, n, rule in export_pairs],
|
|
1308
|
+
},
|
|
1309
|
+
"unpaired": {
|
|
1310
|
+
"deleted_files": [p for p in deleted if p not in paired_old],
|
|
1311
|
+
"added_files": [p for p in added if p not in paired_new],
|
|
1312
|
+
"removed_exports": [{"name": r["_key"][0], "file": r["_key"][1]} for r in removed
|
|
1313
|
+
if r["_key"] not in paired_removed],
|
|
1314
|
+
"added_exports": [{"name": c["_key"][0], "file": c["_key"][1]} for c in candidates_new
|
|
1315
|
+
if c["_key"] not in paired_added],
|
|
1316
|
+
},
|
|
1317
|
+
"tier": tier,
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
|
|
1321
|
+
# --------------------------------------------------------------------------
|
|
1322
|
+
# Apply (update-skill write.md §3)
|
|
1323
|
+
# --------------------------------------------------------------------------
|
|
1324
|
+
|
|
1325
|
+
|
|
1326
|
+
UPDATE_TYPES = ("incremental", "gap-driven", "full")
|
|
1327
|
+
NON_BLOCKING = ("medium", "low", "info")
|
|
1328
|
+
EXIT_REFUSED = 3
|
|
1329
|
+
REPLACE_WAIT_SECONDS = 5.0 # how long a Windows rename onto a file held open is retried
|
|
1330
|
+
# The update block apply sets at the top level of the map, in this order.
|
|
1331
|
+
UPDATE_BLOCK_KEYS = ("last_update", "update_type", "test_report_run_id", "files_changed", "exports_affected",
|
|
1332
|
+
"confidence_tier", "manual_sections_preserved")
|
|
1333
|
+
|
|
1334
|
+
|
|
1335
|
+
def _labels(method: object, node: object) -> dict:
|
|
1336
|
+
"""The confidence labels an extraction method implies (create-skill's entry contract)."""
|
|
1337
|
+
if method in ("ast-grep", "ast_bridge"):
|
|
1338
|
+
return {"confidence": "T1", "extraction_method": "ast-grep", "ast_node_type": node,
|
|
1339
|
+
"signature_source": "T1"}
|
|
1340
|
+
return {"confidence": "T1-low", "extraction_method": "source-read", "ast_node_type": None,
|
|
1341
|
+
"signature_source": "T1-low"}
|
|
1342
|
+
|
|
1343
|
+
|
|
1344
|
+
def _line_of(value: object) -> int | None:
|
|
1345
|
+
"""A line number from an int, a digit string or a `file:line` / `file:start-end` location."""
|
|
1346
|
+
if isinstance(value, bool):
|
|
1347
|
+
return None
|
|
1348
|
+
if isinstance(value, int):
|
|
1349
|
+
return value
|
|
1350
|
+
if isinstance(value, str):
|
|
1351
|
+
tail = value.rsplit(":", 1)[-1].split("-", 1)[0].strip()
|
|
1352
|
+
if tail.isdigit():
|
|
1353
|
+
return int(tail)
|
|
1354
|
+
return None
|
|
1355
|
+
|
|
1356
|
+
|
|
1357
|
+
def _location(value: object) -> tuple[str | None, int | None]:
|
|
1358
|
+
"""(file, line) of a `file:line` string or a {file, line} object."""
|
|
1359
|
+
if isinstance(value, dict):
|
|
1360
|
+
return _norm_path(value.get("file") or value.get("source_file")), _line_of(
|
|
1361
|
+
value.get("line", value.get("source_line")))
|
|
1362
|
+
if isinstance(value, str) and ":" in value:
|
|
1363
|
+
path, _, rest = value.rpartition(":")
|
|
1364
|
+
return _norm_path(path), _line_of(rest)
|
|
1365
|
+
return None, None
|
|
1366
|
+
|
|
1367
|
+
|
|
1368
|
+
class _Fresh:
|
|
1369
|
+
"""The fresh records of one run, looked up by export name and file."""
|
|
1370
|
+
|
|
1371
|
+
def __init__(self, worker_files: list, extraction: dict | None, details: dict | None):
|
|
1372
|
+
self.runner = {}
|
|
1373
|
+
self.details = {}
|
|
1374
|
+
self.worker = {}
|
|
1375
|
+
self.surface, self.surface_problem = public_surface(extraction)
|
|
1376
|
+
for record in _exports_of(extraction, "--extraction file"):
|
|
1377
|
+
key = (_name_of(record), _file_of(record))
|
|
1378
|
+
if all(key):
|
|
1379
|
+
self.runner.setdefault(key, record)
|
|
1380
|
+
for record in _exports_of(details, "--export-details file"):
|
|
1381
|
+
key = (_name_of(record), _file_of(record))
|
|
1382
|
+
if all(key):
|
|
1383
|
+
self.details.setdefault(key, record)
|
|
1384
|
+
for block in worker_files:
|
|
1385
|
+
if not isinstance(block, dict):
|
|
1386
|
+
continue
|
|
1387
|
+
path = _norm_path(block.get("file_path"))
|
|
1388
|
+
for record in block.get("exports") or []:
|
|
1389
|
+
if isinstance(record, dict) and _name_of(record) and path:
|
|
1390
|
+
self.worker.setdefault((_name_of(record), path), record)
|
|
1391
|
+
|
|
1392
|
+
def keys_of(self, path: str) -> list[tuple]:
|
|
1393
|
+
keys = {k for source in (self.runner, self.details, self.worker) for k in source if k[1] == path}
|
|
1394
|
+
return sorted(keys)
|
|
1395
|
+
|
|
1396
|
+
def all_keys(self) -> list[tuple]:
|
|
1397
|
+
return sorted({k for source in (self.runner, self.details, self.worker) for k in source})
|
|
1398
|
+
|
|
1399
|
+
def public(self, name: str, path: str) -> bool | None:
|
|
1400
|
+
"""The `public` mark `records` gave the export, else the public-surface rule over --extraction; None when
|
|
1401
|
+
neither marks it."""
|
|
1402
|
+
mark = (self.worker.get((name, path)) or {}).get("public")
|
|
1403
|
+
if isinstance(mark, bool):
|
|
1404
|
+
return mark
|
|
1405
|
+
if self.surface is None:
|
|
1406
|
+
return None
|
|
1407
|
+
record = self.runner.get((name, path)) or self.details.get((name, path)) or {}
|
|
1408
|
+
return self.surface.public(name, path, record.get("language"))
|
|
1409
|
+
|
|
1410
|
+
def record(self, name: str, path: str) -> dict | None:
|
|
1411
|
+
"""The entry fields of an export's fresh record, or None when no source holds it."""
|
|
1412
|
+
key = (name, path)
|
|
1413
|
+
runner, detail, worker = self.runner.get(key), self.details.get(key), self.worker.get(key)
|
|
1414
|
+
if runner is None and detail is None and worker is None:
|
|
1415
|
+
return None
|
|
1416
|
+
sources = [r for r in (runner, detail, worker) if r is not None]
|
|
1417
|
+
|
|
1418
|
+
def first(*fields):
|
|
1419
|
+
for record in sources:
|
|
1420
|
+
for field in fields:
|
|
1421
|
+
value = record.get(field)
|
|
1422
|
+
if value is not None:
|
|
1423
|
+
return value
|
|
1424
|
+
return None
|
|
1425
|
+
|
|
1426
|
+
line = None
|
|
1427
|
+
for record in sources:
|
|
1428
|
+
line = _line_of(record.get("source_line"))
|
|
1429
|
+
if line is None and record is worker:
|
|
1430
|
+
line = _line_of(record.get("location"))
|
|
1431
|
+
if line is not None:
|
|
1432
|
+
break
|
|
1433
|
+
method = first("extraction_method")
|
|
1434
|
+
node = first("ast_node_type") if method in ("ast-grep", "ast_bridge") else None
|
|
1435
|
+
params = None
|
|
1436
|
+
for record in (detail, worker, runner):
|
|
1437
|
+
if record is None:
|
|
1438
|
+
continue
|
|
1439
|
+
value = record.get("params", record.get("parameters"))
|
|
1440
|
+
if value is not None:
|
|
1441
|
+
params = value
|
|
1442
|
+
break
|
|
1443
|
+
return {
|
|
1444
|
+
"export_name": name,
|
|
1445
|
+
"export_type": first("export_type", "type"),
|
|
1446
|
+
"params": params,
|
|
1447
|
+
"return_type": first("return_type"),
|
|
1448
|
+
"source_file": path,
|
|
1449
|
+
"source_line": line,
|
|
1450
|
+
**_labels(method, node),
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
|
|
1454
|
+
def _without_none(fields: dict, keep: tuple = ()) -> dict:
|
|
1455
|
+
return {k: v for k, v in fields.items() if v is not None or k in keep}
|
|
1456
|
+
|
|
1457
|
+
|
|
1458
|
+
def _new_entry(fields: dict, library: str) -> dict:
|
|
1459
|
+
"""An entry in create-skill's key order, from the fields a record gives."""
|
|
1460
|
+
order = ("export_name", "export_type", "source_library", "params", "return_type", "source_file",
|
|
1461
|
+
"source_line", "confidence", "extraction_method", "ast_node_type", "signature_source")
|
|
1462
|
+
merged = {**fields, "source_library": fields.get("source_library") or library}
|
|
1463
|
+
out = {k: merged[k] for k in order if k in merged and (merged[k] is not None or k == "ast_node_type")}
|
|
1464
|
+
return out
|
|
1465
|
+
|
|
1466
|
+
|
|
1467
|
+
def _rewrite(entry: dict, fields: dict) -> dict:
|
|
1468
|
+
"""The entry with `fields` set in place; keys the update does not set keep their values."""
|
|
1469
|
+
out = dict(entry)
|
|
1470
|
+
for key, value in fields.items():
|
|
1471
|
+
if value is None and key not in ("ast_node_type",):
|
|
1472
|
+
continue
|
|
1473
|
+
out[key] = value
|
|
1474
|
+
return out
|
|
1475
|
+
|
|
1476
|
+
|
|
1477
|
+
def _sha256_of(path: Path) -> str:
|
|
1478
|
+
digest = hashlib.sha256()
|
|
1479
|
+
with path.open("rb") as fh:
|
|
1480
|
+
for chunk in iter(lambda: fh.read(65536), b""):
|
|
1481
|
+
digest.update(chunk)
|
|
1482
|
+
return "sha256:" + digest.hexdigest()
|
|
1483
|
+
|
|
1484
|
+
|
|
1485
|
+
class _Map:
|
|
1486
|
+
"""The provenance map being rewritten, with what changed."""
|
|
1487
|
+
|
|
1488
|
+
def __init__(self, provenance: dict | None, skill_name: str):
|
|
1489
|
+
self.doc = json.loads(json.dumps(provenance)) if provenance is not None else {
|
|
1490
|
+
"provenance_version": "2.0", "skill_name": skill_name, "skill_type": "single", "entries": []}
|
|
1491
|
+
if not isinstance(self.doc.get("entries"), list):
|
|
1492
|
+
raise ValueError("provenance `entries` must be an array")
|
|
1493
|
+
libraries = [e.get("source_library") for e in self.doc["entries"]
|
|
1494
|
+
if isinstance(e, dict) and isinstance(e.get("source_library"), str) and e["source_library"]]
|
|
1495
|
+
self.library = libraries[0] if libraries else skill_name
|
|
1496
|
+
self.added: list[str] = []
|
|
1497
|
+
self.updated: list[str] = []
|
|
1498
|
+
self.removed: list[str] = []
|
|
1499
|
+
self.renamed: list[tuple[str, str]] = [] # (old name, new name)
|
|
1500
|
+
self.touched_files: set[str] = set()
|
|
1501
|
+
|
|
1502
|
+
def find(self, name: str, path: str | None, line: int | None = None) -> int | None:
|
|
1503
|
+
"""The index of the entry with this name (and file, and line when several share both)."""
|
|
1504
|
+
hits = [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and _name_of(e) == name
|
|
1505
|
+
and (path is None or _file_of(e) == path)]
|
|
1506
|
+
if len(hits) > 1 and line is not None:
|
|
1507
|
+
hits = [i for i in hits if _line_of(self.doc["entries"][i].get("source_line")) == line]
|
|
1508
|
+
return hits[0] if len(hits) == 1 else None
|
|
1509
|
+
|
|
1510
|
+
def indexes_of(self, path: str) -> list[int]:
|
|
1511
|
+
return [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and _file_of(e) == path]
|
|
1512
|
+
|
|
1513
|
+
def add(self, entry: dict) -> None:
|
|
1514
|
+
self.doc["entries"].append(entry)
|
|
1515
|
+
self.added.append(entry["export_name"])
|
|
1516
|
+
if entry.get("source_file"):
|
|
1517
|
+
self.touched_files.add(_norm_path(entry["source_file"]))
|
|
1518
|
+
|
|
1519
|
+
def update(self, index: int, fields: dict) -> None:
|
|
1520
|
+
before = self.doc["entries"][index]
|
|
1521
|
+
after = _rewrite(before, fields)
|
|
1522
|
+
if after != before:
|
|
1523
|
+
self.doc["entries"][index] = after
|
|
1524
|
+
self.updated.append(_name_of(after))
|
|
1525
|
+
if _name_of(before) and _name_of(after) and _name_of(before) != _name_of(after):
|
|
1526
|
+
self.renamed.append((_name_of(before), _name_of(after)))
|
|
1527
|
+
for path in (_file_of(before), _file_of(after)):
|
|
1528
|
+
if path:
|
|
1529
|
+
self.touched_files.add(path)
|
|
1530
|
+
|
|
1531
|
+
def remove(self, indexes: list[int]) -> None:
|
|
1532
|
+
for index in sorted(set(indexes), reverse=True):
|
|
1533
|
+
entry = self.doc["entries"].pop(index)
|
|
1534
|
+
self.removed.append(_name_of(entry))
|
|
1535
|
+
if _file_of(entry):
|
|
1536
|
+
self.touched_files.add(_file_of(entry))
|
|
1537
|
+
|
|
1538
|
+
|
|
1539
|
+
def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fresh: _Fresh,
|
|
1540
|
+
warnings: list[str], not_public: list[dict]) -> tuple[int, int]:
|
|
1541
|
+
"""Normal mode: the manifest's file and export changes. Returns (files, exports) for the update block; each
|
|
1542
|
+
name it leaves out as off a public-api skill's public surface goes to `not_public`."""
|
|
1543
|
+
per_file = [f for f in _get_list(manifest or {}, "per_file") if isinstance(f, dict)]
|
|
1544
|
+
# the names of the entries this run removes (a DELETED_EXPORT, a DELETED file's, a MOVED file's old path's):
|
|
1545
|
+
# one that comes back in another file is re-added whatever its mark, so no held export is lost
|
|
1546
|
+
removed: set[str] = set()
|
|
1547
|
+
for item in per_file:
|
|
1548
|
+
path = _norm_path(item.get("old_path") if item.get("status") == "MOVED" else item.get("file_path"))
|
|
1549
|
+
if item.get("status") in ("DELETED", "MOVED") and path:
|
|
1550
|
+
removed.update(_name_of(pmap.doc["entries"][i]) for i in pmap.indexes_of(path))
|
|
1551
|
+
removed.update(_name_of(c) for c in item.get("exports_affected") or []
|
|
1552
|
+
if isinstance(c, dict) and c.get("change_type") == "DELETED_EXPORT")
|
|
1553
|
+
listed: set[tuple[str, str]] = set()
|
|
1554
|
+
|
|
1555
|
+
def off_surface(name: str, path: str) -> bool:
|
|
1556
|
+
"""True for a name the run would add that is off the public surface, which it lists."""
|
|
1557
|
+
if name in removed or fresh.public(name, path) is not False:
|
|
1558
|
+
return False
|
|
1559
|
+
if (name, path) not in listed:
|
|
1560
|
+
listed.add((name, path))
|
|
1561
|
+
not_public.append({"name": name, "file": path})
|
|
1562
|
+
return True
|
|
1563
|
+
|
|
1564
|
+
def refresh(index: int | None, name: str, path: str, *, add: bool) -> None:
|
|
1565
|
+
record = fresh.record(name, path)
|
|
1566
|
+
if record is None:
|
|
1567
|
+
warnings.append(f"provenance: {name} in {path}: no extraction record, entry "
|
|
1568
|
+
f"{'kept as it was' if index is not None else 'not added'}")
|
|
1569
|
+
return
|
|
1570
|
+
if index is None:
|
|
1571
|
+
if add and not off_surface(name, path):
|
|
1572
|
+
pmap.add(_new_entry(record, pmap.library))
|
|
1573
|
+
return
|
|
1574
|
+
pmap.update(index, _without_none(record, keep=("ast_node_type",)))
|
|
1575
|
+
|
|
1576
|
+
for item in per_file:
|
|
1577
|
+
status, path = item.get("status"), _norm_path(item.get("file_path"))
|
|
1578
|
+
if not path:
|
|
1579
|
+
continue
|
|
1580
|
+
if status == "DELETED":
|
|
1581
|
+
pmap.remove(pmap.indexes_of(path))
|
|
1582
|
+
continue
|
|
1583
|
+
if status == "MOVED":
|
|
1584
|
+
old = _norm_path(item.get("old_path"))
|
|
1585
|
+
old_entries = {_name_of(pmap.doc["entries"][i]): pmap.doc["entries"][i]
|
|
1586
|
+
for i in (pmap.indexes_of(old) if old else [])}
|
|
1587
|
+
pmap.remove(pmap.indexes_of(old) if old else [])
|
|
1588
|
+
for name, key_path in fresh.keys_of(path):
|
|
1589
|
+
record = fresh.record(name, key_path)
|
|
1590
|
+
carried = old_entries.get(name)
|
|
1591
|
+
if carried is None and off_surface(name, key_path):
|
|
1592
|
+
continue
|
|
1593
|
+
entry = _rewrite(carried, _without_none(record, keep=("ast_node_type",))) if carried \
|
|
1594
|
+
else _new_entry(record, pmap.library)
|
|
1595
|
+
pmap.add(entry)
|
|
1596
|
+
continue
|
|
1597
|
+
if status == "ADDED":
|
|
1598
|
+
for name, key_path in fresh.keys_of(path):
|
|
1599
|
+
index = pmap.find(name, key_path)
|
|
1600
|
+
refresh(index, name, key_path, add=True)
|
|
1601
|
+
continue
|
|
1602
|
+
for change in item.get("exports_affected") or []:
|
|
1603
|
+
if not isinstance(change, dict) or not _name_of(change):
|
|
1604
|
+
continue
|
|
1605
|
+
name, kind = _name_of(change), change.get("change_type")
|
|
1606
|
+
# a move across files (a MOVED_EXPORT, or a MODIFIED_EXPORT that also moved) names the file its entry is at
|
|
1607
|
+
index = pmap.find(name, _norm_path(change.get("old_file")) or path, _line_of(change.get("old_line")))
|
|
1608
|
+
if kind == "DELETED_EXPORT":
|
|
1609
|
+
if index is not None:
|
|
1610
|
+
pmap.remove([index])
|
|
1611
|
+
elif kind == "MOVED_EXPORT":
|
|
1612
|
+
record = fresh.record(name, path)
|
|
1613
|
+
line = record["source_line"] if record and record.get("source_line") is not None \
|
|
1614
|
+
else _line_of(change.get("new_line"))
|
|
1615
|
+
if index is not None and line is not None:
|
|
1616
|
+
fields = {"source_file": path, "source_line": line}
|
|
1617
|
+
if record is not None and record["extraction_method"] == "ast-grep":
|
|
1618
|
+
# the runner matched it: its labels, never its export_type, params or return_type
|
|
1619
|
+
fields.update(confidence=record["confidence"], extraction_method=record["extraction_method"])
|
|
1620
|
+
if record["ast_node_type"] is not None:
|
|
1621
|
+
fields["ast_node_type"] = record["ast_node_type"]
|
|
1622
|
+
entry = pmap.doc["entries"][index]
|
|
1623
|
+
# its signature_source only when the record holds each signature part the entry does
|
|
1624
|
+
if all(record.get(part) is not None for part in ("params", "return_type")
|
|
1625
|
+
if entry.get(part) is not None):
|
|
1626
|
+
fields["signature_source"] = record["signature_source"]
|
|
1627
|
+
pmap.update(index, fields)
|
|
1628
|
+
elif kind in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1629
|
+
refresh(index, name, path, add=True)
|
|
1630
|
+
for rename in _get_list(categories or {}, "category_c", "renamed_exports"):
|
|
1631
|
+
if not isinstance(rename, dict):
|
|
1632
|
+
continue
|
|
1633
|
+
old_name, new_name = rename.get("old_name"), rename.get("new_name")
|
|
1634
|
+
path = _norm_path(rename.get("file"))
|
|
1635
|
+
old_path = _norm_path(rename.get("old_file")) or path
|
|
1636
|
+
if not (isinstance(old_name, str) and isinstance(new_name, str) and path):
|
|
1637
|
+
continue
|
|
1638
|
+
index = pmap.find(old_name, old_path)
|
|
1639
|
+
record = fresh.record(new_name, path)
|
|
1640
|
+
if index is None:
|
|
1641
|
+
continue
|
|
1642
|
+
fields = {"export_name": new_name, "source_file": path}
|
|
1643
|
+
if record is not None:
|
|
1644
|
+
fields.update(_without_none(record, keep=("ast_node_type",)))
|
|
1645
|
+
pmap.update(index, fields)
|
|
1646
|
+
files = len(per_file)
|
|
1647
|
+
total = manifest.get("total_export_changes") if isinstance(manifest, dict) else None
|
|
1648
|
+
return files, total if isinstance(total, int) else 0
|
|
1649
|
+
|
|
1650
|
+
|
|
1651
|
+
def _apply_docs_only(pmap: _Map, records: dict) -> tuple[int, int]:
|
|
1652
|
+
changed = [u for u in records.get("changed_urls") or [] if isinstance(u, str) and u]
|
|
1653
|
+
exports = [e for e in records.get("exports") or [] if isinstance(e, dict)]
|
|
1654
|
+
for url in changed:
|
|
1655
|
+
old = {_name_of(pmap.doc["entries"][i]): pmap.doc["entries"][i] for i in pmap.indexes_of(url)}
|
|
1656
|
+
pmap.remove(pmap.indexes_of(url))
|
|
1657
|
+
for record in exports:
|
|
1658
|
+
if record.get("url") != url or not _name_of(record):
|
|
1659
|
+
continue
|
|
1660
|
+
fields = {"export_name": _name_of(record), "export_type": record.get("export_type", record.get("type")),
|
|
1661
|
+
"params": record.get("params", record.get("parameters")),
|
|
1662
|
+
"return_type": record.get("return_type"), "source_file": url,
|
|
1663
|
+
"confidence": "T3", "signature_source": "T3"}
|
|
1664
|
+
carried = old.get(_name_of(record))
|
|
1665
|
+
entry = _rewrite(carried, _without_none(fields)) if carried else _new_entry(
|
|
1666
|
+
_without_none(fields), pmap.library)
|
|
1667
|
+
pmap.add(entry)
|
|
1668
|
+
return len(changed), len(exports)
|
|
1669
|
+
|
|
1670
|
+
|
|
1671
|
+
def _drift_note(drift: tuple | None) -> str:
|
|
1672
|
+
return f"drift override: HEAD {drift[0]} is not pinned {drift[1]}" if drift else ""
|
|
1673
|
+
|
|
1674
|
+
|
|
1675
|
+
def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, drift: tuple | None,
|
|
1676
|
+
warnings: list[str]) -> tuple[int, int] | list[dict]:
|
|
1677
|
+
"""Gap-driven mode: one write per verification record. A list return is the refusal."""
|
|
1678
|
+
verification = [r for r in records.get("verification") or [] if isinstance(r, dict) and _name_of(r)]
|
|
1679
|
+
reextracted = _Fresh(records.get("files") or [], None, None)
|
|
1680
|
+
merged = {}
|
|
1681
|
+
for record in _exports_of(merge_records, "--merge-records file") if merge_records is not None else []:
|
|
1682
|
+
if _name_of(record):
|
|
1683
|
+
merged.setdefault(_name_of(record), record)
|
|
1684
|
+
|
|
1685
|
+
def in_map(r: dict) -> bool:
|
|
1686
|
+
return bool(r.get("in_map")) or bool(r.get("map_entry"))
|
|
1687
|
+
|
|
1688
|
+
blocking = [{"export_name": _name_of(r), "severity": r.get("severity")} for r in verification
|
|
1689
|
+
if r.get("verification") == "unknown" and not in_map(r)
|
|
1690
|
+
and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT")
|
|
1691
|
+
and str(r.get("severity") or "").strip().lower() not in NON_BLOCKING]
|
|
1692
|
+
if blocking:
|
|
1693
|
+
return blocking
|
|
1694
|
+
note = _drift_note(drift)
|
|
1695
|
+
pinned_keys: set[tuple] = set()
|
|
1696
|
+
for r in verification:
|
|
1697
|
+
name, outcome, category = _name_of(r), r.get("verification"), r.get("gap_category")
|
|
1698
|
+
entry_file, entry_line = _location(r.get("map_entry"))
|
|
1699
|
+
index = pmap.find(name, entry_file, entry_line) if r.get("map_entry") else None
|
|
1700
|
+
lines = r.get("pinned_definition_lines")
|
|
1701
|
+
r5 = f"; definition lines per the test report: {lines}" if category == "MOVED_EXPORT" else ""
|
|
1702
|
+
if outcome == "rescoped":
|
|
1703
|
+
if index is not None:
|
|
1704
|
+
pmap.remove([index])
|
|
1705
|
+
elif outcome == "verified" and index is not None:
|
|
1706
|
+
if drift and category == "MOVED_EXPORT":
|
|
1707
|
+
warnings.append(f"provenance: {name}: verified at HEAD only ({note}{r5})")
|
|
1708
|
+
elif outcome == "moved" and index is not None:
|
|
1709
|
+
path, line = _location(r.get("new_location"))
|
|
1710
|
+
if path and line is not None:
|
|
1711
|
+
pmap.update(index, {"source_file": path, "source_line": line})
|
|
1712
|
+
elif outcome == "re-extracted":
|
|
1713
|
+
if r.get("reachability") == "internal-unreachable":
|
|
1714
|
+
continue # not public API: merge queued it as a metadata update
|
|
1715
|
+
path, line = _location(r.get("new_location"))
|
|
1716
|
+
record = reextracted.record(name, path) if path else None
|
|
1717
|
+
if record is None:
|
|
1718
|
+
warnings.append(f"provenance: {name}: re-extracted, but no record of it in the run's files")
|
|
1719
|
+
continue
|
|
1720
|
+
existing = pmap.find(name, path)
|
|
1721
|
+
if existing is None:
|
|
1722
|
+
pmap.add(_new_entry(record, pmap.library))
|
|
1723
|
+
else:
|
|
1724
|
+
pmap.update(existing, _without_none(record, keep=("ast_node_type",)))
|
|
1725
|
+
elif outcome in ("verified", "moved") and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1726
|
+
if r.get("reachability") == "internal-unreachable":
|
|
1727
|
+
continue
|
|
1728
|
+
where = r.get("source_citation") if outcome == "verified" else r.get("new_location")
|
|
1729
|
+
path, line = _location(where)
|
|
1730
|
+
if not path or line is None or (name, path) in pinned_keys or pmap.find(name, path) is not None:
|
|
1731
|
+
continue
|
|
1732
|
+
pinned_keys.add((name, path))
|
|
1733
|
+
extra = merged.get(name, {})
|
|
1734
|
+
pmap.add(_new_entry(_without_none({
|
|
1735
|
+
"export_name": name,
|
|
1736
|
+
"export_type": extra.get("export_type") or r.get("export_type"),
|
|
1737
|
+
"params": extra.get("params"), "return_type": extra.get("return_type"),
|
|
1738
|
+
"source_file": path, "source_line": line, **_labels("source-read", None)},
|
|
1739
|
+
keep=("ast_node_type",)), pmap.library))
|
|
1740
|
+
elif outcome == "unknown" and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1741
|
+
extra = merged.get(name, {})
|
|
1742
|
+
pmap.add(_new_entry(_without_none({
|
|
1743
|
+
"export_name": name, "export_type": extra.get("export_type") or r.get("export_type"),
|
|
1744
|
+
"params": extra.get("params"), "return_type": extra.get("return_type"),
|
|
1745
|
+
**_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library))
|
|
1746
|
+
else:
|
|
1747
|
+
if r.get("unknown_reason") == "drift-override":
|
|
1748
|
+
warnings.append(f"provenance: {name}: unknown ({note}; no line taken from HEAD{r5})")
|
|
1749
|
+
elif drift:
|
|
1750
|
+
warnings.append(f"provenance: {name}: {outcome} ({note}{r5})")
|
|
1751
|
+
else:
|
|
1752
|
+
warnings.append(f"provenance: {name}: {outcome}{r5}")
|
|
1753
|
+
return len(pmap.touched_files), len(verification)
|
|
1754
|
+
|
|
1755
|
+
|
|
1756
|
+
def _apply_files(pmap: _Map, compare: dict | None, new_files: dict | None, promoted: list | None,
|
|
1757
|
+
source_root: Path | None) -> dict:
|
|
1758
|
+
"""The file_entries[] changes of Category D and the promoted documents."""
|
|
1759
|
+
rows = pmap.doc.get("file_entries")
|
|
1760
|
+
if rows is None:
|
|
1761
|
+
rows = []
|
|
1762
|
+
if not isinstance(rows, list):
|
|
1763
|
+
raise ValueError("provenance `file_entries` must be an array")
|
|
1764
|
+
done = {"added": [], "updated": [], "removed": []}
|
|
1765
|
+
by_source = {_norm_path(r.get("source_file")): i for i, r in enumerate(rows) if isinstance(r, dict)}
|
|
1766
|
+
gone = set()
|
|
1767
|
+
for row in _get_list(compare or {}, "comparisons"):
|
|
1768
|
+
if not isinstance(row, dict):
|
|
1769
|
+
continue
|
|
1770
|
+
path, kind = _norm_path(row.get("source_file")), row.get("classification")
|
|
1771
|
+
index = by_source.get(path)
|
|
1772
|
+
if index is None:
|
|
1773
|
+
continue
|
|
1774
|
+
if kind == "MODIFIED_FILE" and isinstance(row.get("current_hash"), str):
|
|
1775
|
+
if rows[index].get("content_hash") != row["current_hash"]:
|
|
1776
|
+
rows[index] = {**rows[index], "content_hash": row["current_hash"]}
|
|
1777
|
+
done["updated"].append(path)
|
|
1778
|
+
elif kind == "DELETED_FILE":
|
|
1779
|
+
gone.add(index)
|
|
1780
|
+
done["removed"].append(path)
|
|
1781
|
+
rows = [r for i, r in enumerate(rows) if i not in gone]
|
|
1782
|
+
held = {_norm_path(r.get("source_file")) for r in rows if isinstance(r, dict)}
|
|
1783
|
+
for item in _get_list(new_files or {}, "new_files"):
|
|
1784
|
+
if not isinstance(item, dict):
|
|
1785
|
+
continue
|
|
1786
|
+
path, kind = _norm_path(item.get("source_file")), item.get("kind")
|
|
1787
|
+
if not path or path in held or kind not in ("script", "asset"):
|
|
1788
|
+
continue
|
|
1789
|
+
if source_root is None:
|
|
1790
|
+
raise ValueError("--new-files needs --source-root, to hash each new file")
|
|
1791
|
+
try:
|
|
1792
|
+
digest = _sha256_of(source_root / path)
|
|
1793
|
+
except OSError as exc:
|
|
1794
|
+
raise ValueError(f"cannot hash new file {path} under {source_root}: {exc}") from exc
|
|
1795
|
+
rows.append({"file_name": f"{kind}s/{Path(path).name}", "file_type": kind, "source_file": path,
|
|
1796
|
+
"confidence": "T1-low", "extraction_method": "file-copy", "content_hash": digest})
|
|
1797
|
+
held.add(path)
|
|
1798
|
+
done["added"].append(path)
|
|
1799
|
+
for doc in promoted or []:
|
|
1800
|
+
if not isinstance(doc, dict):
|
|
1801
|
+
continue
|
|
1802
|
+
path = _norm_path(doc.get("path"))
|
|
1803
|
+
if not path or path in held or not isinstance(doc.get("content_hash"), str):
|
|
1804
|
+
continue
|
|
1805
|
+
rows.append({"file_name": f"docs/authoritative/{path}", "file_type": "doc", "source_file": path,
|
|
1806
|
+
"content_hash": doc["content_hash"], "confidence": "T1-low",
|
|
1807
|
+
"extraction_method": "promoted-authoritative"})
|
|
1808
|
+
held.add(path)
|
|
1809
|
+
done["added"].append(path)
|
|
1810
|
+
if rows or "file_entries" in pmap.doc:
|
|
1811
|
+
pmap.doc["file_entries"] = rows
|
|
1812
|
+
return done
|
|
1813
|
+
|
|
1814
|
+
|
|
1815
|
+
REFERENCE_APP = "reference-app"
|
|
1816
|
+
|
|
1817
|
+
|
|
1818
|
+
def _export_name(item: object) -> str | None:
|
|
1819
|
+
"""The name of a metadata.json exports[] item: a string, or an object's name or export_name."""
|
|
1820
|
+
if isinstance(item, str):
|
|
1821
|
+
return item or None
|
|
1822
|
+
return _name_of(item) if isinstance(item, dict) else None
|
|
1823
|
+
|
|
1824
|
+
|
|
1825
|
+
def exports_after(metadata: object, pmap: _Map, update_type: str) -> list:
|
|
1826
|
+
"""metadata.json's exports[] after the update (see "Exports" in the module docstring)."""
|
|
1827
|
+
if not isinstance(metadata, dict):
|
|
1828
|
+
raise ValueError("--metadata-in file must hold a JSON object (the skill's metadata.json)")
|
|
1829
|
+
if metadata.get("scope_type") == REFERENCE_APP:
|
|
1830
|
+
return []
|
|
1831
|
+
current = metadata.get("exports") if isinstance(metadata.get("exports"), list) else []
|
|
1832
|
+
held = {_name_of(e) for e in pmap.doc["entries"] if isinstance(e, dict)} - {None}
|
|
1833
|
+
renamed = dict(pmap.renamed)
|
|
1834
|
+
gone = ({n for n in pmap.removed if n} | set(renamed)) - held
|
|
1835
|
+
if update_type == "full":
|
|
1836
|
+
gone |= {n for n in map(_export_name, current) if n and n not in held}
|
|
1837
|
+
out: list = []
|
|
1838
|
+
seen: set[str] = set()
|
|
1839
|
+
|
|
1840
|
+
def put(item: object, name: str | None) -> None:
|
|
1841
|
+
if name and name not in seen:
|
|
1842
|
+
seen.add(name)
|
|
1843
|
+
out.append(item)
|
|
1844
|
+
|
|
1845
|
+
for item in current:
|
|
1846
|
+
name = _export_name(item)
|
|
1847
|
+
if name in gone:
|
|
1848
|
+
new = renamed.get(name)
|
|
1849
|
+
put(new, new if new in held else None)
|
|
1850
|
+
else:
|
|
1851
|
+
put(item, name)
|
|
1852
|
+
for name in pmap.added + [new for _old, new in pmap.renamed]:
|
|
1853
|
+
put(name, name if name in held else None)
|
|
1854
|
+
return out
|
|
1855
|
+
|
|
1856
|
+
|
|
1857
|
+
_DETECTOR = None
|
|
1858
|
+
|
|
1859
|
+
|
|
1860
|
+
def _detector():
|
|
1861
|
+
"""skf-detect-scripts-assets.py, loaded once from this folder: the purpose a new script's or asset's row
|
|
1862
|
+
takes, read from the file as create-skill's detector reads it."""
|
|
1863
|
+
global _DETECTOR
|
|
1864
|
+
if _DETECTOR is None:
|
|
1865
|
+
path = Path(__file__).resolve().parent / "skf-detect-scripts-assets.py"
|
|
1866
|
+
spec = importlib.util.spec_from_file_location("skf_detect_scripts_assets", path)
|
|
1867
|
+
if spec is None or spec.loader is None or not path.is_file():
|
|
1868
|
+
raise ValueError(f"cannot load {path.name} beside {Path(__file__).name}: re-install SKF")
|
|
1869
|
+
module = importlib.util.module_from_spec(spec)
|
|
1870
|
+
spec.loader.exec_module(module)
|
|
1871
|
+
_DETECTOR = module
|
|
1872
|
+
return _DETECTOR
|
|
1873
|
+
|
|
1874
|
+
|
|
1875
|
+
def _purpose(source_file: str, kind: str, source_root: Path | None) -> str:
|
|
1876
|
+
"""A new script's or asset's purpose: the detector's, read from the file under `source_root`, else its name."""
|
|
1877
|
+
path = source_root / source_file if source_root is not None else None
|
|
1878
|
+
if path is None or not path.is_file():
|
|
1879
|
+
return Path(source_file).name
|
|
1880
|
+
detector = _detector()
|
|
1881
|
+
hint = detector.asset_type(path, source_root) if kind == "asset" else None
|
|
1882
|
+
return detector.extract_purpose(path, asset_type_hint=hint)
|
|
1883
|
+
|
|
1884
|
+
|
|
1885
|
+
def file_rows_after(metadata: dict, pmap: _Map, source_root: Path | None) -> dict[str, list]:
|
|
1886
|
+
"""metadata.json's scripts[] and assets[] after the update (see "Lists" in the module docstring)."""
|
|
1887
|
+
by_source: dict[str, dict] = {}
|
|
1888
|
+
by_file: dict[str, dict] = {}
|
|
1889
|
+
for key in ("scripts", "assets"):
|
|
1890
|
+
rows = metadata.get(key)
|
|
1891
|
+
for row in rows if isinstance(rows, list) else []:
|
|
1892
|
+
if not isinstance(row, dict):
|
|
1893
|
+
continue
|
|
1894
|
+
source = _norm_path(row.get("source_file"))
|
|
1895
|
+
if source:
|
|
1896
|
+
by_source.setdefault(source, row)
|
|
1897
|
+
if isinstance(row.get("file"), str):
|
|
1898
|
+
by_file.setdefault(row["file"], row)
|
|
1899
|
+
lists: dict[str, list] = {"scripts": [], "assets": []}
|
|
1900
|
+
entries = pmap.doc.get("file_entries")
|
|
1901
|
+
for entry in entries if isinstance(entries, list) else []:
|
|
1902
|
+
if not isinstance(entry, dict) or _file_type(entry) not in ("script", "asset"):
|
|
1903
|
+
continue
|
|
1904
|
+
kind = _file_type(entry)
|
|
1905
|
+
source = _norm_path(entry.get("source_file"))
|
|
1906
|
+
held = by_source.get(source) if source else None
|
|
1907
|
+
if held is None and isinstance(entry.get("file_name"), str):
|
|
1908
|
+
held = by_file.get(entry["file_name"])
|
|
1909
|
+
if held is None:
|
|
1910
|
+
held = {"file": entry.get("file_name") or f"{kind}s/{Path(source or '').name}",
|
|
1911
|
+
"purpose": _purpose(source, kind, source_root) if source else "",
|
|
1912
|
+
"source_file": source, "confidence": entry.get("confidence") or "T1-low"}
|
|
1913
|
+
lists[f"{kind}s"].append(held)
|
|
1914
|
+
return lists
|
|
1915
|
+
|
|
1916
|
+
|
|
1917
|
+
def apply_update(*, update_type: str, provenance: dict | None, skill_name: str, manifest: dict | None = None,
|
|
1918
|
+
categories: dict | None = None, extraction: dict | None = None, details: dict | None = None,
|
|
1919
|
+
records: dict | None = None, merge_records: dict | None = None, compare: dict | None = None,
|
|
1920
|
+
new_files: dict | None = None, promoted: list | None = None, source_root: Path | None = None,
|
|
1921
|
+
generation_date: str, test_report_run_id: str | None, confidence_tier: str,
|
|
1922
|
+
manual_sections_preserved: int, source_commit: str | None = None,
|
|
1923
|
+
source_ref: str | None = None, drift: tuple | None = None,
|
|
1924
|
+
metadata: dict | None = None) -> tuple[dict, dict]:
|
|
1925
|
+
"""(new map, summary) of one update (see "Apply" in the module docstring). With `metadata` (the skill's
|
|
1926
|
+
metadata.json), the summary's `lists` holds the exports[], scripts[] and assets[] it takes after the update.
|
|
1927
|
+
|
|
1928
|
+
A refusal returns ({}, summary) with status "refused" and no map.
|
|
1929
|
+
Raises ValueError for inputs it cannot read.
|
|
1930
|
+
"""
|
|
1931
|
+
if update_type not in UPDATE_TYPES:
|
|
1932
|
+
raise ValueError(f"--update-type must be one of {', '.join(UPDATE_TYPES)}, got {update_type!r}")
|
|
1933
|
+
if provenance is None and update_type != "full":
|
|
1934
|
+
raise ValueError("--provenance-map is required unless --update-type is full (a degraded run)")
|
|
1935
|
+
records = records if isinstance(records, dict) else {}
|
|
1936
|
+
pmap = _Map(provenance, skill_name)
|
|
1937
|
+
warnings: list[str] = []
|
|
1938
|
+
not_public: list[dict] = []
|
|
1939
|
+
mode = records.get("mode")
|
|
1940
|
+
if update_type == "gap-driven":
|
|
1941
|
+
outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings)
|
|
1942
|
+
if isinstance(outcome, list):
|
|
1943
|
+
return {}, {"status": "refused", "blocking_unresolved": outcome}
|
|
1944
|
+
files, exports = outcome
|
|
1945
|
+
elif mode == "docs-only":
|
|
1946
|
+
files, exports = _apply_docs_only(pmap, records)
|
|
1947
|
+
else:
|
|
1948
|
+
fresh = _Fresh(records.get("files") or [], extraction, details)
|
|
1949
|
+
if update_type == "full":
|
|
1950
|
+
for name, path in fresh.all_keys():
|
|
1951
|
+
index = pmap.find(name, path)
|
|
1952
|
+
record = fresh.record(name, path)
|
|
1953
|
+
if index is None:
|
|
1954
|
+
pmap.add(_new_entry(record, pmap.library))
|
|
1955
|
+
else:
|
|
1956
|
+
pmap.update(index, _without_none(record, keep=("ast_node_type",)))
|
|
1957
|
+
files = len({path for _name, path in fresh.all_keys()})
|
|
1958
|
+
exports = len(fresh.all_keys())
|
|
1959
|
+
else:
|
|
1960
|
+
files, exports = _apply_normal(pmap, manifest, categories, fresh, warnings, not_public)
|
|
1961
|
+
if fresh.surface_problem:
|
|
1962
|
+
warnings.append(f"provenance: the public surface of this public-api skill could not be applied "
|
|
1963
|
+
f"({fresh.surface_problem}): every new export was added")
|
|
1964
|
+
if not_public:
|
|
1965
|
+
warnings.append(_not_public_warning(not_public))
|
|
1966
|
+
file_changes = _apply_files(pmap, compare, new_files, promoted, source_root)
|
|
1967
|
+
doc = pmap.doc
|
|
1968
|
+
for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
|
|
1969
|
+
if value is not None:
|
|
1970
|
+
doc[key] = value or None
|
|
1971
|
+
block = {
|
|
1972
|
+
"last_update": generation_date,
|
|
1973
|
+
"update_type": update_type,
|
|
1974
|
+
"test_report_run_id": test_report_run_id or None,
|
|
1975
|
+
"files_changed": files,
|
|
1976
|
+
"exports_affected": exports,
|
|
1977
|
+
"confidence_tier": confidence_tier,
|
|
1978
|
+
"manual_sections_preserved": manual_sections_preserved,
|
|
1979
|
+
}
|
|
1980
|
+
for key in UPDATE_BLOCK_KEYS:
|
|
1981
|
+
doc[key] = block[key]
|
|
1982
|
+
summary = {
|
|
1983
|
+
"status": "written",
|
|
1984
|
+
"entries": {"added": pmap.added, "updated": pmap.updated, "removed": pmap.removed},
|
|
1985
|
+
"not_public": not_public,
|
|
1986
|
+
"file_entries": file_changes,
|
|
1987
|
+
"warnings": warnings,
|
|
1988
|
+
}
|
|
1989
|
+
if metadata is not None:
|
|
1990
|
+
summary["lists"] = {"exports": exports_after(metadata, pmap, update_type),
|
|
1991
|
+
**file_rows_after(metadata, pmap, source_root)}
|
|
1992
|
+
return doc, summary
|
|
1993
|
+
|
|
1994
|
+
|
|
1995
|
+
GENERATION_DATE_RE = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ")
|
|
1996
|
+
# the stats fields skf-render-metadata-stats.py writes, some only for a shape or a judgment
|
|
1997
|
+
STATS_HELPER_FIELDS = ("exports_documented", "exports_public_api", "exports_internal", "exports_total",
|
|
1998
|
+
"public_api_coverage", "total_coverage", "scripts_count", "assets_count",
|
|
1999
|
+
"pattern_surfaces_documented", "effective_denominator")
|
|
2000
|
+
|
|
2001
|
+
|
|
2002
|
+
def update_metadata(metadata: object, lists: object, stats: object, *, version: str, generation_date: str,
|
|
2003
|
+
source_commit: str | None = None, source_ref: str | None = None) -> dict:
|
|
2004
|
+
"""The metadata.json an update leaves (see "Metadata" in the module docstring). Raises ValueError."""
|
|
2005
|
+
if not isinstance(metadata, dict):
|
|
2006
|
+
raise ValueError("--metadata file must hold a JSON object (the skill's metadata.json)")
|
|
2007
|
+
if not isinstance(lists, dict) or not all(isinstance(lists.get(k), list) for k in ("exports", "scripts",
|
|
2008
|
+
"assets")):
|
|
2009
|
+
raise ValueError("--lists file must hold the exports, scripts and assets lists apply wrote (--lists-out)")
|
|
2010
|
+
if not isinstance(stats, dict) or not isinstance(stats.get("stats"), dict) \
|
|
2011
|
+
or not isinstance(stats.get("confidence_distribution"), dict):
|
|
2012
|
+
raise ValueError("--stats file must hold the stats helper's JSON: a stats and a confidence_distribution "
|
|
2013
|
+
"object")
|
|
2014
|
+
if not version.strip():
|
|
2015
|
+
raise ValueError("--version must not be empty")
|
|
2016
|
+
if not GENERATION_DATE_RE.fullmatch(generation_date):
|
|
2017
|
+
raise ValueError(f"--generation-date must be YYYY-MM-DDTHH:MM:SSZ, read from the clock; got "
|
|
2018
|
+
f"{generation_date!r}")
|
|
2019
|
+
for kind in ("scripts", "assets"):
|
|
2020
|
+
counted = stats["stats"].get(f"{kind}_count")
|
|
2021
|
+
if counted != len(lists[kind]):
|
|
2022
|
+
raise ValueError(f"the stats count {counted} {kind} but apply's lists hold {len(lists[kind])}: run the "
|
|
2023
|
+
f"stats helper again with the lists' {kind}")
|
|
2024
|
+
out = {k: v for k, v in metadata.items() if k not in ("last_update", "update_type")}
|
|
2025
|
+
out["version"] = version
|
|
2026
|
+
out["generation_date"] = generation_date
|
|
2027
|
+
for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
|
|
2028
|
+
if value is not None:
|
|
2029
|
+
out[key] = value or None
|
|
2030
|
+
out["exports"] = lists["exports"]
|
|
2031
|
+
for kind in ("scripts", "assets"):
|
|
2032
|
+
if lists[kind]:
|
|
2033
|
+
out[kind] = lists[kind]
|
|
2034
|
+
else:
|
|
2035
|
+
out.pop(kind, None)
|
|
2036
|
+
old_stats = metadata.get("stats") if isinstance(metadata.get("stats"), dict) else {}
|
|
2037
|
+
kept = {k: v for k, v in old_stats.items() if k not in STATS_HELPER_FIELDS and k not in stats["stats"]}
|
|
2038
|
+
out["stats"] = {**stats["stats"], **kept}
|
|
2039
|
+
out["confidence_distribution"] = stats["confidence_distribution"]
|
|
2040
|
+
return out
|
|
2041
|
+
|
|
2042
|
+
|
|
2043
|
+
NOT_PUBLIC_NAMED = 10 # the names the not_public warning spells out; the summary lists them all
|
|
2044
|
+
|
|
2045
|
+
|
|
2046
|
+
def _not_public_warning(not_public: list[dict]) -> str:
|
|
2047
|
+
"""The one warning for every name apply left out as off a public-api skill's public surface."""
|
|
2048
|
+
names = [item["name"] for item in not_public]
|
|
2049
|
+
shown = ", ".join(names[:NOT_PUBLIC_NAMED])
|
|
2050
|
+
more = f" and {len(names) - NOT_PUBLIC_NAMED} more" if len(names) > NOT_PUBLIC_NAMED else ""
|
|
2051
|
+
return (f"provenance: {len(names)} new export(s) off the public surface of this public-api skill not added "
|
|
2052
|
+
f"(the summary's not_public lists each with its file): {shown}{more}")
|
|
2053
|
+
|
|
2054
|
+
|
|
2055
|
+
def _write_json_atomic(path: Path, value) -> None:
|
|
2056
|
+
"""Write `value` as indented JSON through a temporary file beside `path` and one rename, retried on Windows,
|
|
2057
|
+
where a scanner may hold the target open for a moment."""
|
|
2058
|
+
tmp = path.with_name(f".{path.name}.skf-{os.getpid()}-tmp")
|
|
2059
|
+
try:
|
|
2060
|
+
tmp.write_bytes((json.dumps(value, indent=2, ensure_ascii=False) + "\n").encode("utf-8"))
|
|
2061
|
+
deadline = time.monotonic() + REPLACE_WAIT_SECONDS
|
|
2062
|
+
while True:
|
|
2063
|
+
try:
|
|
2064
|
+
os.replace(tmp, path)
|
|
2065
|
+
break
|
|
2066
|
+
except PermissionError:
|
|
2067
|
+
if os.name != "nt" or time.monotonic() >= deadline:
|
|
2068
|
+
raise
|
|
2069
|
+
time.sleep(0.05)
|
|
2070
|
+
except OSError:
|
|
2071
|
+
try:
|
|
2072
|
+
tmp.unlink()
|
|
2073
|
+
except OSError:
|
|
2074
|
+
pass
|
|
2075
|
+
raise
|
|
2076
|
+
|
|
2077
|
+
|
|
2078
|
+
# --------------------------------------------------------------------------
|
|
2079
|
+
# Records (update-skill re-extract.md §4, gap-driven.md §4a)
|
|
2080
|
+
# --------------------------------------------------------------------------
|
|
2081
|
+
|
|
2082
|
+
|
|
2083
|
+
# The fields a worker's patch may set: the ones no tool records, and the two
|
|
2084
|
+
# a patch sets only where the seed has none.
|
|
2085
|
+
PATCH_FIELDS = ("signature", "members", "docstring", "qmd_evidence")
|
|
2086
|
+
PATCH_IF_NULL = ("params", "return_type")
|
|
2087
|
+
|
|
2088
|
+
|
|
2089
|
+
def _typed_param(param, language: object) -> str | None:
|
|
2090
|
+
"""One parameter as the provenance map writes it: skf-extraction-inventory.py's typed_param, so create-skill
|
|
2091
|
+
and update-skill write one form."""
|
|
2092
|
+
return _inventory().typed_param(param, language if isinstance(language, str) else None)
|
|
2093
|
+
|
|
2094
|
+
|
|
2095
|
+
def _typed_params(value, language: object) -> list | None:
|
|
2096
|
+
if not isinstance(value, list):
|
|
2097
|
+
return None
|
|
2098
|
+
return [s for s in (_typed_param(p, language) for p in value) if s is not None]
|
|
2099
|
+
|
|
2100
|
+
|
|
2101
|
+
def _seed(record: dict, path: str, read_by_eye: bool) -> dict:
|
|
2102
|
+
"""The re-extraction record of one export from the runner's (or, read by eye, step 2's) record."""
|
|
2103
|
+
line = _line_of(record.get("source_line"))
|
|
2104
|
+
out = {
|
|
2105
|
+
"name": _name_of(record),
|
|
2106
|
+
"type": record.get("export_type", record.get("type")),
|
|
2107
|
+
"signature": record.get("signature"),
|
|
2108
|
+
"location": f"{path}:{line}" if line is not None else None,
|
|
2109
|
+
"params": _typed_params(record.get("params", record.get("parameters")), record.get("language")),
|
|
2110
|
+
"return_type": record.get("return_type"),
|
|
2111
|
+
}
|
|
2112
|
+
labels = _labels("source-read" if read_by_eye else record.get("extraction_method"), record.get("ast_node_type"))
|
|
2113
|
+
out.update({key: labels[key] for key in ("confidence", "extraction_method", "ast_node_type")})
|
|
2114
|
+
out["ast_recipe"] = None if read_by_eye else record.get("ast_recipe")
|
|
2115
|
+
return out
|
|
2116
|
+
|
|
2117
|
+
|
|
2118
|
+
def _patch_blocks(folder: Path) -> list[dict]:
|
|
2119
|
+
"""Every per-file block of the patch files in `folder` (a file holds one block or a list of them), in file
|
|
2120
|
+
name order. Raises ValueError."""
|
|
2121
|
+
if not folder.is_dir():
|
|
2122
|
+
raise ValueError(f"--patches {folder} is not a folder")
|
|
2123
|
+
blocks = []
|
|
2124
|
+
for path in sorted(folder.glob("*.json"), key=lambda p: p.name):
|
|
2125
|
+
doc = _load_json_file(path, "patch file")
|
|
2126
|
+
for block in doc if isinstance(doc, list) else [doc]:
|
|
2127
|
+
if not isinstance(block, dict) or not _norm_path(block.get("file_path")) \
|
|
2128
|
+
or not isinstance(block.get("exports"), list):
|
|
2129
|
+
raise ValueError(f"patch file {path} holds a block with no file_path or exports list")
|
|
2130
|
+
blocks.append(block)
|
|
2131
|
+
return blocks
|
|
2132
|
+
|
|
2133
|
+
|
|
2134
|
+
def build_records(extraction: dict | None, details: dict | None, files: list | None,
|
|
2135
|
+
patches: list[dict]) -> tuple[dict, dict]:
|
|
2136
|
+
"""(reextract-records.json, summary): each export's record seeded from the runner's and step 2's records, with
|
|
2137
|
+
the workers' patches merged in (see "Records" in the module docstring)."""
|
|
2138
|
+
runner, extra = {}, {}
|
|
2139
|
+
for source, doc, what in ((runner, extraction, "--extraction file"), (extra, details, "--export-details file")):
|
|
2140
|
+
for record in _exports_of(doc, what):
|
|
2141
|
+
key = (_name_of(record), _file_of(record))
|
|
2142
|
+
if all(key):
|
|
2143
|
+
source.setdefault(key, record)
|
|
2144
|
+
seeded: dict[tuple, dict] = {}
|
|
2145
|
+
languages: dict[tuple, object] = {}
|
|
2146
|
+
warnings = []
|
|
2147
|
+
|
|
2148
|
+
def fill_if_null(key: tuple, record: dict) -> None:
|
|
2149
|
+
"""Set params and return_type from `record` where the seed of `key` has none."""
|
|
2150
|
+
for field in PATCH_IF_NULL:
|
|
2151
|
+
value = record.get(field)
|
|
2152
|
+
if field == "params" and value is None:
|
|
2153
|
+
value = record.get("parameters")
|
|
2154
|
+
if seeded[key][field] is None and value is not None:
|
|
2155
|
+
seeded[key][field] = _typed_params(value, languages.get(key)) if field == "params" else value
|
|
2156
|
+
|
|
2157
|
+
for key, record in runner.items():
|
|
2158
|
+
seeded[key], languages[key] = _seed(record, key[1], read_by_eye=False), record.get("language")
|
|
2159
|
+
fill_if_null(key, extra.get(key, {}))
|
|
2160
|
+
for key, record in extra.items():
|
|
2161
|
+
if key in runner:
|
|
2162
|
+
continue
|
|
2163
|
+
if record.get("export_type", record.get("type")) is None:
|
|
2164
|
+
warnings.append(f"re-extract: {key[0]} in {key[1]}: an export-details record with no export_type the "
|
|
2165
|
+
f"runner does not hold; not added")
|
|
2166
|
+
continue
|
|
2167
|
+
seeded[key], languages[key] = _seed(record, key[1], read_by_eye=True), record.get("language")
|
|
2168
|
+
for block in patches:
|
|
2169
|
+
path = _norm_path(block.get("file_path"))
|
|
2170
|
+
for patch in block.get("exports") or []:
|
|
2171
|
+
key = (_name_of(patch) if isinstance(patch, dict) else None, path)
|
|
2172
|
+
if key not in seeded:
|
|
2173
|
+
warnings.append(f"re-extract: a patch names {key[0] or 'no export'} in {path}, which neither the "
|
|
2174
|
+
f"runner nor step 2 recorded; not added")
|
|
2175
|
+
continue
|
|
2176
|
+
for field in PATCH_FIELDS:
|
|
2177
|
+
if patch.get(field) is not None:
|
|
2178
|
+
seeded[key][field] = patch[field]
|
|
2179
|
+
fill_if_null(key, patch)
|
|
2180
|
+
surface, _problem = public_surface(extraction) # apply warns when a public-api run gives no surface
|
|
2181
|
+
for key in seeded if surface is not None else ():
|
|
2182
|
+
mark = surface.public(key[0], key[1], languages.get(key))
|
|
2183
|
+
if mark is not None:
|
|
2184
|
+
seeded[key]["public"] = mark
|
|
2185
|
+
paths = list(dict.fromkeys([p for p in (_norm_path(f) for f in files or []) if p]
|
|
2186
|
+
+ sorted({key[1] for key in seeded})))
|
|
2187
|
+
blocks = [{"file_path": path, "exports": [seeded[key] for key in sorted(seeded) if key[1] == path]}
|
|
2188
|
+
for path in paths]
|
|
2189
|
+
exports = [record for block in blocks for record in block["exports"]]
|
|
2190
|
+
summary = {
|
|
2191
|
+
"files_extracted": len(blocks),
|
|
2192
|
+
"exports_extracted": len(exports),
|
|
2193
|
+
"confidence_breakdown": {
|
|
2194
|
+
"T1": sum(1 for r in exports if r["confidence"] == "T1"),
|
|
2195
|
+
"T1-low": sum(1 for r in exports if r["confidence"] == "T1-low"),
|
|
2196
|
+
"T2": sum(1 for r in exports if r.get("qmd_evidence") is not None),
|
|
2197
|
+
},
|
|
2198
|
+
"marked_not_public": sum(1 for r in exports if r.get("public") is False),
|
|
2199
|
+
"warnings": warnings,
|
|
2200
|
+
}
|
|
2201
|
+
return {"mode": "normal", "files": blocks}, summary
|
|
2202
|
+
|
|
2203
|
+
|
|
2204
|
+
# --------------------------------------------------------------------------
|
|
2205
|
+
# Gap records (update-skill gap-driven.md §3, §4 and §4a)
|
|
2206
|
+
# --------------------------------------------------------------------------
|
|
2207
|
+
|
|
2208
|
+
|
|
2209
|
+
EXPORT_CATEGORIES = ("NEW_EXPORT", "MODIFIED_EXPORT", "MOVED_EXPORT", "DELETED_EXPORT")
|
|
2210
|
+
DOCUMENT_CATEGORIES = ("missing-export", "missing-type") # routed to §4a by category, never halting
|
|
2211
|
+
DRIFT_STATUSES = ("ok", "skipped", "overridden")
|
|
2212
|
+
# The drift gate's reasons, gap-driven.md §3: the first that fits.
|
|
2213
|
+
DRIFT_RESCOPE = "a public API recount from the tree (rule R1)"
|
|
2214
|
+
DRIFT_COMPLETENESS = "a line from the tree (rule R3)"
|
|
2215
|
+
DRIFT_NOT_IN_MAP = "a line and a signature from the tree"
|
|
2216
|
+
DRIFT_IN_MAP = "a signature from the tree"
|
|
2217
|
+
REACHABILITY = ("public", "internal-unreachable")
|
|
2218
|
+
VERIFIER_MISSING = "provenance: spot-checks not run: skf-verify-provenance-completeness.py is missing; re-install SKF"
|
|
2219
|
+
OUTCOMES = ("verified", "moved", "missing", "re-extracted", "rescoped", "unknown")
|
|
2220
|
+
_DEFINITION_LINES = re.compile(r"definition lines?\s*\(([\d,\s]+)\)", re.IGNORECASE)
|
|
2221
|
+
_VERIFIER = None
|
|
2222
|
+
|
|
2223
|
+
|
|
2224
|
+
def _verifier():
|
|
2225
|
+
"""skf-verify-provenance-completeness.py, loaded once from this folder for its definition-lines rules, or None
|
|
2226
|
+
when it cannot be loaded."""
|
|
2227
|
+
global _VERIFIER
|
|
2228
|
+
if _VERIFIER is None:
|
|
2229
|
+
path = Path(__file__).resolve().parent / "skf-verify-provenance-completeness.py"
|
|
2230
|
+
try:
|
|
2231
|
+
spec = importlib.util.spec_from_file_location("skf_verify_provenance_completeness", path)
|
|
2232
|
+
if spec is None or spec.loader is None:
|
|
2233
|
+
raise ImportError(f"cannot load {path}")
|
|
2234
|
+
module = importlib.util.module_from_spec(spec)
|
|
2235
|
+
spec.loader.exec_module(module)
|
|
2236
|
+
except (OSError, ImportError, SyntaxError):
|
|
2237
|
+
_VERIFIER = False
|
|
2238
|
+
else:
|
|
2239
|
+
_VERIFIER = module
|
|
2240
|
+
return _VERIFIER or None
|
|
2241
|
+
|
|
2242
|
+
|
|
2243
|
+
def _blocking(severity: object) -> bool:
|
|
2244
|
+
return str(severity or "").strip().lower() not in NON_BLOCKING
|
|
2245
|
+
|
|
2246
|
+
|
|
2247
|
+
def _gap_judgments(doc) -> dict:
|
|
2248
|
+
"""The answers gap-records reads from a --judgments file, checked. Raises ValueError."""
|
|
2249
|
+
if not isinstance(doc, dict) or not all(isinstance(v, dict) for v in doc.values()):
|
|
2250
|
+
raise ValueError("--judgments must map each gap id to an object of answers")
|
|
2251
|
+
for gid, answers in doc.items():
|
|
2252
|
+
line = answers.get("declaring_line")
|
|
2253
|
+
if "declaring_line" in answers and line is not None and (isinstance(line, bool) or not isinstance(line, int)):
|
|
2254
|
+
raise ValueError(f"--judgments: {gid}'s `declaring_line` must be a line number or null")
|
|
2255
|
+
if "reachability" in answers and answers["reachability"] not in REACHABILITY:
|
|
2256
|
+
raise ValueError(f"--judgments: {gid}'s `reachability` must be one of {', '.join(REACHABILITY)}")
|
|
2257
|
+
if "docstring" in answers and not isinstance(answers["docstring"], (str, type(None))):
|
|
2258
|
+
raise ValueError(f"--judgments: {gid}'s `docstring` must be a string or null")
|
|
2259
|
+
return doc
|
|
2260
|
+
|
|
2261
|
+
|
|
2262
|
+
def _drift_reason(entry: dict) -> str | None:
|
|
2263
|
+
"""The drift gate's reason an export-bearing entry needs the tree, or None for one that passes."""
|
|
2264
|
+
category = entry.get("change_category")
|
|
2265
|
+
if category == "DELETED_EXPORT":
|
|
2266
|
+
return DRIFT_RESCOPE
|
|
2267
|
+
if category not in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
2268
|
+
return None
|
|
2269
|
+
if entry.get("provenance_completeness"):
|
|
2270
|
+
return DRIFT_COMPLETENESS
|
|
2271
|
+
status = (entry.get("map_match") or {}).get("status")
|
|
2272
|
+
return DRIFT_IN_MAP if status in ("found", "ambiguous") else DRIFT_NOT_IN_MAP
|
|
2273
|
+
|
|
2274
|
+
|
|
2275
|
+
def _spot_check(root: Path | None, path: str | None, name: str, line: int | None, export_type: object,
|
|
2276
|
+
answers: dict) -> tuple[str, int | None, bool, str | None]:
|
|
2277
|
+
"""(outcome, the line a `moved` export moves to, whether it waits on a `declaring_line` answer, why the
|
|
2278
|
+
check failed)."""
|
|
2279
|
+
verifier = _verifier()
|
|
2280
|
+
if verifier is None or root is None or not path or line is None:
|
|
2281
|
+
return "unknown", None, False, None
|
|
2282
|
+
try:
|
|
2283
|
+
report = verifier.definition_lines_report(path, name, root, line,
|
|
2284
|
+
export_type if isinstance(export_type, str) else None)
|
|
2285
|
+
except Exception as exc: # noqa: BLE001 - a file it cannot read, or any verifier failure: this entry only
|
|
2286
|
+
return "unknown", None, False, f"{type(exc).__name__}: {exc}"
|
|
2287
|
+
check = report.get("line_check")
|
|
2288
|
+
if check == "file-missing":
|
|
2289
|
+
return "missing", None, False, None
|
|
2290
|
+
if check == "skipped-export-type":
|
|
2291
|
+
return "verified", None, False, None
|
|
2292
|
+
if check == "skipped-language":
|
|
2293
|
+
if "declaring_line" not in answers:
|
|
2294
|
+
return "unknown", None, True, None
|
|
2295
|
+
declared = answers["declaring_line"]
|
|
2296
|
+
if declared is None:
|
|
2297
|
+
return "unknown", None, False, None
|
|
2298
|
+
return ("verified", None, False, None) if declared == line else ("moved", declared, False, None)
|
|
2299
|
+
if report.get("line_is_definition"):
|
|
2300
|
+
return "verified", None, False, None
|
|
2301
|
+
defs = report.get("definition_lines") or []
|
|
2302
|
+
if len(defs) == 1 and defs[0] != line:
|
|
2303
|
+
return "moved", defs[0], False, None
|
|
2304
|
+
return "unknown", None, False, None
|
|
2305
|
+
|
|
2306
|
+
|
|
2307
|
+
def _pinned_lines(remediation: object) -> list[int] | None:
|
|
2308
|
+
"""The definition lines a provenance-line gap's remediation lists, `... definition line (5, 7) ...`."""
|
|
2309
|
+
m = _DEFINITION_LINES.search(remediation) if isinstance(remediation, str) else None
|
|
2310
|
+
lines = [int(n) for n in re.findall(r"\d+", m.group(1))] if m else []
|
|
2311
|
+
return lines or None
|
|
2312
|
+
|
|
2313
|
+
|
|
2314
|
+
def _where(path: object, line: object) -> str | None:
|
|
2315
|
+
file, number = _norm_path(path), _line_of(line)
|
|
2316
|
+
return f"{file}:{number}" if file and number is not None else None
|
|
2317
|
+
|
|
2318
|
+
|
|
2319
|
+
class _GapEntry:
|
|
2320
|
+
"""One export-bearing manifest entry as gap-records routes it."""
|
|
2321
|
+
|
|
2322
|
+
def __init__(self, entry: dict, answers: dict):
|
|
2323
|
+
self.entry = entry
|
|
2324
|
+
self.answers = answers
|
|
2325
|
+
self.name = _name_of(entry) or ""
|
|
2326
|
+
self.category = entry.get("change_category")
|
|
2327
|
+
match = entry.get("map_match") if isinstance(entry.get("map_match"), dict) else {}
|
|
2328
|
+
self.match_status = match.get("status") or "not-found"
|
|
2329
|
+
found = match.get("entry") if self.match_status == "found" and isinstance(match.get("entry"), dict) else None
|
|
2330
|
+
self.map_entry = {"source_file": found.get("source_file"), "source_line": found.get("source_line")} \
|
|
2331
|
+
if found else None
|
|
2332
|
+
self.map_export_type = found.get("export_type") if found else None
|
|
2333
|
+
self.in_map = bool(match.get("candidates"))
|
|
2334
|
+
citation = entry.get("source_citation")
|
|
2335
|
+
self.citation = citation if isinstance(citation, dict) else None
|
|
2336
|
+
self.recorded: tuple[str | None, int | None] = (None, None) # the file and line a check read
|
|
2337
|
+
self.outcome: str | None = None
|
|
2338
|
+
self.new_line: int | None = None
|
|
2339
|
+
self.unknown_reason: str | None = None
|
|
2340
|
+
self.reextract = False
|
|
2341
|
+
self.record: dict | None = None # the §4a record it matched
|
|
2342
|
+
self.record_path: str | None = None
|
|
2343
|
+
self.waits = False # on a declaring_line answer
|
|
2344
|
+
self.failure: str | None = None # why its spot-check failed
|
|
2345
|
+
|
|
2346
|
+
@property
|
|
2347
|
+
def blocking(self) -> bool:
|
|
2348
|
+
return _blocking(self.entry.get("severity"))
|
|
2349
|
+
|
|
2350
|
+
@property
|
|
2351
|
+
def paths(self) -> list[str]:
|
|
2352
|
+
return [p for p in (_norm_path(p) for p in self.entry.get("resolved_paths") or []) if p]
|
|
2353
|
+
|
|
2354
|
+
def check(self, root: Path | None, path: object, line: object, export_type: object) -> None:
|
|
2355
|
+
self.recorded = (_norm_path(path), _line_of(line))
|
|
2356
|
+
self.outcome, self.new_line, self.waits, self.failure = _spot_check(
|
|
2357
|
+
root, self.recorded[0], self.name, self.recorded[1], export_type, self.answers)
|
|
2358
|
+
|
|
2359
|
+
def route(self, root: Path | None, drift: bool) -> None:
|
|
2360
|
+
"""Spot-check the entry, or send it to targeted re-extraction (see "Gap records" in the module docstring)."""
|
|
2361
|
+
if self.category == "DELETED_EXPORT":
|
|
2362
|
+
self.outcome = "rescoped"
|
|
2363
|
+
if self.map_entry:
|
|
2364
|
+
self.recorded = (_norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"]))
|
|
2365
|
+
return
|
|
2366
|
+
if self.category == "MOVED_EXPORT":
|
|
2367
|
+
if self.map_entry is None:
|
|
2368
|
+
self.outcome = "unknown"
|
|
2369
|
+
return
|
|
2370
|
+
self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
|
|
2371
|
+
if drift and self.outcome == "moved":
|
|
2372
|
+
self.outcome, self.new_line, self.unknown_reason = "unknown", None, "drift-override"
|
|
2373
|
+
return
|
|
2374
|
+
if self.match_status == "found":
|
|
2375
|
+
self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
|
|
2376
|
+
return
|
|
2377
|
+
if self.match_status == "ambiguous":
|
|
2378
|
+
self.outcome = "unknown"
|
|
2379
|
+
return
|
|
2380
|
+
if self.citation:
|
|
2381
|
+
self.check(root, self.citation.get("file"), self.citation.get("line"), None)
|
|
2382
|
+
if self.waits or self.outcome in ("verified", "moved"):
|
|
2383
|
+
return
|
|
2384
|
+
if self.entry.get("provenance_completeness") and root is not None:
|
|
2385
|
+
self.reextract = True
|
|
2386
|
+
elif self.blocking:
|
|
2387
|
+
self.reextract = True
|
|
2388
|
+
elif self.entry.get("category") in DOCUMENT_CATEGORIES and self.paths:
|
|
2389
|
+
self.reextract = True
|
|
2390
|
+
else:
|
|
2391
|
+
self.outcome = "unknown"
|
|
2392
|
+
|
|
2393
|
+
def match(self, blocks: dict[str, list[dict]]) -> tuple[int, int]:
|
|
2394
|
+
"""Take the first export of the entry's name in the blocks of its resolved files; (files scanned, exports
|
|
2395
|
+
found in them)."""
|
|
2396
|
+
exports = 0
|
|
2397
|
+
for path in self.paths:
|
|
2398
|
+
for record in blocks.get(path, []):
|
|
2399
|
+
exports += 1
|
|
2400
|
+
if self.record is None and _name_of(record) == self.name:
|
|
2401
|
+
self.record, self.record_path = record, path
|
|
2402
|
+
if self.record is not None:
|
|
2403
|
+
self.outcome = "re-extracted"
|
|
2404
|
+
return len(self.paths), exports
|
|
2405
|
+
|
|
2406
|
+
@property
|
|
2407
|
+
def located(self) -> bool:
|
|
2408
|
+
return self.outcome in ("verified", "moved", "re-extracted")
|
|
2409
|
+
|
|
2410
|
+
@property
|
|
2411
|
+
def internal(self) -> bool:
|
|
2412
|
+
return self.category == "NEW_EXPORT" and self.answers.get("reachability") == "internal-unreachable"
|
|
2413
|
+
|
|
2414
|
+
def needs(self) -> list[str]:
|
|
2415
|
+
"""The answers the entry still waits on once routed."""
|
|
2416
|
+
want = []
|
|
2417
|
+
if self.category == "NEW_EXPORT" and self.located and "reachability" not in self.answers:
|
|
2418
|
+
want.append("reachability")
|
|
2419
|
+
if self.outcome == "re-extracted" and "docstring" not in self.answers:
|
|
2420
|
+
want.append("docstring")
|
|
2421
|
+
return want
|
|
2422
|
+
|
|
2423
|
+
def new_location(self) -> str | None:
|
|
2424
|
+
if self.outcome == "moved":
|
|
2425
|
+
return _where(self.recorded[0], self.new_line)
|
|
2426
|
+
if self.outcome == "re-extracted":
|
|
2427
|
+
return (self.record or {}).get("location")
|
|
2428
|
+
return None
|
|
2429
|
+
|
|
2430
|
+
def verification(self) -> dict:
|
|
2431
|
+
recorded = _where(*self.recorded)
|
|
2432
|
+
if self.outcome == "re-extracted":
|
|
2433
|
+
citation = self.new_location()
|
|
2434
|
+
elif self.outcome in ("verified", "moved", "missing", "rescoped") and recorded:
|
|
2435
|
+
citation = recorded
|
|
2436
|
+
else:
|
|
2437
|
+
citation = "unknown"
|
|
2438
|
+
return {
|
|
2439
|
+
"export_name": self.name,
|
|
2440
|
+
"gap_category": self.category,
|
|
2441
|
+
"severity": self.entry.get("severity"),
|
|
2442
|
+
"verification": self.outcome,
|
|
2443
|
+
"in_map": self.in_map,
|
|
2444
|
+
"map_entry": self.map_entry,
|
|
2445
|
+
"provenance_citation": citation,
|
|
2446
|
+
"source_citation": self.citation,
|
|
2447
|
+
"new_location": self.new_location(),
|
|
2448
|
+
"unknown_reason": self.unknown_reason,
|
|
2449
|
+
"pinned_definition_lines": _pinned_lines(self.entry.get("remediation"))
|
|
2450
|
+
if self.category == "MOVED_EXPORT" else None,
|
|
2451
|
+
"resolution_source": "remediation-paths" if self.outcome == "re-extracted" else None,
|
|
2452
|
+
"reachability": self.answers.get("reachability") if self.category == "NEW_EXPORT" and self.located
|
|
2453
|
+
else None,
|
|
2454
|
+
"export_type": None,
|
|
2455
|
+
}
|
|
2456
|
+
|
|
2457
|
+
def label(self, provenance: list[dict]) -> str:
|
|
2458
|
+
"""T1, T1-low or unlabeled: the confidence_breakdown bin of the entry."""
|
|
2459
|
+
if self.internal:
|
|
2460
|
+
return "unlabeled"
|
|
2461
|
+
method = None
|
|
2462
|
+
if self.outcome == "re-extracted":
|
|
2463
|
+
method = (self.record or {}).get("extraction_method")
|
|
2464
|
+
elif self.map_entry is not None:
|
|
2465
|
+
file, line = _norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"])
|
|
2466
|
+
for entry in provenance:
|
|
2467
|
+
if (_name_of(entry) == self.name and _file_of(entry) == file
|
|
2468
|
+
and _line_of(entry.get("source_line")) == line):
|
|
2469
|
+
method = entry.get("extraction_method")
|
|
2470
|
+
break
|
|
2471
|
+
elif self.located and self.category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
2472
|
+
method = "source-read" # a pinned cited export the map lacks: write.md §3 writes it source-read
|
|
2473
|
+
if method in ("ast-grep", "ast_bridge"):
|
|
2474
|
+
return "T1"
|
|
2475
|
+
if method in ("source-read", "source_reading"):
|
|
2476
|
+
return "T1-low"
|
|
2477
|
+
return "unlabeled"
|
|
2478
|
+
|
|
2479
|
+
|
|
2480
|
+
def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path | None, drift_status: str = "ok",
|
|
2481
|
+
judgments: dict | None = None, remediation: dict | None = None, plan: bool = False,
|
|
2482
|
+
tier: str | None = None) -> tuple[dict | None, dict]:
|
|
2483
|
+
"""(reextract-records.json or None, the summary): see "Gap records" in the module docstring."""
|
|
2484
|
+
if not isinstance(manifest, dict) or not isinstance(manifest.get("entries"), list):
|
|
2485
|
+
raise ValueError("the change manifest has no `entries` array: is it translate's output?")
|
|
2486
|
+
entries = [e for e in manifest["entries"] if isinstance(e, dict)]
|
|
2487
|
+
judgments = judgments or {}
|
|
2488
|
+
root = source_root if source_root is not None and source_root.is_dir() else None
|
|
2489
|
+
forwarded = [{"gap_id": e.get("gap_id"), "change_category": e.get("change_category")}
|
|
2490
|
+
for e in entries if e.get("change_category") not in EXPORT_CATEGORIES]
|
|
2491
|
+
routed = [_GapEntry(e, judgments.get(e.get("gap_id")) or {}) for e in entries
|
|
2492
|
+
if e.get("change_category") in EXPORT_CATEGORIES]
|
|
2493
|
+
drift = drift_status == "overridden"
|
|
2494
|
+
if drift:
|
|
2495
|
+
blocked = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "change_category": g.category,
|
|
2496
|
+
"severity": g.entry.get("severity"), "reason": _drift_reason(g.entry)}
|
|
2497
|
+
for g in routed if _drift_reason(g.entry)]
|
|
2498
|
+
if blocked:
|
|
2499
|
+
return None, {"status": "drift-blocked", "drift_blocked": blocked}
|
|
2500
|
+
bare = [g.name for g in routed if g.category == "DELETED_EXPORT" and not (
|
|
2501
|
+
isinstance(g.entry.get("rescope"), dict) and g.entry["rescope"].get("amendment")
|
|
2502
|
+
and g.entry["rescope"].get("exclude"))]
|
|
2503
|
+
if bare:
|
|
2504
|
+
return None, {"status": "blocked", "rescope_without_amendment": bare}
|
|
2505
|
+
warnings = []
|
|
2506
|
+
if root is not None and _verifier() is None:
|
|
2507
|
+
warnings.append(VERIFIER_MISSING)
|
|
2508
|
+
for g in routed:
|
|
2509
|
+
g.route(root, drift)
|
|
2510
|
+
warnings += [f"provenance: spot-check failed for {g.name}: {g.failure}" for g in routed if g.failure]
|
|
2511
|
+
waiting = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": ["declaring_line"],
|
|
2512
|
+
"file": g.recorded[0], "line": g.recorded[1]} for g in routed if g.waits]
|
|
2513
|
+
if waiting:
|
|
2514
|
+
return None, {"status": "needs-judgment", "needs_judgment": waiting, "warnings": warnings}
|
|
2515
|
+
to_scan = [g for g in routed if g.reextract]
|
|
2516
|
+
files = list(dict.fromkeys(p for g in to_scan for p in g.paths))
|
|
2517
|
+
if plan:
|
|
2518
|
+
return None, {"status": "planned", "reextract": {
|
|
2519
|
+
"entries": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
|
|
2520
|
+
"resolved_paths": g.paths} for g in to_scan], "files": files}, "warnings": warnings}
|
|
2521
|
+
blocks: dict[str, list[dict]] = {}
|
|
2522
|
+
scanned = 0
|
|
2523
|
+
if files:
|
|
2524
|
+
if remediation is None:
|
|
2525
|
+
raise ValueError("targeted re-extraction has files to match: pass --remediation-records, the output "
|
|
2526
|
+
"of `records` over them")
|
|
2527
|
+
if not isinstance(remediation, dict) or not isinstance(remediation.get("files"), list):
|
|
2528
|
+
raise ValueError("--remediation-records has no `files` array: is it the output of `records`?")
|
|
2529
|
+
for block in remediation["files"]:
|
|
2530
|
+
if isinstance(block, dict) and _norm_path(block.get("file_path")):
|
|
2531
|
+
scanned += 1
|
|
2532
|
+
blocks.setdefault(_norm_path(block["file_path"]), []).extend(
|
|
2533
|
+
r for r in block.get("exports") or [] if isinstance(r, dict))
|
|
2534
|
+
unresolved = []
|
|
2535
|
+
for g in to_scan:
|
|
2536
|
+
files_scanned, found = g.match(blocks)
|
|
2537
|
+
if g.record is not None:
|
|
2538
|
+
continue
|
|
2539
|
+
if not g.blocking and g.entry.get("category") in DOCUMENT_CATEGORIES:
|
|
2540
|
+
g.outcome = "unknown"
|
|
2541
|
+
continue
|
|
2542
|
+
unresolved.append({"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
|
|
2543
|
+
"remediation_paths": g.entry.get("remediation_paths") or [],
|
|
2544
|
+
"rejected_paths": g.entry.get("rejected_paths") or [],
|
|
2545
|
+
"files_scanned": files_scanned, "exports_found_in_scan": found})
|
|
2546
|
+
if unresolved:
|
|
2547
|
+
return None, {"status": "unresolved", "unresolved": unresolved, "warnings": warnings}
|
|
2548
|
+
needs = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": g.needs(),
|
|
2549
|
+
"file": _location(g.new_location())[0] if g.outcome != "verified" else g.recorded[0],
|
|
2550
|
+
"line": _location(g.new_location())[1] if g.outcome != "verified" else g.recorded[1]}
|
|
2551
|
+
for g in routed if g.needs()]
|
|
2552
|
+
if needs:
|
|
2553
|
+
return None, {"status": "needs-judgment", "needs_judgment": needs, "warnings": warnings}
|
|
2554
|
+
kept: dict[str, list[dict]] = {}
|
|
2555
|
+
for g in routed:
|
|
2556
|
+
if g.record is not None and not g.internal:
|
|
2557
|
+
record = dict(g.record)
|
|
2558
|
+
record.pop("public", None) # a repair documents and maps every record: the mark is a normal update's
|
|
2559
|
+
if g.answers.get("docstring") is not None:
|
|
2560
|
+
record["docstring"] = g.answers["docstring"]
|
|
2561
|
+
if record not in kept.setdefault(g.record_path, []):
|
|
2562
|
+
kept[g.record_path].append(record)
|
|
2563
|
+
breakdown = {"T1": 0, "T1-low": 0, "unlabeled": 0, "T2": 0}
|
|
2564
|
+
provenance_entries = _entries_of(provenance)
|
|
2565
|
+
for g in routed:
|
|
2566
|
+
breakdown[g.label(provenance_entries)] += 1
|
|
2567
|
+
breakdown["unlabeled"] += len(forwarded)
|
|
2568
|
+
records = {
|
|
2569
|
+
"mode": "gap-driven",
|
|
2570
|
+
"files_extracted": scanned,
|
|
2571
|
+
"exports_extracted": len(entries),
|
|
2572
|
+
"confidence_breakdown": breakdown,
|
|
2573
|
+
"verification": [g.verification() for g in routed],
|
|
2574
|
+
"files": [{"file_path": path, "exports": exports} for path, exports in kept.items()],
|
|
2575
|
+
}
|
|
2576
|
+
matched = [g for g in to_scan if g.record is not None]
|
|
2577
|
+
targeted = {"resolved_count": len(matched), "files_scanned": scanned,
|
|
2578
|
+
"exports_matched": len({(g.record_path, _name_of(g.record)) for g in matched}),
|
|
2579
|
+
"tier": tier} if to_scan else None
|
|
2580
|
+
summary = {
|
|
2581
|
+
"status": "written",
|
|
2582
|
+
"gap_count": len(entries),
|
|
2583
|
+
"files_extracted": scanned,
|
|
2584
|
+
"exports_extracted": len(entries),
|
|
2585
|
+
"confidence_breakdown": breakdown,
|
|
2586
|
+
"counts": {**{outcome: sum(1 for g in routed if g.outcome == outcome and not g.internal)
|
|
2587
|
+
for outcome in OUTCOMES}, "reclassified": sum(1 for g in routed if g.internal)},
|
|
2588
|
+
"reclassified": [g.name for g in routed if g.internal],
|
|
2589
|
+
"forwarded": forwarded,
|
|
2590
|
+
"targeted_reextraction": targeted,
|
|
2591
|
+
"warnings": warnings,
|
|
2592
|
+
}
|
|
2593
|
+
return records, summary
|
|
2594
|
+
|
|
2595
|
+
|
|
340
2596
|
# --------------------------------------------------------------------------
|
|
341
2597
|
# CLI
|
|
342
2598
|
# --------------------------------------------------------------------------
|
|
343
2599
|
|
|
344
2600
|
|
|
2601
|
+
def _category_c(args: argparse.Namespace, own: dict | None) -> dict | None:
|
|
2602
|
+
"""Category C from --category-c and --ccc-pairs, else the input's own
|
|
2603
|
+
(`own`, None when there is none). Raises ValueError."""
|
|
2604
|
+
if not args.category_c and not args.ccc_pairs:
|
|
2605
|
+
return own
|
|
2606
|
+
cat_c = dict(own or {})
|
|
2607
|
+
if args.category_c:
|
|
2608
|
+
doc = _load_helper_file(Path(args.category_c), "--category-c file", "category_c")
|
|
2609
|
+
if not isinstance(doc["category_c"], dict):
|
|
2610
|
+
raise ValueError(f"--category-c file {args.category_c} has no `category_c` object")
|
|
2611
|
+
cat_c = dict(doc["category_c"])
|
|
2612
|
+
if args.ccc_pairs:
|
|
2613
|
+
pairs = _load_json_file(Path(args.ccc_pairs), "--ccc-pairs file")
|
|
2614
|
+
if not isinstance(pairs, dict):
|
|
2615
|
+
raise ValueError(f"--ccc-pairs file {args.ccc_pairs} must hold a JSON object")
|
|
2616
|
+
for key in ("renamed_files", "renamed_exports"):
|
|
2617
|
+
cat_c[key] = list(_get_list(cat_c, key)) + [p for p in _get_list(pairs, key) if isinstance(p, dict)]
|
|
2618
|
+
return cat_c
|
|
2619
|
+
|
|
2620
|
+
|
|
2621
|
+
def _payload(args: argparse.Namespace) -> dict:
|
|
2622
|
+
"""The category JSON of a build or deletion-ratio call: the input with
|
|
2623
|
+
the helper files' slices assembled in. Raises ValueError."""
|
|
2624
|
+
payload = _load_input(Path(args.input) if args.input else None)
|
|
2625
|
+
cat_c = _category_c(args, payload.get("category_c") if isinstance(payload.get("category_c"), dict) else None)
|
|
2626
|
+
if cat_c is not None:
|
|
2627
|
+
payload = {**payload, "category_c": cat_c}
|
|
2628
|
+
category_a_doc = (
|
|
2629
|
+
_load_helper_file(Path(args.category_a), "--category-a file", "category_a") if args.category_a else None
|
|
2630
|
+
)
|
|
2631
|
+
diff = _load_helper_file(Path(args.category_b_diff), "--category-b-diff file", "removed") \
|
|
2632
|
+
if args.category_b_diff else None
|
|
2633
|
+
return assemble(payload, category_a_doc=category_a_doc, diff=diff)
|
|
2634
|
+
|
|
2635
|
+
|
|
2636
|
+
def docs_only_manifest(doc_hashes: dict) -> dict:
|
|
2637
|
+
"""A docs-only skill's change manifest from compare-hashes' output."""
|
|
2638
|
+
def urls(key: str) -> list[str]:
|
|
2639
|
+
return [e["url"] for e in _get_list(doc_hashes, key) if isinstance(e, dict) and isinstance(e.get("url"), str)]
|
|
2640
|
+
|
|
2641
|
+
changed, failed = urls("changed"), urls("fetch_failed")
|
|
2642
|
+
return {"mode": "docs-only", "no_changes": not changed, "changed_urls": changed, "fetch_failed": failed,
|
|
2643
|
+
"counts": {"docs_changed": len(changed), "docs_fetch_failed": len(failed)}}
|
|
2644
|
+
|
|
2645
|
+
|
|
2646
|
+
def _category_d(args: argparse.Namespace) -> dict | None:
|
|
2647
|
+
"""Category D from --file-compare (typed by --provenance-map) and --new-files, else None. Raises ValueError."""
|
|
2648
|
+
if not args.file_compare and not args.new_files:
|
|
2649
|
+
if args.provenance_map:
|
|
2650
|
+
raise ValueError("--provenance-map goes with --file-compare")
|
|
2651
|
+
return None
|
|
2652
|
+
if args.file_compare and not args.provenance_map:
|
|
2653
|
+
raise ValueError("--file-compare needs --provenance-map, which types each row by its file_entries[] row")
|
|
2654
|
+
compare = _load_helper_file(Path(args.file_compare), "--file-compare file", "comparisons") \
|
|
2655
|
+
if args.file_compare else None
|
|
2656
|
+
new_files = _load_helper_file(Path(args.new_files), "--new-files file", "new_files") if args.new_files else None
|
|
2657
|
+
provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file") if args.provenance_map else None
|
|
2658
|
+
if provenance is not None and not isinstance(provenance, dict):
|
|
2659
|
+
raise ValueError(f"--provenance-map file {args.provenance_map} must hold a JSON object")
|
|
2660
|
+
return category_d_from_files(compare, new_files, provenance)
|
|
2661
|
+
|
|
2662
|
+
|
|
345
2663
|
def _cmd_build(args: argparse.Namespace) -> int:
|
|
346
2664
|
try:
|
|
347
|
-
|
|
2665
|
+
if args.doc_hashes:
|
|
2666
|
+
if (args.input or args.category_a or args.category_b_diff or args.category_c or args.ccc_pairs
|
|
2667
|
+
or args.file_compare or args.new_files or args.provenance_map):
|
|
2668
|
+
raise ValueError("--doc-hashes takes no other input")
|
|
2669
|
+
doc_hashes = _load_helper_file(Path(args.doc_hashes), "--doc-hashes file", "changed")
|
|
2670
|
+
manifest = docs_only_manifest(doc_hashes)
|
|
2671
|
+
else:
|
|
2672
|
+
payload = _payload(args)
|
|
2673
|
+
cat_d = _category_d(args)
|
|
2674
|
+
if cat_d is not None:
|
|
2675
|
+
payload = {**payload, "category_d": cat_d}
|
|
2676
|
+
manifest = build_manifest(payload)
|
|
348
2677
|
except ValueError as exc:
|
|
349
2678
|
print(f"error: {exc}", file=sys.stderr)
|
|
350
2679
|
return 1
|
|
351
|
-
manifest = build_manifest(payload)
|
|
352
2680
|
json.dump(manifest, sys.stdout, indent=2)
|
|
353
2681
|
sys.stdout.write("\n")
|
|
354
2682
|
return 0
|
|
@@ -356,7 +2684,7 @@ def _cmd_build(args: argparse.Namespace) -> int:
|
|
|
356
2684
|
|
|
357
2685
|
def _cmd_deletion_ratio(args: argparse.Namespace) -> int:
|
|
358
2686
|
try:
|
|
359
|
-
payload =
|
|
2687
|
+
payload = _payload(args)
|
|
360
2688
|
except ValueError as exc:
|
|
361
2689
|
print(f"error: {exc}", file=sys.stderr)
|
|
362
2690
|
return 1
|
|
@@ -379,6 +2707,271 @@ def _cmd_deletion_ratio(args: argparse.Namespace) -> int:
|
|
|
379
2707
|
return 0
|
|
380
2708
|
|
|
381
2709
|
|
|
2710
|
+
def _optional_json(value: str | None, what: str):
|
|
2711
|
+
return _load_json_file(Path(value), what) if value else None
|
|
2712
|
+
|
|
2713
|
+
|
|
2714
|
+
def _emit(result: dict, output: str | None) -> None:
|
|
2715
|
+
if output:
|
|
2716
|
+
_write_json_atomic(Path(output), result)
|
|
2717
|
+
print(json.dumps({"status": "ok", "output": output}))
|
|
2718
|
+
else:
|
|
2719
|
+
json.dump(result, sys.stdout, indent=2)
|
|
2720
|
+
sys.stdout.write("\n")
|
|
2721
|
+
|
|
2722
|
+
|
|
2723
|
+
def _cmd_rename_candidates(args: argparse.Namespace) -> int:
|
|
2724
|
+
try:
|
|
2725
|
+
category_a_doc = _load_helper_file(Path(args.category_a), "--category-a file", "category_a")
|
|
2726
|
+
diff = _load_helper_file(Path(args.category_b_diff), "--category-b-diff file", "removed") \
|
|
2727
|
+
if args.category_b_diff else None
|
|
2728
|
+
provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file")
|
|
2729
|
+
result = rename_candidates(
|
|
2730
|
+
category_a_doc, diff, provenance,
|
|
2731
|
+
_optional_json(args.extraction, "--extraction file"),
|
|
2732
|
+
_optional_json(args.export_details, "--export-details file"),
|
|
2733
|
+
args.tier,
|
|
2734
|
+
source_root=Path(args.source_root) if args.source_root else None,
|
|
2735
|
+
sizes_commit=args.sizes_commit or None,
|
|
2736
|
+
)
|
|
2737
|
+
_emit(result, args.output)
|
|
2738
|
+
except ValueError as exc:
|
|
2739
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2740
|
+
return 1
|
|
2741
|
+
except OSError as exc:
|
|
2742
|
+
print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
|
|
2743
|
+
return 1
|
|
2744
|
+
return 0
|
|
2745
|
+
|
|
2746
|
+
|
|
2747
|
+
def _apply_categories(args: argparse.Namespace) -> dict | None:
|
|
2748
|
+
"""The category JSON apply reads: --input, with --category-c and --ccc-pairs in place of its category_c."""
|
|
2749
|
+
categories = _optional_json(args.input, "--input file")
|
|
2750
|
+
if categories is not None and not isinstance(categories, dict):
|
|
2751
|
+
raise ValueError("--input file must hold a JSON object")
|
|
2752
|
+
own = (categories or {}).get("category_c")
|
|
2753
|
+
cat_c = _category_c(args, own if isinstance(own, dict) else None)
|
|
2754
|
+
if cat_c is None:
|
|
2755
|
+
return categories
|
|
2756
|
+
return {**(categories or {}), "category_c": cat_c}
|
|
2757
|
+
|
|
2758
|
+
|
|
2759
|
+
def _metadata_in(path: str | None) -> dict | None:
|
|
2760
|
+
"""apply's --metadata-in: the skill's metadata.json, an object."""
|
|
2761
|
+
if not path:
|
|
2762
|
+
return None
|
|
2763
|
+
metadata = _load_json_file(Path(path), "--metadata-in file")
|
|
2764
|
+
if not isinstance(metadata, dict):
|
|
2765
|
+
raise ValueError("--metadata-in file must hold a JSON object (the skill's metadata.json)")
|
|
2766
|
+
return metadata
|
|
2767
|
+
|
|
2768
|
+
|
|
2769
|
+
def _cmd_metadata(args: argparse.Namespace) -> int:
|
|
2770
|
+
target = Path(args.metadata)
|
|
2771
|
+
try:
|
|
2772
|
+
metadata = update_metadata(
|
|
2773
|
+
_load_json_file(target, "--metadata file"), _load_json_file(Path(args.lists), "--lists file"),
|
|
2774
|
+
_load_json_file(Path(args.stats), "--stats file"), version=args.version,
|
|
2775
|
+
generation_date=args.generation_date, source_commit=args.source_commit, source_ref=args.source_ref)
|
|
2776
|
+
_write_json_atomic(target, metadata)
|
|
2777
|
+
except ValueError as exc:
|
|
2778
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2779
|
+
return 1
|
|
2780
|
+
except OSError as exc:
|
|
2781
|
+
print(f"error: cannot write {args.metadata}: {exc}", file=sys.stderr)
|
|
2782
|
+
return 1
|
|
2783
|
+
print(json.dumps({"status": "written", "metadata": target.as_posix(), "version": metadata["version"],
|
|
2784
|
+
"exports": len(metadata["exports"]), "scripts": len(metadata.get("scripts") or []),
|
|
2785
|
+
"assets": len(metadata.get("assets") or [])}))
|
|
2786
|
+
return 0
|
|
2787
|
+
|
|
2788
|
+
|
|
2789
|
+
def _cmd_apply(args: argparse.Namespace) -> int:
|
|
2790
|
+
if bool(args.drift_head) != bool(args.drift_pinned):
|
|
2791
|
+
print("error: --drift-head and --drift-pinned go together", file=sys.stderr)
|
|
2792
|
+
return 1
|
|
2793
|
+
if bool(args.metadata_in) != bool(args.lists_out):
|
|
2794
|
+
print("error: --metadata-in and --lists-out go together", file=sys.stderr)
|
|
2795
|
+
return 1
|
|
2796
|
+
try:
|
|
2797
|
+
promoted = _optional_json(args.promoted_docs, "--promoted-docs file")
|
|
2798
|
+
if promoted is not None and not isinstance(promoted, list):
|
|
2799
|
+
raise ValueError("--promoted-docs file must hold a JSON array")
|
|
2800
|
+
doc, summary = apply_update(
|
|
2801
|
+
update_type=args.update_type,
|
|
2802
|
+
provenance=_optional_json(args.provenance_map, "--provenance-map file"),
|
|
2803
|
+
skill_name=args.skill_name,
|
|
2804
|
+
manifest=_optional_json(args.manifest, "--manifest file"),
|
|
2805
|
+
categories=_apply_categories(args),
|
|
2806
|
+
extraction=_optional_json(args.extraction, "--extraction file"),
|
|
2807
|
+
details=_optional_json(args.export_details, "--export-details file"),
|
|
2808
|
+
records=_optional_json(args.reextract_records, "--reextract-records file"),
|
|
2809
|
+
merge_records=_optional_json(args.merge_records, "--merge-records file"),
|
|
2810
|
+
compare=_optional_json(args.file_compare, "--file-compare file"),
|
|
2811
|
+
new_files=_optional_json(args.new_files, "--new-files file"),
|
|
2812
|
+
promoted=promoted,
|
|
2813
|
+
source_root=Path(args.source_root) if args.source_root else None,
|
|
2814
|
+
generation_date=args.generation_date,
|
|
2815
|
+
test_report_run_id=args.test_report_run_id,
|
|
2816
|
+
confidence_tier=args.confidence_tier,
|
|
2817
|
+
manual_sections_preserved=args.manual_sections_preserved,
|
|
2818
|
+
source_commit=args.source_commit,
|
|
2819
|
+
source_ref=args.source_ref,
|
|
2820
|
+
drift=(args.drift_head, args.drift_pinned) if args.drift_head else None,
|
|
2821
|
+
metadata=_metadata_in(args.metadata_in),
|
|
2822
|
+
)
|
|
2823
|
+
except ValueError as exc:
|
|
2824
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2825
|
+
return 1
|
|
2826
|
+
if summary["status"] == "refused":
|
|
2827
|
+
print(json.dumps(summary))
|
|
2828
|
+
return EXIT_REFUSED
|
|
2829
|
+
lists = summary.pop("lists", None)
|
|
2830
|
+
for path, value in ((args.not_public_out, summary["not_public"]), (args.lists_out, lists),
|
|
2831
|
+
(args.output, doc)):
|
|
2832
|
+
try:
|
|
2833
|
+
if path:
|
|
2834
|
+
_write_json_atomic(Path(path), value)
|
|
2835
|
+
except OSError as exc:
|
|
2836
|
+
print(f"error: cannot write {path}: {exc}", file=sys.stderr)
|
|
2837
|
+
return 1
|
|
2838
|
+
print(json.dumps({**summary, "map": args.output}))
|
|
2839
|
+
return 0
|
|
2840
|
+
|
|
2841
|
+
|
|
2842
|
+
def _cmd_records(args: argparse.Namespace) -> int:
|
|
2843
|
+
try:
|
|
2844
|
+
files = _optional_json(args.files_from, "--files-from file")
|
|
2845
|
+
if files is not None and not (isinstance(files, list) and all(isinstance(f, str) for f in files)):
|
|
2846
|
+
raise ValueError(f"--files-from file {args.files_from} must hold a JSON array of paths")
|
|
2847
|
+
records, summary = build_records(
|
|
2848
|
+
_optional_json(args.extraction, "--extraction file"),
|
|
2849
|
+
_optional_json(args.export_details, "--export-details file"),
|
|
2850
|
+
files,
|
|
2851
|
+
_patch_blocks(Path(args.patches)) if args.patches else [],
|
|
2852
|
+
)
|
|
2853
|
+
_write_json_atomic(Path(args.output), records)
|
|
2854
|
+
except ValueError as exc:
|
|
2855
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2856
|
+
return 1
|
|
2857
|
+
except OSError as exc:
|
|
2858
|
+
print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
|
|
2859
|
+
return 1
|
|
2860
|
+
print(json.dumps({"status": "written", "output": args.output, **summary}))
|
|
2861
|
+
return 0
|
|
2862
|
+
|
|
2863
|
+
|
|
2864
|
+
def _cmd_gap_records(args: argparse.Namespace) -> int:
|
|
2865
|
+
try:
|
|
2866
|
+
if args.plan and args.output:
|
|
2867
|
+
raise ValueError("--plan writes no records: drop -o")
|
|
2868
|
+
if not args.plan and not args.output:
|
|
2869
|
+
raise ValueError("-o is required without --plan")
|
|
2870
|
+
if args.files_out and not args.plan:
|
|
2871
|
+
raise ValueError("--files-out goes with --plan")
|
|
2872
|
+
if bool(args.evidence) != bool(args.tier):
|
|
2873
|
+
raise ValueError("--evidence and --tier go together")
|
|
2874
|
+
judgments = _gap_judgments(_load_json_file(Path(args.judgments), "--judgments file")) \
|
|
2875
|
+
if args.judgments else {}
|
|
2876
|
+
records, summary = gap_records(
|
|
2877
|
+
_load_json_file(Path(args.manifest), "--manifest file"),
|
|
2878
|
+
_load_json_file(Path(args.provenance_map), "--provenance-map file"),
|
|
2879
|
+
source_root=Path(args.source_root) if args.source_root else None,
|
|
2880
|
+
drift_status=args.drift_status,
|
|
2881
|
+
judgments=judgments,
|
|
2882
|
+
remediation=_optional_json(args.remediation_records, "--remediation-records file"),
|
|
2883
|
+
plan=args.plan,
|
|
2884
|
+
tier=args.tier,
|
|
2885
|
+
)
|
|
2886
|
+
if summary["status"] == "planned" and args.files_out:
|
|
2887
|
+
_write_json_atomic(Path(args.files_out), summary["reextract"]["files"])
|
|
2888
|
+
if records is not None:
|
|
2889
|
+
_write_json_atomic(Path(args.output), records)
|
|
2890
|
+
summary = {"status": summary["status"], "output": args.output,
|
|
2891
|
+
**{k: v for k, v in summary.items() if k != "status"}}
|
|
2892
|
+
if args.evidence and summary["targeted_reextraction"] is not None:
|
|
2893
|
+
with open(args.evidence, "a", encoding="utf-8", newline="\n") as fh:
|
|
2894
|
+
fh.write(json.dumps({"targeted_reextraction": summary["targeted_reextraction"]}) + "\n")
|
|
2895
|
+
except ValueError as exc:
|
|
2896
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2897
|
+
return 1
|
|
2898
|
+
except OSError as exc:
|
|
2899
|
+
print(f"error: cannot write: {exc}", file=sys.stderr)
|
|
2900
|
+
return 1
|
|
2901
|
+
print(json.dumps(summary, indent=2))
|
|
2902
|
+
return 0
|
|
2903
|
+
|
|
2904
|
+
|
|
2905
|
+
# --------------------------------------------------------------------------
|
|
2906
|
+
# Baseline gaps (update-skill detect-changes Category B)
|
|
2907
|
+
# --------------------------------------------------------------------------
|
|
2908
|
+
|
|
2909
|
+
|
|
2910
|
+
VERIFIER_FILE = "skf-verify-provenance-completeness.py"
|
|
2911
|
+
|
|
2912
|
+
|
|
2913
|
+
def _baseline_verifier():
|
|
2914
|
+
"""The verifier beside this script, for its `baseline_gaps`. Unlike `_verifier()`, which lets a gap-driven
|
|
2915
|
+
spot-check go on as unknown, one that cannot be loaded (or has no `baseline_gaps`) is a ValueError: with no
|
|
2916
|
+
verifier no gap is listed, and every name the runner leaves out would read as a deleted export."""
|
|
2917
|
+
path = Path(__file__).resolve().parent / VERIFIER_FILE
|
|
2918
|
+
where = f"cannot load {VERIFIER_FILE} beside {Path(__file__).name}"
|
|
2919
|
+
try:
|
|
2920
|
+
spec = importlib.util.spec_from_file_location("skf_verify_provenance_completeness_baseline", path)
|
|
2921
|
+
if spec is None or spec.loader is None or not path.is_file():
|
|
2922
|
+
raise ImportError(f"no file at {path}")
|
|
2923
|
+
module = importlib.util.module_from_spec(spec)
|
|
2924
|
+
spec.loader.exec_module(module)
|
|
2925
|
+
except Exception as exc: # whatever the script raises, the call reports it
|
|
2926
|
+
raise ValueError(f"{where}: {exc}; re-install SKF") from exc
|
|
2927
|
+
if not callable(getattr(module, "baseline_gaps", None)):
|
|
2928
|
+
raise ValueError(f"{where}: it has no baseline_gaps; re-install SKF")
|
|
2929
|
+
return module
|
|
2930
|
+
|
|
2931
|
+
|
|
2932
|
+
def baseline_gaps(provenance: dict, extraction: dict, files: list, source_root: Path, verifier) -> dict:
|
|
2933
|
+
"""{"gaps", "unchecked"}: the map entries of `files` the runner's `extraction` lacks whose file still declares
|
|
2934
|
+
them (see "Baseline gaps" in the module docstring)."""
|
|
2935
|
+
if not isinstance(extraction, dict):
|
|
2936
|
+
raise ValueError("--extraction file has no `exports` array: is it the helper's output?")
|
|
2937
|
+
held = {(name, file) for e in _exports_of(extraction, "--extraction file")
|
|
2938
|
+
if (name := _name_of(e)) is not None and (file := _file_of(e)) is not None}
|
|
2939
|
+
diff = extraction.get("entry_point_diff")
|
|
2940
|
+
listed = {(item["name"], _norm_path(item["file"]))
|
|
2941
|
+
for item in (diff.get("extraction_gaps") or [] if isinstance(diff, dict) else [])
|
|
2942
|
+
if isinstance(item, dict) and isinstance(item.get("name"), str) and _norm_path(item.get("file"))}
|
|
2943
|
+
if provenance.get("entries") is not None and not isinstance(provenance["entries"], list):
|
|
2944
|
+
raise ValueError("provenance `entries` must be an array") # missing or null: no entry (#684)
|
|
2945
|
+
wanted = {f for f in (_norm_path(p) for p in files) if f is not None}
|
|
2946
|
+
gaps, unchecked = verifier.baseline_gaps(provenance, source_root, held, wanted, listed)
|
|
2947
|
+
return {"gaps": gaps, "unchecked": unchecked}
|
|
2948
|
+
|
|
2949
|
+
|
|
2950
|
+
def _cmd_baseline_gaps(args: argparse.Namespace) -> int:
|
|
2951
|
+
try:
|
|
2952
|
+
source_root = Path(args.source_root)
|
|
2953
|
+
if not source_root.is_dir():
|
|
2954
|
+
raise ValueError(f"--source-root {args.source_root} is not a folder")
|
|
2955
|
+
files = _load_json_file(Path(args.files), "--files file")
|
|
2956
|
+
if not (isinstance(files, list) and all(isinstance(f, str) for f in files)):
|
|
2957
|
+
raise ValueError(f"--files file {args.files} must hold a JSON array of paths")
|
|
2958
|
+
provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file")
|
|
2959
|
+
if not isinstance(provenance, dict):
|
|
2960
|
+
raise ValueError(f"--provenance-map file {args.provenance_map} must hold a JSON object")
|
|
2961
|
+
extraction = _load_json_file(Path(args.extraction), "--extraction file")
|
|
2962
|
+
result = baseline_gaps(provenance, extraction, files, source_root, _baseline_verifier())
|
|
2963
|
+
_write_json_atomic(Path(args.output), result)
|
|
2964
|
+
except ValueError as exc:
|
|
2965
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2966
|
+
return 1
|
|
2967
|
+
except OSError as exc:
|
|
2968
|
+
print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
|
|
2969
|
+
return 1
|
|
2970
|
+
print(json.dumps({"status": "written", "output": args.output, "gaps": len(result["gaps"]),
|
|
2971
|
+
"unchecked": len(result["unchecked"])}))
|
|
2972
|
+
return 0
|
|
2973
|
+
|
|
2974
|
+
|
|
382
2975
|
def _build_parser() -> argparse.ArgumentParser:
|
|
383
2976
|
parser = argparse.ArgumentParser(
|
|
384
2977
|
prog="skf-build-change-manifest",
|
|
@@ -390,23 +2983,155 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
390
2983
|
sub = parser.add_subparsers(dest="cmd", required=True)
|
|
391
2984
|
|
|
392
2985
|
p_build = sub.add_parser("build", help="aggregate categories into manifest")
|
|
393
|
-
p_build.add_argument(
|
|
394
|
-
|
|
395
|
-
|
|
2986
|
+
p_build.add_argument("--doc-hashes", metavar="FILE",
|
|
2987
|
+
help="a docs-only skill: skf-detect-docs.py compare-hashes output, the only input")
|
|
2988
|
+
p_build.add_argument("--file-compare", metavar="FILE",
|
|
2989
|
+
help="skf-hash-content.py compare output: Category D's modified and deleted files")
|
|
2990
|
+
p_build.add_argument("--new-files", metavar="FILE", help="skf-new-file-diff.py output: Category D's new files")
|
|
2991
|
+
p_build.add_argument("--provenance-map", metavar="FILE",
|
|
2992
|
+
help="the map --file-compare compared: each row's file_type")
|
|
396
2993
|
p_build.set_defaults(func=_cmd_build)
|
|
397
2994
|
|
|
398
2995
|
p_ratio = sub.add_parser(
|
|
399
2996
|
"deletion-ratio",
|
|
400
2997
|
help="compute §2.2 trigger; requires --provenance-map",
|
|
401
2998
|
)
|
|
402
|
-
p_ratio.add_argument(
|
|
403
|
-
"--input", default=None, help="path to JSON input (default: stdin)"
|
|
404
|
-
)
|
|
405
2999
|
p_ratio.add_argument(
|
|
406
3000
|
"--provenance-map", required=True, help="path to provenance-map.json"
|
|
407
3001
|
)
|
|
408
3002
|
p_ratio.set_defaults(func=_cmd_deletion_ratio)
|
|
409
3003
|
|
|
3004
|
+
p_rename = sub.add_parser(
|
|
3005
|
+
"rename-candidates",
|
|
3006
|
+
help="Category C by fixed rules: renamed files and renamed exports",
|
|
3007
|
+
)
|
|
3008
|
+
p_rename.add_argument("--category-a", required=True, metavar="FILE",
|
|
3009
|
+
help="skf-classify-changed-files.py classify output (deleted and added files)")
|
|
3010
|
+
p_rename.add_argument("--category-b-diff", metavar="FILE",
|
|
3011
|
+
help="skf-structural-diff.py output (removed and added exports)")
|
|
3012
|
+
p_rename.add_argument("--provenance-map", required=True, metavar="FILE",
|
|
3013
|
+
help="the provenance map: the deleted files' exports")
|
|
3014
|
+
p_rename.add_argument("--extraction", metavar="FILE", help="the recipe runner's output: the added files' exports")
|
|
3015
|
+
p_rename.add_argument("--export-details", metavar="FILE",
|
|
3016
|
+
help="the workers' export details: params, return types and exports read by eye")
|
|
3017
|
+
p_rename.add_argument("--tier", required=True, choices=TIERS, help="the forge tier, which picks the rules")
|
|
3018
|
+
p_rename.add_argument("--source-root", metavar="DIR", help="the source tree: the added files' sizes")
|
|
3019
|
+
p_rename.add_argument("--sizes-commit", metavar="COMMIT",
|
|
3020
|
+
help="the commit the deleted files' sizes are read at (git cat-file -s)")
|
|
3021
|
+
p_rename.add_argument("-o", "--output", metavar="FILE", help="write the JSON here instead of stdout")
|
|
3022
|
+
p_rename.set_defaults(func=_cmd_rename_candidates)
|
|
3023
|
+
|
|
3024
|
+
p_apply = sub.add_parser("apply", help="write the provenance map an update leaves")
|
|
3025
|
+
p_apply.add_argument("--update-type", required=True, choices=UPDATE_TYPES,
|
|
3026
|
+
help="incremental (normal mode), gap-driven, or full (degraded mode)")
|
|
3027
|
+
p_apply.add_argument("--provenance-map", metavar="FILE", help="the map the update started from")
|
|
3028
|
+
p_apply.add_argument("--manifest", metavar="FILE", help="the change manifest (build's output)")
|
|
3029
|
+
p_apply.add_argument("--input", metavar="FILE", help="the category JSON: category_c's renamed exports")
|
|
3030
|
+
p_apply.add_argument("--extraction", metavar="FILE", help="the recipe runner's output")
|
|
3031
|
+
p_apply.add_argument("--export-details", metavar="FILE", help="the workers' export details")
|
|
3032
|
+
p_apply.add_argument("--reextract-records", metavar="FILE", help="step 3's records")
|
|
3033
|
+
p_apply.add_argument("--merge-records", metavar="FILE", help="merge's records of the exports no fresh record holds")
|
|
3034
|
+
p_apply.add_argument("--file-compare", metavar="FILE", help="skf-hash-content.py compare output")
|
|
3035
|
+
p_apply.add_argument("--new-files", metavar="FILE", help="skf-new-file-diff.py output")
|
|
3036
|
+
p_apply.add_argument("--promoted-docs", metavar="FILE", help="the documents step 2 promoted, with their hashes")
|
|
3037
|
+
p_apply.add_argument("--source-root", metavar="DIR", help="the source tree: the new files' hashes")
|
|
3038
|
+
p_apply.add_argument("--skill-name", required=True, help="the skill name: source_library of a new entry")
|
|
3039
|
+
p_apply.add_argument("--generation-date", required=True, help="last_update: the generation_date written")
|
|
3040
|
+
p_apply.add_argument("--test-report-run-id", default=None, help="the run id of the test report applied")
|
|
3041
|
+
p_apply.add_argument("--confidence-tier", required=True, help="the forge tier the update ran at")
|
|
3042
|
+
p_apply.add_argument("--manual-sections-preserved", required=True, type=int,
|
|
3043
|
+
help="the [MANUAL] blocks the post-merge check found intact")
|
|
3044
|
+
p_apply.add_argument("--source-commit", default=None, help="the map's source_commit (empty: null)")
|
|
3045
|
+
p_apply.add_argument("--source-ref", default=None, help="the map's source_ref (empty: null)")
|
|
3046
|
+
p_apply.add_argument("--drift-head", default=None, help="under the drift override: HEAD's short SHA")
|
|
3047
|
+
p_apply.add_argument("--drift-pinned", default=None, help="under the drift override: the pinned short SHA")
|
|
3048
|
+
p_apply.add_argument("--not-public-out", metavar="FILE",
|
|
3049
|
+
help="also write the summary's not_public list here (a JSON array)")
|
|
3050
|
+
p_apply.add_argument("--metadata-in", metavar="FILE",
|
|
3051
|
+
help="the skill's metadata.json, whose exports[], scripts[] and assets[] the update "
|
|
3052
|
+
"starts from")
|
|
3053
|
+
p_apply.add_argument("--lists-out", metavar="FILE",
|
|
3054
|
+
help="write the exports[], scripts[] and assets[] metadata.json takes after the update "
|
|
3055
|
+
"here (a JSON object)")
|
|
3056
|
+
p_apply.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the new map")
|
|
3057
|
+
p_apply.set_defaults(func=_cmd_apply)
|
|
3058
|
+
|
|
3059
|
+
p_meta = sub.add_parser(
|
|
3060
|
+
"metadata", help="write the metadata.json an update leaves",
|
|
3061
|
+
description="Rewrite the skill's metadata.json in place. It sets version and generation_date, "
|
|
3062
|
+
"source_commit and source_ref when given, exports, scripts and assets from --lists (a script "
|
|
3063
|
+
"or asset list left out when empty), confidence_distribution from --stats and the stats "
|
|
3064
|
+
"helper's stats fields, dropping one an earlier run wrote that --stats lacks; it removes "
|
|
3065
|
+
"last_update and update_type and keeps every other field, a patched one included. It refuses "
|
|
3066
|
+
"a --stats whose scripts_count or assets_count is not the length of --lists' scripts or assets.")
|
|
3067
|
+
p_meta.add_argument("--metadata", required=True, metavar="FILE",
|
|
3068
|
+
help="the skill's metadata.json, rewritten in place")
|
|
3069
|
+
p_meta.add_argument("--lists", required=True, metavar="FILE", help="the lists apply wrote (--lists-out)")
|
|
3070
|
+
p_meta.add_argument("--stats", required=True, metavar="FILE",
|
|
3071
|
+
help="the JSON skf-render-metadata-stats.py wrote (its -o file)")
|
|
3072
|
+
p_meta.add_argument("--version", required=True, help="the version this update writes")
|
|
3073
|
+
p_meta.add_argument("--generation-date", required=True, help="the clock's UTC time, YYYY-MM-DDTHH:MM:SSZ")
|
|
3074
|
+
p_meta.add_argument("--source-commit", default=None, help="the commit this update read (an empty one: null)")
|
|
3075
|
+
p_meta.add_argument("--source-ref", default=None, help="the ref this update read (an empty one: null)")
|
|
3076
|
+
p_meta.set_defaults(func=_cmd_metadata)
|
|
3077
|
+
|
|
3078
|
+
p_records = sub.add_parser("records", help="write step 3's re-extraction records from the runner's and the "
|
|
3079
|
+
"workers' files")
|
|
3080
|
+
p_records.add_argument("--extraction", metavar="FILE", help="the recipe runner's output")
|
|
3081
|
+
p_records.add_argument("--export-details", metavar="FILE",
|
|
3082
|
+
help="step 2's export details: params and return types, and the exports read by eye")
|
|
3083
|
+
p_records.add_argument("--files-from", metavar="FILE",
|
|
3084
|
+
help="the files step 3 extracts (a JSON array), each a block even with no export")
|
|
3085
|
+
p_records.add_argument("--patches", metavar="DIR", help="the folder of the workers' per-file patches")
|
|
3086
|
+
p_records.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the records")
|
|
3087
|
+
p_records.set_defaults(func=_cmd_records)
|
|
3088
|
+
|
|
3089
|
+
p_gap = sub.add_parser("gap-records", help="spot-check, route and record a gap-driven repair's manifest entries")
|
|
3090
|
+
p_gap.add_argument("--manifest", required=True, metavar="FILE", help="skf-parse-gaps.py translate's manifest")
|
|
3091
|
+
p_gap.add_argument("--provenance-map", required=True, metavar="FILE", help="the map the repair starts from")
|
|
3092
|
+
p_gap.add_argument("--source-root", metavar="DIR", help="the source tree the spot-checks read")
|
|
3093
|
+
p_gap.add_argument("--drift-status", choices=DRIFT_STATUSES, default="ok",
|
|
3094
|
+
help="the workspace drift check's status; overridden runs the drift gate")
|
|
3095
|
+
p_gap.add_argument("--judgments", metavar="FILE", help="the answers to its needs_judgment[], keyed by gap id")
|
|
3096
|
+
p_gap.add_argument("--remediation-records", metavar="FILE",
|
|
3097
|
+
help="`records` output over the files targeted re-extraction scanned")
|
|
3098
|
+
p_gap.add_argument("--plan", action="store_true",
|
|
3099
|
+
help="print the answers it needs and the files targeted re-extraction scans; write no records")
|
|
3100
|
+
p_gap.add_argument("--files-out", metavar="FILE", help="with --plan: write those files here, a JSON array")
|
|
3101
|
+
p_gap.add_argument("--evidence", metavar="FILE",
|
|
3102
|
+
help="append the targeted_reextraction record to this JSON-lines file when it ran")
|
|
3103
|
+
p_gap.add_argument("--tier", help="with --evidence: the forge tier the record names")
|
|
3104
|
+
p_gap.add_argument("-o", "--output", metavar="FILE", help="where to write the records (without --plan)")
|
|
3105
|
+
p_gap.set_defaults(func=_cmd_gap_records)
|
|
3106
|
+
|
|
3107
|
+
p_base = sub.add_parser("baseline-gaps", help="list the map entries the runner left out that their modified "
|
|
3108
|
+
"file still declares, for Category B's step 2 to read")
|
|
3109
|
+
p_base.add_argument("--provenance-map", required=True, metavar="FILE", help="the map the update starts from")
|
|
3110
|
+
p_base.add_argument("--extraction", required=True, metavar="FILE",
|
|
3111
|
+
help="the recipe runner's output ({\"exports\": []} when it could not run)")
|
|
3112
|
+
p_base.add_argument("--files", required=True, metavar="FILE",
|
|
3113
|
+
help="the files whose entries are checked: Category A's modified-files.json")
|
|
3114
|
+
p_base.add_argument("--source-root", required=True, metavar="DIR", help="the source tree the files are read in")
|
|
3115
|
+
p_base.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the gaps")
|
|
3116
|
+
p_base.set_defaults(func=_cmd_baseline_gaps)
|
|
3117
|
+
|
|
3118
|
+
for p in (p_build, p_ratio):
|
|
3119
|
+
p.add_argument(
|
|
3120
|
+
"--input", default=None, help="path to JSON input (default: stdin)"
|
|
3121
|
+
)
|
|
3122
|
+
p.add_argument(
|
|
3123
|
+
"--category-a", metavar="FILE",
|
|
3124
|
+
help="skf-classify-changed-files.py classify output: the Category A slice and its moves",
|
|
3125
|
+
)
|
|
3126
|
+
p.add_argument(
|
|
3127
|
+
"--category-b-diff", metavar="FILE",
|
|
3128
|
+
help="skf-structural-diff.py output over the modified files: the Category B slice",
|
|
3129
|
+
)
|
|
3130
|
+
for p in (p_build, p_ratio, p_apply):
|
|
3131
|
+
p.add_argument("--category-c", metavar="FILE", help="rename-candidates output: the Category C slice")
|
|
3132
|
+
p.add_argument("--ccc-pairs", metavar="FILE",
|
|
3133
|
+
help="the CCC check's pairs, {renamed_files, renamed_exports}, added to Category C")
|
|
3134
|
+
|
|
410
3135
|
return parser
|
|
411
3136
|
|
|
412
3137
|
|