bmad-module-skill-forge 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +51 -49
- package/docs/404.md +10 -8
- package/docs/_data/pinned.yaml +22 -13
- package/docs/_internal/STABILITY.md +58 -35
- package/docs/agents.md +39 -53
- package/docs/architecture.md +65 -51
- package/docs/bmad-synergy.md +53 -51
- package/docs/campaign.md +127 -75
- package/docs/concepts.md +34 -28
- package/docs/examples.md +70 -68
- package/docs/forge-auto.md +34 -31
- package/docs/getting-started.md +118 -57
- package/docs/how-it-works.md +25 -21
- package/docs/index.md +15 -15
- package/docs/skill-model.md +70 -59
- package/docs/troubleshooting.md +135 -18
- package/docs/verifying-a-skill.md +83 -57
- package/docs/why-skf.md +24 -22
- package/docs/workflows.md +274 -110
- package/package.json +18 -10
- package/src/forger/preferences.yaml +8 -1
- package/src/knowledge/agentskills-spec.md +2 -0
- package/src/knowledge/ccc-bridge.md +40 -24
- package/src/knowledge/confidence-tiers.md +9 -5
- package/src/knowledge/overview.md +3 -3
- package/src/knowledge/progressive-capability.md +8 -8
- package/src/knowledge/provenance-tracking.md +9 -5
- package/src/knowledge/qmd-registry.md +2 -2
- package/src/knowledge/skf-knowledge-index.csv +2 -2
- package/src/knowledge/skill-lifecycle.md +18 -11
- package/src/knowledge/split-body-strategy.md +0 -3
- package/src/knowledge/tool-resolution.md +4 -2
- package/src/knowledge/version-paths.md +101 -15
- package/src/module-help.csv +3 -3
- package/src/module.yaml +10 -0
- package/src/shared/data/ast-grep-recipes.yaml +608 -0
- package/src/shared/data/ide-context-files.json +30 -0
- package/src/shared/health-check.md +32 -12
- package/src/shared/references/description-guard-protocol.md +14 -9
- package/src/shared/references/feasibility-report-schema.md +19 -13
- package/src/shared/references/headless-gate-convention.md +55 -3
- package/src/shared/references/output-contract-schema.md +19 -8
- package/src/shared/references/pipeline-contracts.md +94 -29
- package/src/shared/references/tessl-review.md +77 -0
- package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
- package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
- package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
- package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
- package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
- package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
- package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
- package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
- package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
- package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
- package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
- package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
- package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +30 -14
- package/src/shared/scripts/skf-build-change-manifest.py +2203 -24
- package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
- package/src/shared/scripts/skf-check-unit-records.py +184 -0
- package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
- package/src/shared/scripts/skf-classify-changed-files.py +694 -0
- package/src/shared/scripts/skf-comention-pairs.py +1333 -123
- package/src/shared/scripts/skf-compare-file-hashes.py +1 -1
- package/src/shared/scripts/skf-count-imports.py +1867 -0
- package/src/shared/scripts/skf-count-tokens.py +1 -1
- package/src/shared/scripts/skf-derive-assembly-shape.py +3 -2
- package/src/shared/scripts/skf-description-guard.py +117 -9
- package/src/shared/scripts/skf-detect-docs.py +892 -4
- package/src/shared/scripts/skf-detect-language.py +486 -107
- package/src/shared/scripts/skf-detect-registry.py +887 -0
- package/src/shared/scripts/skf-detect-scripts-assets.py +1 -1
- package/src/shared/scripts/skf-detect-tools.py +215 -27
- package/src/shared/scripts/skf-detect-workspaces.py +506 -30
- package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
- package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
- package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
- package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
- package/src/shared/scripts/skf-extract-public-api.py +4084 -85
- package/src/shared/scripts/skf-extraction-inventory.py +746 -0
- package/src/shared/scripts/skf-extraction-snapshot.py +539 -0
- package/src/shared/scripts/skf-fetch-temporal.py +674 -0
- package/src/shared/scripts/skf-find-cycles.py +97 -29
- package/src/shared/scripts/skf-find-test-report.py +476 -0
- package/src/shared/scripts/skf-forge-tier-rw.py +444 -203
- package/src/shared/scripts/skf-github-fetch.py +365 -0
- package/src/shared/scripts/skf-github-probe.py +661 -0
- package/src/shared/scripts/skf-hash-content.py +166 -12
- package/src/shared/scripts/skf-language-corpora.py +1 -1
- package/src/shared/scripts/skf-load-provenance.py +159 -27
- package/src/shared/scripts/skf-manifest-ops.py +1 -1
- package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
- package/src/shared/scripts/skf-merge-doc-urls.py +1 -1
- package/src/shared/scripts/skf-names-present.py +221 -0
- package/src/shared/scripts/skf-pair-intersect.py +119 -28
- package/src/shared/scripts/skf-parse-gaps.py +1384 -0
- package/src/shared/scripts/skf-preapply.py +1 -1
- package/src/shared/scripts/skf-preflight.py +161 -27
- package/src/shared/scripts/skf-promote-staged.py +430 -0
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +39 -7
- package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
- package/src/shared/scripts/skf-quick-batch.py +774 -0
- package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
- package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
- package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
- package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
- package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
- package/src/shared/scripts/skf-resolve-authoritative-files.py +128 -13
- package/src/shared/scripts/skf-resolve-package.py +523 -106
- package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
- package/src/shared/scripts/skf-run-lock.py +577 -0
- package/src/shared/scripts/skf-scan-manifests.py +998 -57
- package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
- package/src/shared/scripts/skf-severity-classify.py +712 -108
- package/src/shared/scripts/skf-shape-detect.py +298 -204
- package/src/shared/scripts/skf-shard-body.py +9 -8
- package/src/shared/scripts/skf-skill-inventory.py +2412 -89
- package/src/shared/scripts/skf-skills-module.py +472 -0
- package/src/shared/scripts/skf-source-tree.py +1796 -0
- package/src/shared/scripts/skf-structural-diff.py +670 -119
- package/src/shared/scripts/skf-tessl-review.py +706 -0
- package/src/shared/scripts/skf-update-active-symlink.py +5 -5
- package/src/shared/scripts/skf-update-run-state.py +624 -0
- package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
- package/src/shared/scripts/skf-validate-brief-schema.py +4 -1
- package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
- package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
- package/src/shared/scripts/skf-validate-output.py +712 -64
- package/src/shared/scripts/skf-validate-pins.py +1 -1
- package/src/shared/scripts/skf-verify-no-trace.py +116 -10
- package/src/shared/scripts/skf-verify-provenance-completeness.py +2919 -54
- package/src/shared/scripts/skf-write-skill-brief.py +367 -19
- package/src/shared/tool-requirements.yaml +201 -0
- package/src/skf-analyze-source/SKILL.md +41 -36
- package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
- package/src/skf-analyze-source/customize.toml +36 -24
- package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
- package/src/skf-analyze-source/references/continue.md +2 -13
- package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
- package/src/skf-analyze-source/references/generate-briefs.md +121 -55
- package/src/skf-analyze-source/references/headless-contract.md +42 -15
- package/src/skf-analyze-source/references/identify-units.md +96 -110
- package/src/skf-analyze-source/references/init.md +73 -71
- package/src/skf-analyze-source/references/map-and-detect.md +123 -105
- package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
- package/src/skf-analyze-source/references/recommend.md +51 -70
- package/src/skf-analyze-source/references/scan-project.md +60 -84
- package/src/skf-analyze-source/references/scan-root.md +39 -0
- package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
- package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
- package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
- package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
- package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
- package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
- package/src/skf-audit-skill/SKILL.md +43 -51
- package/src/skf-audit-skill/assets/drift-report-template.md +27 -3
- package/src/skf-audit-skill/customize.toml +41 -20
- package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
- package/src/skf-audit-skill/references/doc-drift.md +141 -0
- package/src/skf-audit-skill/references/headless-contract.md +29 -0
- package/src/skf-audit-skill/references/init.md +148 -210
- package/src/skf-audit-skill/references/re-index.md +96 -108
- package/src/skf-audit-skill/references/report.md +96 -33
- package/src/skf-audit-skill/references/semantic-diff.md +27 -18
- package/src/skf-audit-skill/references/severity-classify.md +79 -65
- package/src/skf-audit-skill/references/severity-rules.md +15 -6
- package/src/skf-audit-skill/references/structural-diff.md +75 -90
- package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
- package/src/skf-audit-skill/scripts/render-drift-tables.py +685 -0
- package/src/skf-brief-skill/SKILL.md +36 -25
- package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +17 -24
- package/src/skf-brief-skill/assets/skill-brief-schema.md +29 -15
- package/src/skf-brief-skill/customize.toml +48 -20
- package/src/skf-brief-skill/references/analyze-target.md +142 -125
- package/src/skf-brief-skill/references/confirm-brief.md +10 -8
- package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
- package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
- package/src/skf-brief-skill/references/gather-intent.md +87 -182
- package/src/skf-brief-skill/references/headless-args.md +84 -16
- package/src/skf-brief-skill/references/health-check.md +1 -1
- package/src/skf-brief-skill/references/invocation-contract.md +18 -13
- package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
- package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
- package/src/skf-brief-skill/references/scope-definition.md +123 -47
- package/src/skf-brief-skill/references/step-auto-brief.md +24 -80
- package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
- package/src/skf-brief-skill/references/version-resolution.md +3 -2
- package/src/skf-brief-skill/references/write-brief.md +66 -57
- package/src/skf-campaign/SKILL.md +31 -34
- package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
- package/src/skf-campaign/customize.toml +47 -16
- package/src/skf-campaign/references/campaign-contracts.md +39 -29
- package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
- package/src/skf-campaign/references/health-check.md +7 -16
- package/src/skf-campaign/references/step-01-setup.md +46 -84
- package/src/skf-campaign/references/step-02-strategy.md +31 -30
- package/src/skf-campaign/references/step-03-pins.md +26 -18
- package/src/skf-campaign/references/step-04-provenance.md +24 -23
- package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
- package/src/skf-campaign/references/step-06-batch.md +41 -34
- package/src/skf-campaign/references/step-07-capstone.md +40 -25
- package/src/skf-campaign/references/step-08-verify.md +28 -29
- package/src/skf-campaign/references/step-09-refine.md +29 -30
- package/src/skf-campaign/references/step-10-export.md +48 -36
- package/src/skf-campaign/references/step-11-maintenance.md +31 -34
- package/src/skf-campaign/references/step-resume.md +37 -68
- package/src/skf-campaign/scripts/campaign-deps.py +57 -18
- package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
- package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
- package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
- package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
- package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
- package/src/skf-campaign/scripts/campaign-report.py +204 -37
- package/src/skf-campaign/scripts/campaign-state.py +1128 -0
- package/src/skf-campaign/scripts/campaign-status.py +10 -74
- package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
- package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
- package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
- package/src/skf-campaign/templates/campaign-report-template.md +5 -1
- package/src/skf-campaign/templates/kickoff-template.md +0 -14
- package/src/skf-create-skill/SKILL.md +27 -20
- package/src/skf-create-skill/assets/compile-assembly-rules.md +6 -5
- package/src/skf-create-skill/assets/skill-sections.md +17 -6
- package/src/skf-create-skill/customize.toml +20 -8
- package/src/skf-create-skill/references/authoritative-files-protocol.md +41 -42
- package/src/skf-create-skill/references/batch-mode.md +81 -0
- package/src/skf-create-skill/references/ccc-index-check.md +27 -0
- package/src/skf-create-skill/references/compile.md +107 -29
- package/src/skf-create-skill/references/component-extraction.md +113 -91
- package/src/skf-create-skill/references/enrich.md +41 -23
- package/src/skf-create-skill/references/entry-points-by-hand.md +20 -0
- package/src/skf-create-skill/references/extract.md +200 -161
- package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +2 -2
- package/src/skf-create-skill/references/extraction-patterns.md +584 -196
- package/src/skf-create-skill/references/generate-artifacts.md +145 -93
- package/src/skf-create-skill/references/health-check.md +1 -1
- package/src/skf-create-skill/references/load-brief.md +53 -36
- package/src/skf-create-skill/references/relabel-rule.md +12 -0
- package/src/skf-create-skill/references/report.md +28 -59
- package/src/skf-create-skill/references/source-resolution-protocols.md +49 -155
- package/src/skf-create-skill/references/step-auto-shard.md +20 -93
- package/src/skf-create-skill/references/step-doc-rot.md +44 -60
- package/src/skf-create-skill/references/step-doc-sources.md +55 -18
- package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
- package/src/skf-create-skill/references/sub/fetch-docs.md +106 -57
- package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
- package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
- package/src/skf-create-skill/references/validate.md +173 -99
- package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
- package/src/skf-create-stack-skill/SKILL.md +29 -56
- package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
- package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
- package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
- package/src/skf-create-stack-skill/customize.toml +23 -12
- package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
- package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
- package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
- package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
- package/src/skf-create-stack-skill/references/generate-output.md +138 -128
- package/src/skf-create-stack-skill/references/init.md +90 -32
- package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
- package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
- package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
- package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
- package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
- package/src/skf-create-stack-skill/references/report.md +39 -33
- package/src/skf-create-stack-skill/references/validate.md +20 -39
- package/src/skf-drop-skill/SKILL.md +28 -51
- package/src/skf-drop-skill/customize.toml +44 -30
- package/src/skf-drop-skill/references/execute.md +79 -146
- package/src/skf-drop-skill/references/invocation-contract.md +64 -0
- package/src/skf-drop-skill/references/report.md +46 -30
- package/src/skf-drop-skill/references/select.md +118 -124
- package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
- package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
- package/src/skf-export-skill/SKILL.md +40 -62
- package/src/skf-export-skill/assets/managed-section-format.md +14 -50
- package/src/skf-export-skill/assets/snippet-format.md +3 -1
- package/src/skf-export-skill/customize.toml +42 -13
- package/src/skf-export-skill/references/generate-snippet.md +54 -46
- package/src/skf-export-skill/references/health-check.md +2 -0
- package/src/skf-export-skill/references/invocation-contract.md +15 -0
- package/src/skf-export-skill/references/load-skill.md +88 -62
- package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
- package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
- package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
- package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
- package/src/skf-export-skill/references/package.md +15 -73
- package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
- package/src/skf-export-skill/references/result-envelope.md +31 -16
- package/src/skf-export-skill/references/summary.md +41 -17
- package/src/skf-export-skill/references/token-report.md +4 -12
- package/src/skf-export-skill/references/update-context.md +118 -237
- package/src/skf-forger/SKILL.md +61 -37
- package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
- package/src/skf-forger/customize.toml +43 -0
- package/src/skf-forger/references/pipeline-mode.md +73 -18
- package/src/skf-forger/scripts/forge-status.py +299 -0
- package/src/skf-forger/scripts/parse-pipeline.py +322 -54
- package/src/skf-forger/scripts/pipeline-gate.py +428 -0
- package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
- package/src/skf-quick-skill/SKILL.md +54 -53
- package/src/skf-quick-skill/assets/skill-template.md +4 -2
- package/src/skf-quick-skill/customize.toml +51 -20
- package/src/skf-quick-skill/references/batch-mode.md +74 -21
- package/src/skf-quick-skill/references/compile.md +30 -31
- package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
- package/src/skf-quick-skill/references/finalize.md +69 -18
- package/src/skf-quick-skill/references/halt-contract.md +41 -29
- package/src/skf-quick-skill/references/health-check.md +5 -3
- package/src/skf-quick-skill/references/quick-extract.md +114 -45
- package/src/skf-quick-skill/references/registry-resolution.md +9 -64
- package/src/skf-quick-skill/references/resolve-target.md +116 -66
- package/src/skf-quick-skill/references/write-and-validate.md +79 -40
- package/src/skf-refine-architecture/SKILL.md +43 -53
- package/src/skf-refine-architecture/customize.toml +49 -20
- package/src/skf-refine-architecture/references/compile.md +97 -54
- package/src/skf-refine-architecture/references/exit-codes.md +30 -0
- package/src/skf-refine-architecture/references/finding-storage.md +21 -0
- package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
- package/src/skf-refine-architecture/references/improvements.md +14 -20
- package/src/skf-refine-architecture/references/init.md +98 -57
- package/src/skf-refine-architecture/references/issue-detection.md +32 -39
- package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
- package/src/skf-refine-architecture/references/report.md +65 -54
- package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
- package/src/skf-rename-skill/SKILL.md +19 -54
- package/src/skf-rename-skill/customize.toml +32 -26
- package/src/skf-rename-skill/references/execute.md +230 -184
- package/src/skf-rename-skill/references/exit-codes.md +7 -7
- package/src/skf-rename-skill/references/health-check.md +1 -7
- package/src/skf-rename-skill/references/invocation-contract.md +25 -0
- package/src/skf-rename-skill/references/report.md +63 -52
- package/src/skf-rename-skill/references/select.md +197 -115
- package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
- package/src/skf-setup/SKILL.md +26 -26
- package/src/skf-setup/customize.toml +14 -10
- package/src/skf-setup/references/auto-index.md +50 -63
- package/src/skf-setup/references/ccc-index.md +29 -89
- package/src/skf-setup/references/detect-and-tier.md +58 -66
- package/src/skf-setup/references/health-check.md +2 -0
- package/src/skf-setup/references/invocation-contract.md +14 -0
- package/src/skf-setup/references/report.md +43 -166
- package/src/skf-setup/references/tier-rules.md +5 -5
- package/src/skf-setup/references/write-config.md +34 -57
- package/src/skf-test-skill/SKILL.md +17 -57
- package/src/skf-test-skill/assets/output-section-formats.md +33 -34
- package/src/skf-test-skill/customize.toml +27 -13
- package/src/skf-test-skill/references/coherence-check.md +134 -62
- package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
- package/src/skf-test-skill/references/coverage-check.md +217 -254
- package/src/skf-test-skill/references/detect-mode.md +5 -5
- package/src/skf-test-skill/references/external-validators.md +94 -82
- package/src/skf-test-skill/references/health-check.md +6 -7
- package/src/skf-test-skill/references/init.md +163 -88
- package/src/skf-test-skill/references/invocation-contract.md +66 -0
- package/src/skf-test-skill/references/migration-section-rules.md +5 -2
- package/src/skf-test-skill/references/report.md +162 -142
- package/src/skf-test-skill/references/score.md +80 -107
- package/src/skf-test-skill/references/scoring-rules.md +41 -155
- package/src/skf-test-skill/references/source-access-protocol.md +19 -92
- package/src/skf-test-skill/references/step-hard-gate.md +51 -33
- package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
- package/src/skf-test-skill/scripts/build-result-context.py +287 -0
- package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
- package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
- package/src/skf-test-skill/scripts/compute-score.py +223 -45
- package/src/skf-test-skill/scripts/gap-ledger.py +1381 -0
- package/src/skf-test-skill/scripts/hard-gate.py +203 -0
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +760 -0
- package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
- package/src/skf-test-skill/scripts/reconcile-coverage.py +273 -79
- package/src/skf-test-skill/scripts/score-signatures.py +459 -0
- package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
- package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +127 -52
- package/src/skf-test-skill/templates/test-report-template.md +21 -15
- package/src/skf-update-skill/SKILL.md +38 -25
- package/src/skf-update-skill/customize.toml +27 -12
- package/src/skf-update-skill/references/detect-changes.md +233 -222
- package/src/skf-update-skill/references/gap-driven.md +336 -0
- package/src/skf-update-skill/references/health-check.md +31 -5
- package/src/skf-update-skill/references/init.md +250 -84
- package/src/skf-update-skill/references/invocation-contract.md +12 -0
- package/src/skf-update-skill/references/manual-section-rules.md +6 -23
- package/src/skf-update-skill/references/merge.md +180 -47
- package/src/skf-update-skill/references/re-extract.md +53 -262
- package/src/skf-update-skill/references/report.md +100 -35
- package/src/skf-update-skill/references/write.md +331 -149
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +29 -7
- package/src/skf-verify-stack/SKILL.md +23 -42
- package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
- package/src/skf-verify-stack/customize.toml +55 -22
- package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
- package/src/skf-verify-stack/references/coverage.md +67 -64
- package/src/skf-verify-stack/references/exit-codes.md +21 -8
- package/src/skf-verify-stack/references/init.md +107 -62
- package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
- package/src/skf-verify-stack/references/integrations.md +106 -65
- package/src/skf-verify-stack/references/report.md +59 -42
- package/src/skf-verify-stack/references/requirements.md +32 -9
- package/src/skf-verify-stack/references/synthesize.md +53 -39
- package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
- package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
- package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
- package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
- package/tools/cli/commands/install.js +11 -4
- package/tools/cli/commands/status.js +15 -3
- package/tools/cli/commands/update.js +11 -4
- package/tools/cli/lib/tool-check.js +320 -0
- package/tools/cli/lib/version-check.js +22 -9
- package/tools/cli/skf-cli.js +4 -1
- package/.gitattributes +0 -16
- package/.nvmrc +0 -1
- package/docs/_internal/RELEASING.md +0 -645
- package/release-audits/v1.0.0-launch-audit.md +0 -728
- package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
- package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
- package/src/skf-campaign/manifest.yaml +0 -15
- package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
- package/src/skf-create-skill/references/ecosystem-check.md +0 -74
- package/src/skf-drop-skill/references/headless-contract.md +0 -18
- package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
- package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
- package/src/skf-update-skill/references/validate.md +0 -124
- package/tools/validate-docs-drift.js +0 -233
- package/tools/validate-docs-links.js +0 -665
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# /// script
|
|
2
|
-
# requires-python = ">=3.
|
|
2
|
+
# requires-python = ">=3.11"
|
|
3
3
|
# dependencies = []
|
|
4
4
|
# ///
|
|
5
5
|
"""SKF Build Change Manifest — aggregate category A/B/C/D detection results.
|
|
@@ -26,15 +26,107 @@ schema, plus the provenance map for the denominator.
|
|
|
26
26
|
|
|
27
27
|
Subcommands:
|
|
28
28
|
|
|
29
|
-
build [--input <file>] [--
|
|
29
|
+
build [--input <file>] [--category-a <file>] [--category-b-diff <file>]
|
|
30
|
+
[--category-c <file>] [--ccc-pairs <file>]
|
|
31
|
+
[--file-compare <file> --provenance-map <file>] [--new-files <file>]
|
|
30
32
|
Reads category JSON from stdin or `--input <file>`, emits the
|
|
31
|
-
unified manifest envelope.
|
|
33
|
+
unified manifest envelope (see "Manifest" below).
|
|
34
|
+
|
|
35
|
+
build --doc-hashes <file>
|
|
36
|
+
A docs-only skill's manifest, from skf-detect-docs.py
|
|
37
|
+
compare-hashes' output and nothing else (see "Docs-only" below).
|
|
32
38
|
|
|
33
39
|
deletion-ratio --provenance-map <file> [--input <file>]
|
|
40
|
+
[--category-a <file>] [--category-b-diff <file>]
|
|
41
|
+
[--category-c <file>] [--ccc-pairs <file>]
|
|
34
42
|
Same category JSON input. Reads provenance entries[] from
|
|
35
43
|
<file>, computes the §2.2 trigger envelope. Auto-skips when
|
|
36
44
|
degraded_mode or update_mode==gap-driven is set in the input.
|
|
37
45
|
|
|
46
|
+
rename-candidates --category-a <file> --provenance-map <file>
|
|
47
|
+
--tier Quick|Forge|Forge+|Deep [--category-b-diff <file>]
|
|
48
|
+
[--extraction <file>] [--export-details <file>]
|
|
49
|
+
[--source-root <dir> [--sizes-commit <commit>]] [-o <file>]
|
|
50
|
+
Category C by fixed rules (see "Rename candidates" below): pairs
|
|
51
|
+
Category A's deleted files with its added ones, and the diff's
|
|
52
|
+
removed exports with its added ones and the added files' exports.
|
|
53
|
+
Prints {"category_c": {...}, "evidence": {...}, "unpaired": {...}}.
|
|
54
|
+
|
|
55
|
+
apply --update-type incremental|gap-driven|full --skill-name <name>
|
|
56
|
+
--generation-date <iso> --confidence-tier <tier>
|
|
57
|
+
--manual-sections-preserved <n> -o <file> [inputs...]
|
|
58
|
+
The provenance map an update writes (see "Apply" below): the old
|
|
59
|
+
map with this run's changes, written to -o through a temporary file
|
|
60
|
+
and a rename. Prints a summary with the warnings for a person.
|
|
61
|
+
|
|
62
|
+
records [--extraction <file>] [--export-details <file>]
|
|
63
|
+
[--files-from <file>] [--patches <dir>] -o <file>
|
|
64
|
+
Step 3's re-extraction records (see "Records" below), written to -o
|
|
65
|
+
through a temporary file and a rename. Prints {"status": "written",
|
|
66
|
+
"output", "files_extracted", "exports_extracted",
|
|
67
|
+
"confidence_breakdown": {"T1", "T1-low", "T2"}, "warnings"}.
|
|
68
|
+
|
|
69
|
+
gap-records --manifest <file> --provenance-map <file> [--source-root <dir>]
|
|
70
|
+
[--drift-status ok|skipped|overridden] [--judgments <file>]
|
|
71
|
+
[--remediation-records <file>] [--evidence <file> --tier <tier>]
|
|
72
|
+
(--plan [--files-out <file>] | -o <file>)
|
|
73
|
+
A gap-driven repair's spot-checks, routing and verification records
|
|
74
|
+
(see "Gap records" below): --plan prints what to answer and which
|
|
75
|
+
files targeted re-extraction scans; without it, the records are
|
|
76
|
+
written to -o through a temporary file and a rename.
|
|
77
|
+
|
|
78
|
+
Helper files in place of typed slices (update-skill detect-changes §2.1):
|
|
79
|
+
|
|
80
|
+
--category-a <file> the output of skf-classify-changed-files.py
|
|
81
|
+
classify: its category_a is the Category A
|
|
82
|
+
slice, and each of its moved_files (a
|
|
83
|
+
same-content move) is a category_c.renamed_files
|
|
84
|
+
item, ahead of the input's own
|
|
85
|
+
--category-b-diff <file> the output of skf-structural-diff.py over the
|
|
86
|
+
modified files: the Category B slice, mapped
|
|
87
|
+
removed[] -> deleted_exports {name, file, old_line}
|
|
88
|
+
added[] -> new_exports {name, file, line}
|
|
89
|
+
changed[] -> modified_exports {name, file,
|
|
90
|
+
old_line, new_line} for an export
|
|
91
|
+
with a field other than `line`, or
|
|
92
|
+
one signature_unverified[] lists
|
|
93
|
+
(its lines null when no field
|
|
94
|
+
changed); moved_exports for an
|
|
95
|
+
export whose only field is `line`
|
|
96
|
+
moved[] -> moved_exports {name, file:
|
|
97
|
+
current_file, old_line:
|
|
98
|
+
previous_line, new_line: line},
|
|
99
|
+
unless it is a modified export
|
|
100
|
+
label_changes[] is not drift, and a name
|
|
101
|
+
ambiguous_names[] lists stays removed and added
|
|
102
|
+
--category-c <file> the output of rename-candidates: its category_c
|
|
103
|
+
is the Category C slice
|
|
104
|
+
--ccc-pairs <file> the pairs the update's CCC check added to what
|
|
105
|
+
rename-candidates left unpaired,
|
|
106
|
+
{"renamed_files": [...], "renamed_exports":
|
|
107
|
+
[...]}, appended to the Category C slice
|
|
108
|
+
--file-compare <file> build only: the output of skf-hash-content.py
|
|
109
|
+
compare, with --provenance-map (the map it
|
|
110
|
+
compared): each MODIFIED_FILE and DELETED_FILE
|
|
111
|
+
row goes to the {type}s_modified or
|
|
112
|
+
{type}s_deleted list of category_d, by the
|
|
113
|
+
file_type of the file_entries[] row of its
|
|
114
|
+
source_file (script, asset or doc; a row with
|
|
115
|
+
none takes the type its file_name's first
|
|
116
|
+
folder names, scripts/, assets/ or docs/, else
|
|
117
|
+
asset)
|
|
118
|
+
--new-files <file> build only: the output of skf-new-file-diff.py:
|
|
119
|
+
each new_files[] item goes to scripts_added or
|
|
120
|
+
assets_added by its kind
|
|
121
|
+
Either Category D file makes the two files the category_d slice. A given
|
|
122
|
+
file replaces the input's own slice. Either way, every category_c
|
|
123
|
+
rename then takes its pair out of the lists it was found in: a renamed
|
|
124
|
+
file's old_path out of category_a.deleted and new_path out of
|
|
125
|
+
category_a.added; a renamed export's old_name out of
|
|
126
|
+
category_b.deleted_exports (the one in its `old_file` when the item gives
|
|
127
|
+
one, else preferably the one in its `file`) and its new_name out of
|
|
128
|
+
category_b.new_exports (preferably the one in its `file`).
|
|
129
|
+
|
|
38
130
|
Input JSON shape (object on stdin or in --input file):
|
|
39
131
|
|
|
40
132
|
{
|
|
@@ -55,27 +147,301 @@ Input JSON shape (object on stdin or in --input file):
|
|
|
55
147
|
},
|
|
56
148
|
"category_d": {
|
|
57
149
|
"scripts_modified": ["path", ...], "scripts_added": [...], "scripts_deleted": [...],
|
|
58
|
-
"assets_modified": [...], "assets_added": [...], "assets_deleted": [...]
|
|
150
|
+
"assets_modified": [...], "assets_added": [...], "assets_deleted": [...],
|
|
151
|
+
"docs_modified": [...], "docs_deleted": [...]
|
|
59
152
|
},
|
|
60
153
|
"degraded_mode": false,
|
|
61
154
|
"update_mode": "normal" | "gap-driven" // optional; influences deletion-ratio only
|
|
62
155
|
}
|
|
63
156
|
|
|
157
|
+
Manifest (build's output):
|
|
158
|
+
|
|
159
|
+
{"no_changes": <every count is 0>, "degraded_mode": <bool>,
|
|
160
|
+
"counts": {files_changed, files_added, files_deleted, files_moved,
|
|
161
|
+
exports_modified, exports_new, exports_deleted,
|
|
162
|
+
exports_renamed, exports_moved, and one count per
|
|
163
|
+
category_d list},
|
|
164
|
+
"total_export_changes": N, "per_file": [...],
|
|
165
|
+
"category_d": {the eight category_d lists, each path once}}
|
|
166
|
+
|
|
167
|
+
A tracked document (a doc row of file_entries[]) has no other detector:
|
|
168
|
+
update-skill's Category A never lists it, so a document-only change still
|
|
169
|
+
counts, and the update records its new hash.
|
|
170
|
+
|
|
64
171
|
Every category and sub-key is optional — missing keys default to
|
|
65
172
|
empty lists. This lets gap-driven runs (which produce no Category D
|
|
66
173
|
results) and degraded runs (which skip Category B) feed the same
|
|
67
174
|
script without sentinel values.
|
|
68
175
|
|
|
176
|
+
Docs-only (update-skill detect-changes §1): build --doc-hashes reads the
|
|
177
|
+
compare-hashes output and prints {"mode": "docs-only", "no_changes":
|
|
178
|
+
<changed[] is empty>, "changed_urls": [each changed[].url],
|
|
179
|
+
"fetch_failed": [each fetch_failed[].url], "counts": {"docs_changed",
|
|
180
|
+
"docs_fetch_failed"}}. It takes no other input.
|
|
181
|
+
|
|
182
|
+
Rename candidates (update-skill detect-changes Category C):
|
|
183
|
+
|
|
184
|
+
Content similarity above 80% (fixed, not configurable) is a rename. A
|
|
185
|
+
file moved with its content unchanged is Category A's moved_files and is
|
|
186
|
+
left out here. For the rest:
|
|
187
|
+
files, Quick tier the added file's size is within 20% of the deleted
|
|
188
|
+
file's (the deleted size is `git cat-file -s` of
|
|
189
|
+
<commit>:<path> under --source-root; with no
|
|
190
|
+
--sizes-commit, a local source whose file is gone,
|
|
191
|
+
the size test is skipped) and the export names
|
|
192
|
+
overlap above 70% (shared names over all names of
|
|
193
|
+
the two files)
|
|
194
|
+
files, Forge+ above 80% of the exports match: an export of the
|
|
195
|
+
added file with the deleted file's export name and
|
|
196
|
+
signature (export_type, params and return_type),
|
|
197
|
+
over the larger file's export count
|
|
198
|
+
exports, Quick the names are above 80% alike (difflib ratio) and
|
|
199
|
+
their export types agree
|
|
200
|
+
exports, Forge+ the signatures are equal (export_type, params and
|
|
201
|
+
return_type, a parameter list or a return type
|
|
202
|
+
given) and the names differ
|
|
203
|
+
A deleted file's exports come from the provenance map, an added file's
|
|
204
|
+
from --extraction and --export-details. A pair is kept only when it is
|
|
205
|
+
each side's best match: the highest similarity wins, and a tie between
|
|
206
|
+
two candidates pairs neither. category_c.renamed_files holds {old_path,
|
|
207
|
+
new_path}; renamed_exports holds {old_name, new_name, file, old_file?},
|
|
208
|
+
with old_file only when it differs from file. evidence lists each pair
|
|
209
|
+
with its rule and similarity, and unpaired the deleted and added files
|
|
210
|
+
and the removed and added exports left over, for the judgment step.
|
|
211
|
+
|
|
212
|
+
Apply (update-skill write.md §3):
|
|
213
|
+
|
|
214
|
+
Reads the provenance map the update started from (--provenance-map;
|
|
215
|
+
without it, a degraded run's new map) and prints nothing but a summary:
|
|
216
|
+
the new map goes to -o. Every entry the run leaves alone keeps its exact
|
|
217
|
+
value; every entry it adds or rewrites carries signature_source, set
|
|
218
|
+
from the tool that produced the signature: ast-grep gives T1 (with
|
|
219
|
+
confidence T1 and the recipe's ast_node_type), a signature read by eye
|
|
220
|
+
(source-read) gives T1-low (confidence T1-low, ast_node_type null).
|
|
221
|
+
A rewritten entry keeps any key the update does not set (notes,
|
|
222
|
+
deprecated). New entries take source_library from the map's entries,
|
|
223
|
+
else --skill-name.
|
|
224
|
+
|
|
225
|
+
incremental (normal mode) reads --manifest (build's output), the renamed
|
|
226
|
+
exports of --category-c and --ccc-pairs (or of --input, the category
|
|
227
|
+
JSON's category_c) and the fresh
|
|
228
|
+
records: --reextract-records ({"mode": "normal", "files": [{file_path,
|
|
229
|
+
exports: [...]}]}, the records `records` wrote), --extraction (the
|
|
230
|
+
recipe runner's exports) and --export-details. A fresh record of an
|
|
231
|
+
export takes its line, export_type and ast labels from the runner, then
|
|
232
|
+
the details, then step 3's record; its params and return_type from the
|
|
233
|
+
details, then step 3's record (the runner's in the map's typed form).
|
|
234
|
+
Per file of the manifest:
|
|
235
|
+
MODIFIED NEW_EXPORT adds an entry, MODIFIED_EXPORT rewrites the
|
|
236
|
+
export's fields from its fresh record, MOVED_EXPORT sets
|
|
237
|
+
its line (and file), DELETED_EXPORT removes it
|
|
238
|
+
ADDED every fresh record of the file becomes an entry
|
|
239
|
+
DELETED every entry of the file is removed
|
|
240
|
+
MOVED the old path's entries are replaced by the new path's
|
|
241
|
+
fresh records, each keeping the keys of the entry of its
|
|
242
|
+
name it replaces
|
|
243
|
+
and each renamed export takes its new name and fresh fields. A docs-only
|
|
244
|
+
run's records ({"mode": "docs-only", "changed_urls": [...], "exports":
|
|
245
|
+
[{name, type, params, return_type, url}]}) replace the entries of each
|
|
246
|
+
changed URL (source_file is the URL; confidence and signature_source
|
|
247
|
+
T3).
|
|
248
|
+
|
|
249
|
+
gap-driven reads --reextract-records ({"mode": "gap-driven",
|
|
250
|
+
"verification": [...], "files": [...]}) and --merge-records ({"exports":
|
|
251
|
+
[{export_name, export_type, params, return_type}]}, merge's output for
|
|
252
|
+
an export no fresh record holds). Each verification record names
|
|
253
|
+
export_name, gap_category, severity, verification, in_map (the map holds
|
|
254
|
+
an entry of that name), map_entry ({source_file, source_line} of the one
|
|
255
|
+
entry the spot-check took, else null), source_citation, new_location,
|
|
256
|
+
unknown_reason, pinned_definition_lines and reachability:
|
|
257
|
+
verified (map_entry) unchanged
|
|
258
|
+
moved (map_entry) source_line (and source_file) from new_location
|
|
259
|
+
re-extracted the entry from its `files` record (gap-driven.md §4a);
|
|
260
|
+
none when its reachability is internal-unreachable
|
|
261
|
+
verified/moved, not in_map, a NEW_EXPORT or MODIFIED_EXPORT whose
|
|
262
|
+
reachability is not internal-unreachable
|
|
263
|
+
one source-read entry at the citation's line
|
|
264
|
+
(verified) or new_location (moved), at most one
|
|
265
|
+
per name and file, export_type and the
|
|
266
|
+
signature from --merge-records
|
|
267
|
+
unknown, not in_map, a NEW_EXPORT or MODIFIED_EXPORT, a Medium, Low or
|
|
268
|
+
Info severity an entry from --merge-records with no
|
|
269
|
+
source_file or source_line
|
|
270
|
+
unknown, not in_map, any other severity (a missing one included)
|
|
271
|
+
refused: exit 3, nothing written
|
|
272
|
+
missing, unknown in_map, unknown on a MOVED_EXPORT
|
|
273
|
+
unchanged, with a warning for a person
|
|
274
|
+
rescoped the entry removed
|
|
275
|
+
Under the drift override (--drift-head and --drift-pinned) each warning
|
|
276
|
+
names the drift.
|
|
277
|
+
|
|
278
|
+
full (degraded mode) makes every fresh record an entry.
|
|
279
|
+
|
|
280
|
+
Every mode then applies the file changes: --file-compare (skf-hash-
|
|
281
|
+
content.py compare's output) sets the content_hash of a MODIFIED_FILE
|
|
282
|
+
row and removes a DELETED_FILE row; --new-files (skf-new-file-diff.py's
|
|
283
|
+
output) adds a row per new script or asset ({kind}s/<file name>, hashed
|
|
284
|
+
under --source-root, extraction_method file-copy); --promoted-docs (the
|
|
285
|
+
documents step 2 promoted, with their hashes) adds a doc row each
|
|
286
|
+
(docs/authoritative/<path>, promoted-authoritative). A row whose
|
|
287
|
+
source_file the map already holds is never added twice.
|
|
288
|
+
|
|
289
|
+
Last it sets the top-level source_commit and source_ref (when given)
|
|
290
|
+
and the update block, replacing an earlier update's values in place:
|
|
291
|
+
last_update (--generation-date), update_type, test_report_run_id,
|
|
292
|
+
files_changed (the manifest's files, or in gap-driven mode the files of
|
|
293
|
+
the entries this run changed), exports_affected (the manifest's export
|
|
294
|
+
changes, or the verification records), confidence_tier and
|
|
295
|
+
manual_sections_preserved. An older update-history key
|
|
296
|
+
(update_operations, update_metadata) stays as it is.
|
|
297
|
+
|
|
298
|
+
Summary on stdout: {"status": "written", "map": <-o>, "entries":
|
|
299
|
+
{"added", "updated", "removed"}, "file_entries": {"added", "updated",
|
|
300
|
+
"removed"}, "warnings": [...]}, or {"status": "refused",
|
|
301
|
+
"blocking_unresolved": [{export_name, severity}]} with exit 3.
|
|
302
|
+
|
|
303
|
+
Records (update-skill re-extract.md §4, and gap-driven.md §4a for the files
|
|
304
|
+
it scans):
|
|
305
|
+
|
|
306
|
+
Writes {"mode": "normal", "files": [{file_path, exports: [...]}]}, one
|
|
307
|
+
block per file --files-from names (and per other file an export names),
|
|
308
|
+
so the model never types a record a tool already wrote. Each export the
|
|
309
|
+
recipe runner found (--extraction) is seeded with its name, type,
|
|
310
|
+
signature, location (file:line), confidence T1, extraction_method
|
|
311
|
+
ast-grep, ast_node_type and ast_recipe, return_type and params, each
|
|
312
|
+
parameter in the provenance map's typed form (skf-extraction-
|
|
313
|
+
inventory.py typed_param: `name: type`, `name?: type`, ` = default`).
|
|
314
|
+
Where the runner left params or return_type null, --export-details
|
|
315
|
+
fills them. An export only --export-details holds (one read by eye,
|
|
316
|
+
with an export_type) is seeded from it as T1-low, source-read, with a
|
|
317
|
+
null ast_node_type and ast_recipe.
|
|
318
|
+
|
|
319
|
+
Then each worker patch in --patches (every *.json there, in name order:
|
|
320
|
+
one block {file_path, exports: [{name, ...}]} or a list of them) is
|
|
321
|
+
merged into the export of its name and file: signature, members,
|
|
322
|
+
docstring and qmd_evidence replace the seed's, and params and
|
|
323
|
+
return_type are set only where the seed has none. A patch for an export
|
|
324
|
+
no seed holds is not added, with a warning. confidence_breakdown counts
|
|
325
|
+
the exports by confidence, and T2 the exports with a qmd_evidence.
|
|
326
|
+
|
|
327
|
+
Gap records (update-skill gap-driven.md §3, §4 and §4a):
|
|
328
|
+
|
|
329
|
+
Reads the change manifest skf-parse-gaps.py translate wrote. Its
|
|
330
|
+
STRUCTURAL_FIX and metadata update entries are forwarded to merge as
|
|
331
|
+
they are (`forwarded`); every other entry is export-bearing. A severity
|
|
332
|
+
is blocking unless it is Medium, Low or Info (NON_BLOCKING), compared
|
|
333
|
+
case-insensitively, so a missing one is blocking. In order:
|
|
334
|
+
|
|
335
|
+
1. Drift gate (--drift-status overridden): each export-bearing entry
|
|
336
|
+
but a MOVED_EXPORT needs the tree, with the first reason that fits:
|
|
337
|
+
a DELETED_EXPORT "a public API recount from the tree (rule R1)", a
|
|
338
|
+
provenance_completeness entry "a line from the tree (rule R3)", one
|
|
339
|
+
whose map_match is not-found "a line and a signature from the tree",
|
|
340
|
+
any other "a signature from the tree". Any such entry: {"status":
|
|
341
|
+
"drift-blocked", "drift_blocked": [{gap_id, name, change_category,
|
|
342
|
+
severity, reason}]}, and no spot-check runs.
|
|
343
|
+
2. A DELETED_EXPORT without a `rescope` that holds its `amendment` and
|
|
344
|
+
`exclude`: {"status": "blocked", "rescope_without_amendment": [names]}.
|
|
345
|
+
3. Spot-checks, by skf-verify-provenance-completeness.py's definition-
|
|
346
|
+
lines rules, loaded from this folder, under --source-root:
|
|
347
|
+
DELETED_EXPORT rescoped, no check
|
|
348
|
+
MOVED_EXPORT checked at its map entry when map_match is
|
|
349
|
+
found (unknown otherwise); under the
|
|
350
|
+
override a `moved` becomes unknown with
|
|
351
|
+
unknown_reason drift-override; its
|
|
352
|
+
pinned_definition_lines are the numbers of
|
|
353
|
+
its remediation's "definition line(s) (...)"
|
|
354
|
+
NEW_EXPORT, MODIFIED_EXPORT
|
|
355
|
+
map_match found checked at the map entry (its export_type)
|
|
356
|
+
ambiguous unknown
|
|
357
|
+
not-found checked at its source_citation (no export
|
|
358
|
+
type) when it has one; when that pins no
|
|
359
|
+
line (unknown or missing), or with no
|
|
360
|
+
citation: targeted re-extraction (§4a) for a
|
|
361
|
+
provenance_completeness entry with a source
|
|
362
|
+
root, for a blocking one whatever its
|
|
363
|
+
resolved_paths, and for a missing-export or
|
|
364
|
+
missing-type one with resolved_paths; else
|
|
365
|
+
unknown
|
|
366
|
+
A check gives `verified` (the line is a definition line, or a module
|
|
367
|
+
or package entry whose file exists), `moved` (one definition line,
|
|
368
|
+
not the recorded one: new_location), `missing` (no such file) or
|
|
369
|
+
`unknown` (several definition lines, none the recorded one, or
|
|
370
|
+
none). A file the rules cover no language of
|
|
371
|
+
takes the answered `declaring_line`: the recorded line is verified,
|
|
372
|
+
another is moved, null is unknown. With no verifier every check is
|
|
373
|
+
unknown, with a `provenance: spot-checks not run` warning; a check
|
|
374
|
+
the verifier fails on (a file it cannot read, or any error) is
|
|
375
|
+
unknown, with a `provenance: spot-check failed for {name}: {error}`
|
|
376
|
+
warning.
|
|
377
|
+
--plan stops here: {"status": "planned", "reextract": {"entries":
|
|
378
|
+
[{gap_id, name, severity, resolved_paths}], "files": [the union of
|
|
379
|
+
their resolved_paths]}, "warnings"}, the files also written to
|
|
380
|
+
--files-out as a JSON array.
|
|
381
|
+
4. Targeted re-extraction (--remediation-records, `records`' output
|
|
382
|
+
over those files): each routed entry with resolved_paths takes the
|
|
383
|
+
first export of its name in the blocks of those files, in their
|
|
384
|
+
order: `re-extracted`, new_location its record's location,
|
|
385
|
+
resolution_source remediation-paths, and the record copied into
|
|
386
|
+
`files` as `records` wrote it, with the answered `docstring` added.
|
|
387
|
+
An entry with no match, or no path to scan, is unresolved, except a
|
|
388
|
+
missing-export or missing-type one that is not blocking, which is
|
|
389
|
+
unknown. Any unresolved entry: {"status": "unresolved",
|
|
390
|
+
"unresolved": [{gap_id, name, severity, remediation_paths,
|
|
391
|
+
rejected_paths, files_scanned, exports_found_in_scan}]}.
|
|
392
|
+
5. The public-reachability gate: every NEW_EXPORT that is verified,
|
|
393
|
+
moved or re-extracted takes the answered `reachability`, `public` or
|
|
394
|
+
`internal-unreachable`. An internal one keeps its outcome in its
|
|
395
|
+
verification record, its record is not in `files`, `counts` tallies
|
|
396
|
+
it under `reclassified` instead of its outcome, and it is listed in
|
|
397
|
+
`reclassified` (merge queues it as a metadata update; apply writes no
|
|
398
|
+
entry for it).
|
|
399
|
+
|
|
400
|
+
Any answer missing in step 3 or 5 (a `docstring` too, for each
|
|
401
|
+
re-extracted entry): {"status": "needs-judgment", "needs_judgment":
|
|
402
|
+
[{gap_id, name, needs: [...], file?, line?}]}. --judgments gives them,
|
|
403
|
+
keyed by gap id (the answers file translate reads; its keys are left).
|
|
404
|
+
Then it writes {"mode": "gap-driven", "files_extracted" (the blocks of
|
|
405
|
+
--remediation-records, 0 when it scanned none), "exports_extracted" (the
|
|
406
|
+
manifest's entries), "confidence_breakdown": {"T1", "T1-low",
|
|
407
|
+
"unlabeled", "T2": 0} (each manifest entry by the extraction_method of
|
|
408
|
+
its re-extracted record or, in the map, of its map entry; a pinned cited
|
|
409
|
+
export the map lacks and the gate passed is T1-low), "verification":
|
|
410
|
+
[one record per export-bearing entry: export_name, gap_category,
|
|
411
|
+
severity, verification, in_map, map_entry, provenance_citation,
|
|
412
|
+
source_citation, new_location, unknown_reason, pinned_definition_lines,
|
|
413
|
+
resolution_source, reachability, export_type (always null: for a cited
|
|
414
|
+
export the map lacks, apply takes it from merge.md's
|
|
415
|
+
merge-records.json)], "files": [...]}, and prints {"status": "written",
|
|
416
|
+
"output", "gap_count", "files_extracted", "exports_extracted",
|
|
417
|
+
"confidence_breakdown", "counts" (per verification outcome, and
|
|
418
|
+
`reclassified`), "reclassified", "forwarded", "targeted_reextraction",
|
|
419
|
+
"warnings"}. When targeted re-extraction ran, `targeted_reextraction`
|
|
420
|
+
is {resolved_count, files_scanned, exports_matched, tier}, also appended
|
|
421
|
+
to --evidence as one JSON line keyed `targeted_reextraction`.
|
|
422
|
+
|
|
69
423
|
Exit codes:
|
|
70
|
-
0
|
|
71
|
-
1
|
|
424
|
+
0 operation succeeded (for gap-records, whatever its `status`)
|
|
425
|
+
1 user error (malformed JSON, bad path, malformed provenance file, a
|
|
426
|
+
helper file that is not the output it names, a gap-records answer of
|
|
427
|
+
the wrong type, or targeted re-extraction to match with no
|
|
428
|
+
--remediation-records)
|
|
429
|
+
3 apply refused: a blocking gap reached the provenance write with no
|
|
430
|
+
source line (nothing written)
|
|
72
431
|
"""
|
|
73
432
|
|
|
74
433
|
from __future__ import annotations
|
|
75
434
|
|
|
76
435
|
import argparse
|
|
436
|
+
import difflib
|
|
437
|
+
import hashlib
|
|
438
|
+
import importlib.util
|
|
77
439
|
import json
|
|
440
|
+
import os
|
|
441
|
+
import re
|
|
442
|
+
import subprocess
|
|
78
443
|
import sys
|
|
444
|
+
import time
|
|
79
445
|
from pathlib import Path
|
|
80
446
|
|
|
81
447
|
|
|
@@ -99,7 +465,10 @@ def _get_list(d: dict, *path: str) -> list:
|
|
|
99
465
|
def _load_input(input_path: Path | None) -> dict:
|
|
100
466
|
"""Read JSON either from --input path or from stdin."""
|
|
101
467
|
if input_path is not None:
|
|
102
|
-
|
|
468
|
+
try:
|
|
469
|
+
text = input_path.read_text(encoding="utf-8")
|
|
470
|
+
except OSError as exc:
|
|
471
|
+
raise ValueError(f"cannot read input {input_path}: {exc}") from exc
|
|
103
472
|
else:
|
|
104
473
|
text = sys.stdin.read()
|
|
105
474
|
try:
|
|
@@ -111,6 +480,130 @@ def _load_input(input_path: Path | None) -> dict:
|
|
|
111
480
|
return data
|
|
112
481
|
|
|
113
482
|
|
|
483
|
+
def _load_helper_file(path: Path, what: str, key: str) -> dict:
|
|
484
|
+
"""A helper's JSON output: an object holding `key`. Raises ValueError."""
|
|
485
|
+
try:
|
|
486
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
487
|
+
except OSError as exc:
|
|
488
|
+
raise ValueError(f"cannot read {what} {path}: {exc}") from exc
|
|
489
|
+
except json.JSONDecodeError as exc:
|
|
490
|
+
raise ValueError(f"{what} {path} is not valid JSON: {exc}") from exc
|
|
491
|
+
if not isinstance(data, dict) or key not in data:
|
|
492
|
+
raise ValueError(f"{what} {path} has no `{key}`: is it the helper's output?")
|
|
493
|
+
return data
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
# --------------------------------------------------------------------------
|
|
497
|
+
# Helper files (update-skill detect-changes §2.1)
|
|
498
|
+
# --------------------------------------------------------------------------
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def category_b_from_diff(diff: dict) -> dict:
|
|
502
|
+
"""Category B's four lists from a skf-structural-diff.py diff of the
|
|
503
|
+
modified files (see the module docstring for the mapping)."""
|
|
504
|
+
lists: dict[str, list] = {
|
|
505
|
+
"modified_exports": [], "new_exports": [], "deleted_exports": [], "moved_exports": [],
|
|
506
|
+
}
|
|
507
|
+
for item in _get_list(diff, "removed"):
|
|
508
|
+
if isinstance(item, dict):
|
|
509
|
+
lists["deleted_exports"].append(
|
|
510
|
+
{"name": item.get("name"), "file": item.get("file"), "old_line": item.get("line")})
|
|
511
|
+
for item in _get_list(diff, "added"):
|
|
512
|
+
if isinstance(item, dict):
|
|
513
|
+
lists["new_exports"].append({"name": item.get("name"), "file": item.get("file"), "line": item.get("line")})
|
|
514
|
+
changed: dict[tuple, dict] = {}
|
|
515
|
+
for item in _get_list(diff, "changed"):
|
|
516
|
+
if not isinstance(item, dict):
|
|
517
|
+
continue
|
|
518
|
+
record = changed.setdefault(
|
|
519
|
+
(item.get("name"), item.get("file")), {"fields": set(), "line": item.get("line"), "old_line": None})
|
|
520
|
+
record["fields"].add(item.get("field"))
|
|
521
|
+
if item.get("field") == "line":
|
|
522
|
+
record["old_line"] = item.get("baseline_value")
|
|
523
|
+
unverified = list(dict.fromkeys(
|
|
524
|
+
(item.get("name"), item.get("file")) for item in _get_list(diff, "signature_unverified")
|
|
525
|
+
if isinstance(item, dict)
|
|
526
|
+
))
|
|
527
|
+
modified: set[tuple] = set()
|
|
528
|
+
for key, record in changed.items():
|
|
529
|
+
old_line = record["old_line"] if record["old_line"] is not None else record["line"]
|
|
530
|
+
entry = {"name": key[0], "file": key[1], "old_line": old_line, "new_line": record["line"]}
|
|
531
|
+
if record["fields"] - {"line"} or key in unverified:
|
|
532
|
+
lists["modified_exports"].append(entry)
|
|
533
|
+
modified.add(key)
|
|
534
|
+
else:
|
|
535
|
+
lists["moved_exports"].append(entry)
|
|
536
|
+
for key in unverified:
|
|
537
|
+
if key not in modified:
|
|
538
|
+
lists["modified_exports"].append({"name": key[0], "file": key[1], "old_line": None, "new_line": None})
|
|
539
|
+
modified.add(key)
|
|
540
|
+
for item in _get_list(diff, "moved"):
|
|
541
|
+
if isinstance(item, dict) and (item.get("name"), item.get("current_file")) not in modified:
|
|
542
|
+
lists["moved_exports"].append({
|
|
543
|
+
"name": item.get("name"), "file": item.get("current_file"),
|
|
544
|
+
"old_line": item.get("previous_line"), "new_line": item.get("line"),
|
|
545
|
+
})
|
|
546
|
+
return lists
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def _take_out(entries: list, name: object, files: tuple) -> None:
|
|
550
|
+
"""Remove the first entry named `name`, preferring one in the first of
|
|
551
|
+
`files` that holds one (a None in `files` stands for any file)."""
|
|
552
|
+
for wanted in files:
|
|
553
|
+
for i, entry in enumerate(entries):
|
|
554
|
+
if isinstance(entry, dict) and entry.get("name") == name and wanted in (None, entry.get("file")):
|
|
555
|
+
del entries[i]
|
|
556
|
+
return
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def apply_renames(payload: dict) -> dict:
|
|
560
|
+
"""Take each Category C rename's pair out of the Category A and B lists
|
|
561
|
+
it was found in (see the module docstring). payload holds category_a
|
|
562
|
+
and category_b as objects, as assemble leaves them."""
|
|
563
|
+
cat_a, cat_b = payload["category_a"], payload["category_b"]
|
|
564
|
+
cat_c = payload.get("category_c", {}) or {}
|
|
565
|
+
for move in _get_list(cat_c, "renamed_files"):
|
|
566
|
+
if not isinstance(move, dict):
|
|
567
|
+
continue
|
|
568
|
+
for key, path in (("deleted", move.get("old_path")), ("added", move.get("new_path"))):
|
|
569
|
+
listed = _get_list(cat_a, key)
|
|
570
|
+
if path in listed:
|
|
571
|
+
cat_a[key] = [p for p in listed if p != path]
|
|
572
|
+
for rename in _get_list(cat_c, "renamed_exports"):
|
|
573
|
+
if not isinstance(rename, dict):
|
|
574
|
+
continue
|
|
575
|
+
deleted = list(_get_list(cat_b, "deleted_exports"))
|
|
576
|
+
new = list(_get_list(cat_b, "new_exports"))
|
|
577
|
+
old_files = (rename["old_file"],) if rename.get("old_file") else (rename.get("file"), None)
|
|
578
|
+
_take_out(deleted, rename.get("old_name"), old_files)
|
|
579
|
+
_take_out(new, rename.get("new_name"), (rename.get("file"), None))
|
|
580
|
+
cat_b["deleted_exports"], cat_b["new_exports"] = deleted, new
|
|
581
|
+
return payload
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def assemble(payload: dict, *, category_a_doc: dict | None = None, diff: dict | None = None) -> dict:
|
|
585
|
+
"""The category JSON build and deletion-ratio read: the input, with the
|
|
586
|
+
helper files' slices in place of its own, and Category C's renames taken
|
|
587
|
+
out of the A and B lists."""
|
|
588
|
+
out = dict(payload)
|
|
589
|
+
cat_c = dict(payload.get("category_c") or {})
|
|
590
|
+
if category_a_doc is not None:
|
|
591
|
+
out["category_a"] = {key: list(_get_list(category_a_doc, "category_a", key))
|
|
592
|
+
for key in ("modified", "added", "deleted")}
|
|
593
|
+
moves = [m for m in _get_list(category_a_doc, "moved_files") if isinstance(m, dict)]
|
|
594
|
+
cat_c["renamed_files"] = moves + list(_get_list(cat_c, "renamed_files"))
|
|
595
|
+
else:
|
|
596
|
+
out["category_a"] = {key: list(_get_list(payload, "category_a", key))
|
|
597
|
+
for key in ("modified", "added", "deleted")}
|
|
598
|
+
if diff is not None:
|
|
599
|
+
out["category_b"] = category_b_from_diff(diff)
|
|
600
|
+
else:
|
|
601
|
+
out["category_b"] = {key: list(_get_list(payload, "category_b", key))
|
|
602
|
+
for key in ("modified_exports", "new_exports", "deleted_exports", "moved_exports")}
|
|
603
|
+
out["category_c"] = cat_c
|
|
604
|
+
return apply_renames(out)
|
|
605
|
+
|
|
606
|
+
|
|
114
607
|
# --------------------------------------------------------------------------
|
|
115
608
|
# Build manifest
|
|
116
609
|
# --------------------------------------------------------------------------
|
|
@@ -135,14 +628,8 @@ def build_manifest(payload: dict) -> dict:
|
|
|
135
628
|
c_renamed_files = _get_list(cat_c, "renamed_files")
|
|
136
629
|
c_renamed_exports = _get_list(cat_c, "renamed_exports")
|
|
137
630
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
"scripts_added": len(_get_list(cat_d, "scripts_added")),
|
|
141
|
-
"scripts_deleted": len(_get_list(cat_d, "scripts_deleted")),
|
|
142
|
-
"assets_modified": len(_get_list(cat_d, "assets_modified")),
|
|
143
|
-
"assets_added": len(_get_list(cat_d, "assets_added")),
|
|
144
|
-
"assets_deleted": len(_get_list(cat_d, "assets_deleted")),
|
|
145
|
-
}
|
|
631
|
+
d_lists = {key: list(dict.fromkeys(p for p in _get_list(cat_d, key) if isinstance(p, str))) for key in D_KEYS}
|
|
632
|
+
d = {key: len(paths) for key, paths in d_lists.items()}
|
|
146
633
|
|
|
147
634
|
counts = {
|
|
148
635
|
"files_changed": len(a_modified),
|
|
@@ -186,9 +673,51 @@ def build_manifest(payload: dict) -> dict:
|
|
|
186
673
|
"counts": counts,
|
|
187
674
|
"total_export_changes": total_export_changes,
|
|
188
675
|
"per_file": per_file,
|
|
676
|
+
"category_d": d_lists,
|
|
189
677
|
}
|
|
190
678
|
|
|
191
679
|
|
|
680
|
+
# Category D's lists, in the order build counts them.
|
|
681
|
+
D_KEYS = ("scripts_modified", "scripts_added", "scripts_deleted", "assets_modified", "assets_added",
|
|
682
|
+
"assets_deleted", "docs_modified", "docs_deleted")
|
|
683
|
+
FILE_TYPES = ("script", "asset", "doc")
|
|
684
|
+
_FILE_TYPE_OF_FOLDER = {"scripts": "script", "assets": "asset", "docs": "doc"}
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def _file_type(row: dict) -> str:
|
|
688
|
+
"""A file_entries[] row's file_type, else the one its file_name's first folder names, else asset."""
|
|
689
|
+
kind = row.get("file_type")
|
|
690
|
+
if kind in FILE_TYPES:
|
|
691
|
+
return kind
|
|
692
|
+
name = row.get("file_name")
|
|
693
|
+
folder = name.strip().replace("\\", "/").split("/", 1)[0] if isinstance(name, str) else ""
|
|
694
|
+
return _FILE_TYPE_OF_FOLDER.get(folder, "asset")
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def category_d_from_files(compare: dict | None, new_files: dict | None, provenance: dict | None) -> dict:
|
|
698
|
+
"""Category D's lists from skf-hash-content.py compare's rows, each typed by the file_entries[] row of its
|
|
699
|
+
source_file in `provenance`, and from skf-new-file-diff.py's new files, by kind. Raises ValueError."""
|
|
700
|
+
rows = (provenance or {}).get("file_entries") or []
|
|
701
|
+
if not isinstance(rows, list):
|
|
702
|
+
raise ValueError("provenance `file_entries` must be an array")
|
|
703
|
+
types: dict[str, str] = {}
|
|
704
|
+
for row in rows:
|
|
705
|
+
if isinstance(row, dict) and _norm_path(row.get("source_file")):
|
|
706
|
+
types.setdefault(_norm_path(row["source_file"]), _file_type(row))
|
|
707
|
+
lists: dict[str, list] = {key: [] for key in D_KEYS}
|
|
708
|
+
changes = {"MODIFIED_FILE": "modified", "DELETED_FILE": "deleted"}
|
|
709
|
+
for row in _get_list(compare or {}, "comparisons"):
|
|
710
|
+
if not isinstance(row, dict) or row.get("classification") not in changes:
|
|
711
|
+
continue
|
|
712
|
+
path = _norm_path(row.get("source_file"))
|
|
713
|
+
if path:
|
|
714
|
+
lists[f"{types.get(path, 'asset')}s_{changes[row['classification']]}"].append(path)
|
|
715
|
+
for item in _get_list(new_files or {}, "new_files"):
|
|
716
|
+
if isinstance(item, dict) and item.get("kind") in ("script", "asset") and _norm_path(item.get("source_file")):
|
|
717
|
+
lists[f"{item['kind']}s_added"].append(_norm_path(item["source_file"]))
|
|
718
|
+
return lists
|
|
719
|
+
|
|
720
|
+
|
|
192
721
|
def _build_per_file(
|
|
193
722
|
*,
|
|
194
723
|
a_modified: list,
|
|
@@ -337,18 +866,1417 @@ def _ratio_skip(*, skip_reason: str) -> dict:
|
|
|
337
866
|
}
|
|
338
867
|
|
|
339
868
|
|
|
869
|
+
# --------------------------------------------------------------------------
|
|
870
|
+
# Shared readers (rename-candidates and apply)
|
|
871
|
+
# --------------------------------------------------------------------------
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def _norm_path(path: object) -> str | None:
|
|
875
|
+
"""A file path compared as structural-diff compares it: `/` separators, no leading `./`."""
|
|
876
|
+
if not isinstance(path, str) or not path.strip():
|
|
877
|
+
return None
|
|
878
|
+
text = path.strip().replace("\\", "/")
|
|
879
|
+
while text.startswith("./"):
|
|
880
|
+
text = text[2:]
|
|
881
|
+
return text
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
def _load_json_file(path: Path, what: str):
|
|
885
|
+
try:
|
|
886
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
887
|
+
except OSError as exc:
|
|
888
|
+
raise ValueError(f"cannot read {what} {path}: {exc}") from exc
|
|
889
|
+
except json.JSONDecodeError as exc:
|
|
890
|
+
raise ValueError(f"{what} {path} is not valid JSON: {exc}") from exc
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
def _exports_of(doc, what: str) -> list[dict]:
|
|
894
|
+
"""The export records of an extraction or export-details file ({"exports": [...]})."""
|
|
895
|
+
if doc is None:
|
|
896
|
+
return []
|
|
897
|
+
if not isinstance(doc, dict) or not isinstance(doc.get("exports"), list):
|
|
898
|
+
raise ValueError(f"{what} has no `exports` array: is it the helper's output?")
|
|
899
|
+
return [e for e in doc["exports"] if isinstance(e, dict)]
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
def _entries_of(provenance) -> list[dict]:
|
|
903
|
+
if provenance is None:
|
|
904
|
+
return []
|
|
905
|
+
if not isinstance(provenance, dict):
|
|
906
|
+
raise ValueError("the provenance map must be a JSON object")
|
|
907
|
+
entries = provenance.get("entries", [])
|
|
908
|
+
if not isinstance(entries, list):
|
|
909
|
+
raise ValueError("provenance `entries` must be an array")
|
|
910
|
+
return [e for e in entries if isinstance(e, dict)]
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def _params(value) -> tuple | None:
|
|
914
|
+
"""A parameter list compared by its text, whitespace removed."""
|
|
915
|
+
if value is None:
|
|
916
|
+
return None
|
|
917
|
+
if not isinstance(value, list):
|
|
918
|
+
return (str(value),)
|
|
919
|
+
out = []
|
|
920
|
+
for item in value:
|
|
921
|
+
if isinstance(item, dict):
|
|
922
|
+
name, kind = item.get("name"), item.get("type")
|
|
923
|
+
item = f"{name}: {kind}" if kind else str(name)
|
|
924
|
+
out.append("".join(str(item).split()))
|
|
925
|
+
return tuple(out)
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
def _signature(record: dict) -> tuple:
|
|
929
|
+
"""(export_type, params, return_type) of an export record, the parts a signature compares."""
|
|
930
|
+
kind = record.get("export_type", record.get("type"))
|
|
931
|
+
params = record.get("params", record.get("parameters"))
|
|
932
|
+
ret = record.get("return_type")
|
|
933
|
+
return (kind if isinstance(kind, str) else None, _params(params),
|
|
934
|
+
"".join(ret.split()) if isinstance(ret, str) else None)
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
def _name_of(record: dict) -> str | None:
|
|
938
|
+
name = record.get("export_name", record.get("name"))
|
|
939
|
+
return name if isinstance(name, str) and name else None
|
|
940
|
+
|
|
941
|
+
|
|
942
|
+
def _file_of(record: dict) -> str | None:
|
|
943
|
+
return _norm_path(record.get("source_file", record.get("file", record.get("file_path"))))
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
# --------------------------------------------------------------------------
|
|
947
|
+
# Rename candidates (update-skill detect-changes Category C)
|
|
948
|
+
# --------------------------------------------------------------------------
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
RENAME_SIMILARITY = 0.80 # content similarity above this is a rename (fixed)
|
|
952
|
+
SIZE_TOLERANCE = 0.20 # Quick tier: the added file's size within 20% of the deleted one's
|
|
953
|
+
NAME_OVERLAP = 0.70 # Quick tier: shared export names above 70%
|
|
954
|
+
TIERS = ("Quick", "Forge", "Forge+", "Deep")
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def _git_size(source_root: Path, commit: str, path: str) -> int | None:
|
|
958
|
+
"""The size of `path` at `commit` (git cat-file -s), or None when git cannot tell."""
|
|
959
|
+
try:
|
|
960
|
+
proc = subprocess.run(["git", "-C", str(source_root), "cat-file", "-s", f"{commit}:{path}"],
|
|
961
|
+
capture_output=True, text=True, timeout=30, check=False)
|
|
962
|
+
except (OSError, subprocess.SubprocessError):
|
|
963
|
+
return None
|
|
964
|
+
if proc.returncode != 0:
|
|
965
|
+
return None
|
|
966
|
+
try:
|
|
967
|
+
return int(proc.stdout.strip())
|
|
968
|
+
except ValueError:
|
|
969
|
+
return None
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
def _file_size(source_root: Path | None, path: str) -> int | None:
|
|
973
|
+
if source_root is None:
|
|
974
|
+
return None
|
|
975
|
+
try:
|
|
976
|
+
return (source_root / path).stat().st_size
|
|
977
|
+
except OSError:
|
|
978
|
+
return None
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
def _best_pairs(candidates: list[tuple]) -> list[tuple]:
|
|
982
|
+
"""Each side's best match only: (score, left, right, rule) tuples, highest score first.
|
|
983
|
+
|
|
984
|
+
A pair is kept when neither side is taken yet and no other candidate of
|
|
985
|
+
either side ties its score, so a tie pairs neither.
|
|
986
|
+
"""
|
|
987
|
+
out, used_left, used_right = [], set(), set()
|
|
988
|
+
ordered = sorted(candidates, key=lambda c: (-c[0], c[1], c[2]))
|
|
989
|
+
for score, left, right, rule in ordered:
|
|
990
|
+
if left in used_left or right in used_right:
|
|
991
|
+
continue
|
|
992
|
+
rivals = [c for c in ordered if c[0] == score and (c[1] == left) != (c[2] == right)
|
|
993
|
+
and c[1] not in used_left and c[2] not in used_right]
|
|
994
|
+
if rivals:
|
|
995
|
+
used_left.add(left)
|
|
996
|
+
used_right.add(right)
|
|
997
|
+
for c in rivals:
|
|
998
|
+
used_left.add(c[1])
|
|
999
|
+
used_right.add(c[2])
|
|
1000
|
+
continue
|
|
1001
|
+
used_left.add(left)
|
|
1002
|
+
used_right.add(right)
|
|
1003
|
+
out.append((score, left, right, rule))
|
|
1004
|
+
return out
|
|
1005
|
+
|
|
1006
|
+
|
|
1007
|
+
def rename_candidates(category_a_doc: dict, diff: dict | None, provenance: dict, extraction: dict | None,
|
|
1008
|
+
details: dict | None, tier: str, *, source_root: Path | None = None,
|
|
1009
|
+
sizes_commit: str | None = None) -> dict:
|
|
1010
|
+
"""Category C by the fixed rules of the module docstring."""
|
|
1011
|
+
if tier not in TIERS:
|
|
1012
|
+
raise ValueError(f"--tier must be one of {', '.join(TIERS)}, got {tier!r}")
|
|
1013
|
+
forge = tier != "Quick"
|
|
1014
|
+
moves = [m for m in _get_list(category_a_doc, "moved_files") if isinstance(m, dict)]
|
|
1015
|
+
moved_old = {_norm_path(m.get("old_path")) for m in moves}
|
|
1016
|
+
moved_new = {_norm_path(m.get("new_path")) for m in moves}
|
|
1017
|
+
deleted = sorted({p for p in map(_norm_path, _get_list(category_a_doc, "category_a", "deleted"))
|
|
1018
|
+
if p and p not in moved_old})
|
|
1019
|
+
added = sorted({p for p in map(_norm_path, _get_list(category_a_doc, "category_a", "added"))
|
|
1020
|
+
if p and p not in moved_new})
|
|
1021
|
+
|
|
1022
|
+
old_by_file: dict[str, dict[str, dict]] = {}
|
|
1023
|
+
for entry in _entries_of(provenance):
|
|
1024
|
+
name, path = _name_of(entry), _file_of(entry)
|
|
1025
|
+
if name and path:
|
|
1026
|
+
old_by_file.setdefault(path, {}).setdefault(name, entry)
|
|
1027
|
+
new_by_file: dict[str, dict[str, dict]] = {}
|
|
1028
|
+
for record in _exports_of(extraction, "--extraction file") + _exports_of(details, "--export-details file"):
|
|
1029
|
+
name, path = _name_of(record), _file_of(record)
|
|
1030
|
+
if name and path:
|
|
1031
|
+
merged = new_by_file.setdefault(path, {}).setdefault(name, {})
|
|
1032
|
+
for key, value in record.items():
|
|
1033
|
+
if merged.get(key) is None:
|
|
1034
|
+
merged[key] = value
|
|
1035
|
+
|
|
1036
|
+
file_candidates = []
|
|
1037
|
+
for old in deleted:
|
|
1038
|
+
old_exports = old_by_file.get(old, {})
|
|
1039
|
+
for new in added:
|
|
1040
|
+
new_exports = new_by_file.get(new, {})
|
|
1041
|
+
if not old_exports or not new_exports:
|
|
1042
|
+
continue
|
|
1043
|
+
if forge:
|
|
1044
|
+
matched = sum(1 for name, e in old_exports.items()
|
|
1045
|
+
if name in new_exports and _signature(e) == _signature(new_exports[name]))
|
|
1046
|
+
score = matched / max(len(old_exports), len(new_exports))
|
|
1047
|
+
if score > RENAME_SIMILARITY:
|
|
1048
|
+
file_candidates.append((round(score, 4), old, new, "signatures"))
|
|
1049
|
+
continue
|
|
1050
|
+
names_old, names_new = set(old_exports), set(new_exports)
|
|
1051
|
+
overlap = len(names_old & names_new) / len(names_old | names_new)
|
|
1052
|
+
if overlap <= NAME_OVERLAP:
|
|
1053
|
+
continue
|
|
1054
|
+
old_size = _git_size(source_root, sizes_commit, old) if (source_root and sizes_commit) else None
|
|
1055
|
+
new_size = _file_size(source_root, new)
|
|
1056
|
+
if old_size is not None:
|
|
1057
|
+
if new_size is None or max(old_size, new_size) == 0:
|
|
1058
|
+
continue
|
|
1059
|
+
if min(old_size, new_size) / max(old_size, new_size) < 1 - SIZE_TOLERANCE:
|
|
1060
|
+
continue
|
|
1061
|
+
rule = "size-and-names"
|
|
1062
|
+
else:
|
|
1063
|
+
rule = "names (size not checked)"
|
|
1064
|
+
file_candidates.append((round(overlap, 4), old, new, rule))
|
|
1065
|
+
file_pairs = _best_pairs(file_candidates)
|
|
1066
|
+
paired_old = {p[1] for p in file_pairs}
|
|
1067
|
+
paired_new = {p[2] for p in file_pairs}
|
|
1068
|
+
|
|
1069
|
+
removed = []
|
|
1070
|
+
for item in _get_list(diff or {}, "removed"):
|
|
1071
|
+
if isinstance(item, dict) and _name_of(item) and _file_of(item):
|
|
1072
|
+
name, path = _name_of(item), _file_of(item)
|
|
1073
|
+
removed.append({**old_by_file.get(path, {}).get(name, {}), **{k: v for k, v in item.items()
|
|
1074
|
+
if v is not None},
|
|
1075
|
+
"_key": (name, path)})
|
|
1076
|
+
candidates_new = []
|
|
1077
|
+
for item in _get_list(diff or {}, "added"):
|
|
1078
|
+
if isinstance(item, dict) and _name_of(item) and _file_of(item):
|
|
1079
|
+
name, path = _name_of(item), _file_of(item)
|
|
1080
|
+
candidates_new.append({**new_by_file.get(path, {}).get(name, {}),
|
|
1081
|
+
**{k: v for k, v in item.items() if v is not None}, "_key": (name, path)})
|
|
1082
|
+
seen_new = {c["_key"] for c in candidates_new}
|
|
1083
|
+
for path in added:
|
|
1084
|
+
if path in paired_new:
|
|
1085
|
+
continue
|
|
1086
|
+
for name, record in new_by_file.get(path, {}).items():
|
|
1087
|
+
if (name, path) not in seen_new:
|
|
1088
|
+
candidates_new.append({**record, "_key": (name, path)})
|
|
1089
|
+
|
|
1090
|
+
export_candidates = []
|
|
1091
|
+
for old in removed:
|
|
1092
|
+
old_sig = _signature(old)
|
|
1093
|
+
for new in candidates_new:
|
|
1094
|
+
if old["_key"][0] == new["_key"][0]:
|
|
1095
|
+
continue
|
|
1096
|
+
new_sig = _signature(new)
|
|
1097
|
+
if forge:
|
|
1098
|
+
if old_sig[1] is None and old_sig[2] is None:
|
|
1099
|
+
continue
|
|
1100
|
+
if old_sig == new_sig:
|
|
1101
|
+
export_candidates.append((1.0, old["_key"], new["_key"], "signature-equal"))
|
|
1102
|
+
continue
|
|
1103
|
+
if old_sig[0] and new_sig[0] and old_sig[0] != new_sig[0]:
|
|
1104
|
+
continue
|
|
1105
|
+
ratio = difflib.SequenceMatcher(None, old["_key"][0], new["_key"][0]).ratio()
|
|
1106
|
+
if ratio > RENAME_SIMILARITY:
|
|
1107
|
+
export_candidates.append((round(ratio, 4), old["_key"], new["_key"], "name-similarity"))
|
|
1108
|
+
export_pairs = _best_pairs(export_candidates)
|
|
1109
|
+
|
|
1110
|
+
renamed_exports = []
|
|
1111
|
+
for _score, (old_name, old_file), (new_name, new_file), _rule in export_pairs:
|
|
1112
|
+
item = {"old_name": old_name, "new_name": new_name, "file": new_file}
|
|
1113
|
+
if old_file != new_file:
|
|
1114
|
+
item["old_file"] = old_file
|
|
1115
|
+
renamed_exports.append(item)
|
|
1116
|
+
paired_removed = {p[1] for p in export_pairs}
|
|
1117
|
+
paired_added = {p[2] for p in export_pairs}
|
|
1118
|
+
return {
|
|
1119
|
+
"category_c": {
|
|
1120
|
+
"renamed_files": [{"old_path": old, "new_path": new} for _s, old, new, _r in file_pairs],
|
|
1121
|
+
"renamed_exports": renamed_exports,
|
|
1122
|
+
},
|
|
1123
|
+
"evidence": {
|
|
1124
|
+
"files": [{"old_path": old, "new_path": new, "rule": rule, "similarity": score}
|
|
1125
|
+
for score, old, new, rule in file_pairs],
|
|
1126
|
+
"exports": [{"old_name": o[0], "new_name": n[0], "file": n[1], "rule": rule, "similarity": score}
|
|
1127
|
+
for score, o, n, rule in export_pairs],
|
|
1128
|
+
},
|
|
1129
|
+
"unpaired": {
|
|
1130
|
+
"deleted_files": [p for p in deleted if p not in paired_old],
|
|
1131
|
+
"added_files": [p for p in added if p not in paired_new],
|
|
1132
|
+
"removed_exports": [{"name": r["_key"][0], "file": r["_key"][1]} for r in removed
|
|
1133
|
+
if r["_key"] not in paired_removed],
|
|
1134
|
+
"added_exports": [{"name": c["_key"][0], "file": c["_key"][1]} for c in candidates_new
|
|
1135
|
+
if c["_key"] not in paired_added],
|
|
1136
|
+
},
|
|
1137
|
+
"tier": tier,
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
|
|
1141
|
+
# --------------------------------------------------------------------------
|
|
1142
|
+
# Apply (update-skill write.md §3)
|
|
1143
|
+
# --------------------------------------------------------------------------
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
UPDATE_TYPES = ("incremental", "gap-driven", "full")
|
|
1147
|
+
NON_BLOCKING = ("medium", "low", "info")
|
|
1148
|
+
EXIT_REFUSED = 3
|
|
1149
|
+
REPLACE_WAIT_SECONDS = 5.0 # how long a Windows rename onto a file held open is retried
|
|
1150
|
+
# The update block apply sets at the top level of the map, in this order.
|
|
1151
|
+
UPDATE_BLOCK_KEYS = ("last_update", "update_type", "test_report_run_id", "files_changed", "exports_affected",
|
|
1152
|
+
"confidence_tier", "manual_sections_preserved")
|
|
1153
|
+
|
|
1154
|
+
|
|
1155
|
+
def _labels(method: object, node: object) -> dict:
|
|
1156
|
+
"""The confidence labels an extraction method implies (create-skill's entry contract)."""
|
|
1157
|
+
if method in ("ast-grep", "ast_bridge"):
|
|
1158
|
+
return {"confidence": "T1", "extraction_method": "ast-grep", "ast_node_type": node,
|
|
1159
|
+
"signature_source": "T1"}
|
|
1160
|
+
return {"confidence": "T1-low", "extraction_method": "source-read", "ast_node_type": None,
|
|
1161
|
+
"signature_source": "T1-low"}
|
|
1162
|
+
|
|
1163
|
+
|
|
1164
|
+
def _line_of(value: object) -> int | None:
|
|
1165
|
+
"""A line number from an int, a digit string or a `file:line` / `file:start-end` location."""
|
|
1166
|
+
if isinstance(value, bool):
|
|
1167
|
+
return None
|
|
1168
|
+
if isinstance(value, int):
|
|
1169
|
+
return value
|
|
1170
|
+
if isinstance(value, str):
|
|
1171
|
+
tail = value.rsplit(":", 1)[-1].split("-", 1)[0].strip()
|
|
1172
|
+
if tail.isdigit():
|
|
1173
|
+
return int(tail)
|
|
1174
|
+
return None
|
|
1175
|
+
|
|
1176
|
+
|
|
1177
|
+
def _location(value: object) -> tuple[str | None, int | None]:
|
|
1178
|
+
"""(file, line) of a `file:line` string or a {file, line} object."""
|
|
1179
|
+
if isinstance(value, dict):
|
|
1180
|
+
return _norm_path(value.get("file") or value.get("source_file")), _line_of(
|
|
1181
|
+
value.get("line", value.get("source_line")))
|
|
1182
|
+
if isinstance(value, str) and ":" in value:
|
|
1183
|
+
path, _, rest = value.rpartition(":")
|
|
1184
|
+
return _norm_path(path), _line_of(rest)
|
|
1185
|
+
return None, None
|
|
1186
|
+
|
|
1187
|
+
|
|
1188
|
+
class _Fresh:
|
|
1189
|
+
"""The fresh records of one run, looked up by export name and file."""
|
|
1190
|
+
|
|
1191
|
+
def __init__(self, worker_files: list, extraction: dict | None, details: dict | None):
|
|
1192
|
+
self.runner = {}
|
|
1193
|
+
self.details = {}
|
|
1194
|
+
self.worker = {}
|
|
1195
|
+
for record in _exports_of(extraction, "--extraction file"):
|
|
1196
|
+
key = (_name_of(record), _file_of(record))
|
|
1197
|
+
if all(key):
|
|
1198
|
+
self.runner.setdefault(key, record)
|
|
1199
|
+
for record in _exports_of(details, "--export-details file"):
|
|
1200
|
+
key = (_name_of(record), _file_of(record))
|
|
1201
|
+
if all(key):
|
|
1202
|
+
self.details.setdefault(key, record)
|
|
1203
|
+
for block in worker_files:
|
|
1204
|
+
if not isinstance(block, dict):
|
|
1205
|
+
continue
|
|
1206
|
+
path = _norm_path(block.get("file_path"))
|
|
1207
|
+
for record in block.get("exports") or []:
|
|
1208
|
+
if isinstance(record, dict) and _name_of(record) and path:
|
|
1209
|
+
self.worker.setdefault((_name_of(record), path), record)
|
|
1210
|
+
|
|
1211
|
+
def keys_of(self, path: str) -> list[tuple]:
|
|
1212
|
+
keys = {k for source in (self.runner, self.details, self.worker) for k in source if k[1] == path}
|
|
1213
|
+
return sorted(keys)
|
|
1214
|
+
|
|
1215
|
+
def all_keys(self) -> list[tuple]:
|
|
1216
|
+
return sorted({k for source in (self.runner, self.details, self.worker) for k in source})
|
|
1217
|
+
|
|
1218
|
+
def record(self, name: str, path: str) -> dict | None:
|
|
1219
|
+
"""The entry fields of an export's fresh record, or None when no source holds it."""
|
|
1220
|
+
key = (name, path)
|
|
1221
|
+
runner, detail, worker = self.runner.get(key), self.details.get(key), self.worker.get(key)
|
|
1222
|
+
if runner is None and detail is None and worker is None:
|
|
1223
|
+
return None
|
|
1224
|
+
sources = [r for r in (runner, detail, worker) if r is not None]
|
|
1225
|
+
|
|
1226
|
+
def first(*fields):
|
|
1227
|
+
for record in sources:
|
|
1228
|
+
for field in fields:
|
|
1229
|
+
value = record.get(field)
|
|
1230
|
+
if value is not None:
|
|
1231
|
+
return value
|
|
1232
|
+
return None
|
|
1233
|
+
|
|
1234
|
+
line = None
|
|
1235
|
+
for record in sources:
|
|
1236
|
+
line = _line_of(record.get("source_line"))
|
|
1237
|
+
if line is None and record is worker:
|
|
1238
|
+
line = _line_of(record.get("location"))
|
|
1239
|
+
if line is not None:
|
|
1240
|
+
break
|
|
1241
|
+
method = first("extraction_method")
|
|
1242
|
+
node = first("ast_node_type") if method in ("ast-grep", "ast_bridge") else None
|
|
1243
|
+
params = None
|
|
1244
|
+
for record in (detail, worker, runner):
|
|
1245
|
+
if record is None:
|
|
1246
|
+
continue
|
|
1247
|
+
value = record.get("params", record.get("parameters"))
|
|
1248
|
+
if value is not None:
|
|
1249
|
+
params = value
|
|
1250
|
+
break
|
|
1251
|
+
return {
|
|
1252
|
+
"export_name": name,
|
|
1253
|
+
"export_type": first("export_type", "type"),
|
|
1254
|
+
"params": params,
|
|
1255
|
+
"return_type": first("return_type"),
|
|
1256
|
+
"source_file": path,
|
|
1257
|
+
"source_line": line,
|
|
1258
|
+
**_labels(method, node),
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
|
|
1262
|
+
def _without_none(fields: dict, keep: tuple = ()) -> dict:
|
|
1263
|
+
return {k: v for k, v in fields.items() if v is not None or k in keep}
|
|
1264
|
+
|
|
1265
|
+
|
|
1266
|
+
def _new_entry(fields: dict, library: str) -> dict:
|
|
1267
|
+
"""An entry in create-skill's key order, from the fields a record gives."""
|
|
1268
|
+
order = ("export_name", "export_type", "source_library", "params", "return_type", "source_file",
|
|
1269
|
+
"source_line", "confidence", "extraction_method", "ast_node_type", "signature_source")
|
|
1270
|
+
merged = {**fields, "source_library": fields.get("source_library") or library}
|
|
1271
|
+
out = {k: merged[k] for k in order if k in merged and (merged[k] is not None or k == "ast_node_type")}
|
|
1272
|
+
return out
|
|
1273
|
+
|
|
1274
|
+
|
|
1275
|
+
def _rewrite(entry: dict, fields: dict) -> dict:
|
|
1276
|
+
"""The entry with `fields` set in place; keys the update does not set keep their values."""
|
|
1277
|
+
out = dict(entry)
|
|
1278
|
+
for key, value in fields.items():
|
|
1279
|
+
if value is None and key not in ("ast_node_type",):
|
|
1280
|
+
continue
|
|
1281
|
+
out[key] = value
|
|
1282
|
+
return out
|
|
1283
|
+
|
|
1284
|
+
|
|
1285
|
+
def _sha256_of(path: Path) -> str:
|
|
1286
|
+
digest = hashlib.sha256()
|
|
1287
|
+
with path.open("rb") as fh:
|
|
1288
|
+
for chunk in iter(lambda: fh.read(65536), b""):
|
|
1289
|
+
digest.update(chunk)
|
|
1290
|
+
return "sha256:" + digest.hexdigest()
|
|
1291
|
+
|
|
1292
|
+
|
|
1293
|
+
class _Map:
|
|
1294
|
+
"""The provenance map being rewritten, with what changed."""
|
|
1295
|
+
|
|
1296
|
+
def __init__(self, provenance: dict | None, skill_name: str):
|
|
1297
|
+
self.doc = json.loads(json.dumps(provenance)) if provenance is not None else {
|
|
1298
|
+
"provenance_version": "2.0", "skill_name": skill_name, "skill_type": "single", "entries": []}
|
|
1299
|
+
if not isinstance(self.doc.get("entries"), list):
|
|
1300
|
+
raise ValueError("provenance `entries` must be an array")
|
|
1301
|
+
libraries = [e.get("source_library") for e in self.doc["entries"]
|
|
1302
|
+
if isinstance(e, dict) and isinstance(e.get("source_library"), str) and e["source_library"]]
|
|
1303
|
+
self.library = libraries[0] if libraries else skill_name
|
|
1304
|
+
self.added: list[str] = []
|
|
1305
|
+
self.updated: list[str] = []
|
|
1306
|
+
self.removed: list[str] = []
|
|
1307
|
+
self.touched_files: set[str] = set()
|
|
1308
|
+
|
|
1309
|
+
def find(self, name: str, path: str | None, line: int | None = None) -> int | None:
|
|
1310
|
+
"""The index of the entry with this name (and file, and line when several share both)."""
|
|
1311
|
+
hits = [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and _name_of(e) == name
|
|
1312
|
+
and (path is None or _file_of(e) == path)]
|
|
1313
|
+
if len(hits) > 1 and line is not None:
|
|
1314
|
+
hits = [i for i in hits if _line_of(self.doc["entries"][i].get("source_line")) == line]
|
|
1315
|
+
return hits[0] if len(hits) == 1 else None
|
|
1316
|
+
|
|
1317
|
+
def indexes_of(self, path: str) -> list[int]:
|
|
1318
|
+
return [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and _file_of(e) == path]
|
|
1319
|
+
|
|
1320
|
+
def add(self, entry: dict) -> None:
|
|
1321
|
+
self.doc["entries"].append(entry)
|
|
1322
|
+
self.added.append(entry["export_name"])
|
|
1323
|
+
if entry.get("source_file"):
|
|
1324
|
+
self.touched_files.add(_norm_path(entry["source_file"]))
|
|
1325
|
+
|
|
1326
|
+
def update(self, index: int, fields: dict) -> None:
|
|
1327
|
+
before = self.doc["entries"][index]
|
|
1328
|
+
after = _rewrite(before, fields)
|
|
1329
|
+
if after != before:
|
|
1330
|
+
self.doc["entries"][index] = after
|
|
1331
|
+
self.updated.append(_name_of(after))
|
|
1332
|
+
for path in (_file_of(before), _file_of(after)):
|
|
1333
|
+
if path:
|
|
1334
|
+
self.touched_files.add(path)
|
|
1335
|
+
|
|
1336
|
+
def remove(self, indexes: list[int]) -> None:
|
|
1337
|
+
for index in sorted(set(indexes), reverse=True):
|
|
1338
|
+
entry = self.doc["entries"].pop(index)
|
|
1339
|
+
self.removed.append(_name_of(entry))
|
|
1340
|
+
if _file_of(entry):
|
|
1341
|
+
self.touched_files.add(_file_of(entry))
|
|
1342
|
+
|
|
1343
|
+
|
|
1344
|
+
def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fresh: _Fresh,
|
|
1345
|
+
warnings: list[str]) -> tuple[int, int]:
|
|
1346
|
+
"""Normal mode: the manifest's file and export changes. Returns (files, exports) for the update block."""
|
|
1347
|
+
per_file = [f for f in _get_list(manifest or {}, "per_file") if isinstance(f, dict)]
|
|
1348
|
+
|
|
1349
|
+
def refresh(index: int | None, name: str, path: str, *, add: bool) -> None:
|
|
1350
|
+
record = fresh.record(name, path)
|
|
1351
|
+
if record is None:
|
|
1352
|
+
warnings.append(f"provenance: {name} in {path}: no extraction record, entry "
|
|
1353
|
+
f"{'kept as it was' if index is not None else 'not added'}")
|
|
1354
|
+
return
|
|
1355
|
+
if index is None:
|
|
1356
|
+
if add:
|
|
1357
|
+
pmap.add(_new_entry(record, pmap.library))
|
|
1358
|
+
return
|
|
1359
|
+
pmap.update(index, _without_none(record, keep=("ast_node_type",)))
|
|
1360
|
+
|
|
1361
|
+
for item in per_file:
|
|
1362
|
+
status, path = item.get("status"), _norm_path(item.get("file_path"))
|
|
1363
|
+
if not path:
|
|
1364
|
+
continue
|
|
1365
|
+
if status == "DELETED":
|
|
1366
|
+
pmap.remove(pmap.indexes_of(path))
|
|
1367
|
+
continue
|
|
1368
|
+
if status == "MOVED":
|
|
1369
|
+
old = _norm_path(item.get("old_path"))
|
|
1370
|
+
old_entries = {_name_of(pmap.doc["entries"][i]): pmap.doc["entries"][i]
|
|
1371
|
+
for i in (pmap.indexes_of(old) if old else [])}
|
|
1372
|
+
pmap.remove(pmap.indexes_of(old) if old else [])
|
|
1373
|
+
for name, key_path in fresh.keys_of(path):
|
|
1374
|
+
record = fresh.record(name, key_path)
|
|
1375
|
+
carried = old_entries.get(name)
|
|
1376
|
+
entry = _rewrite(carried, _without_none(record, keep=("ast_node_type",))) if carried \
|
|
1377
|
+
else _new_entry(record, pmap.library)
|
|
1378
|
+
pmap.add(entry)
|
|
1379
|
+
continue
|
|
1380
|
+
if status == "ADDED":
|
|
1381
|
+
for name, key_path in fresh.keys_of(path):
|
|
1382
|
+
index = pmap.find(name, key_path)
|
|
1383
|
+
refresh(index, name, key_path, add=True)
|
|
1384
|
+
continue
|
|
1385
|
+
for change in item.get("exports_affected") or []:
|
|
1386
|
+
if not isinstance(change, dict) or not _name_of(change):
|
|
1387
|
+
continue
|
|
1388
|
+
name, kind = _name_of(change), change.get("change_type")
|
|
1389
|
+
index = pmap.find(name, path, _line_of(change.get("old_line")))
|
|
1390
|
+
if kind == "DELETED_EXPORT":
|
|
1391
|
+
if index is not None:
|
|
1392
|
+
pmap.remove([index])
|
|
1393
|
+
elif kind == "MOVED_EXPORT":
|
|
1394
|
+
record = fresh.record(name, path)
|
|
1395
|
+
line = record["source_line"] if record and record.get("source_line") is not None \
|
|
1396
|
+
else _line_of(change.get("new_line"))
|
|
1397
|
+
if index is not None and line is not None:
|
|
1398
|
+
pmap.update(index, {"source_file": path, "source_line": line})
|
|
1399
|
+
elif kind in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1400
|
+
refresh(index, name, path, add=True)
|
|
1401
|
+
for rename in _get_list(categories or {}, "category_c", "renamed_exports"):
|
|
1402
|
+
if not isinstance(rename, dict):
|
|
1403
|
+
continue
|
|
1404
|
+
old_name, new_name = rename.get("old_name"), rename.get("new_name")
|
|
1405
|
+
path = _norm_path(rename.get("file"))
|
|
1406
|
+
old_path = _norm_path(rename.get("old_file")) or path
|
|
1407
|
+
if not (isinstance(old_name, str) and isinstance(new_name, str) and path):
|
|
1408
|
+
continue
|
|
1409
|
+
index = pmap.find(old_name, old_path)
|
|
1410
|
+
record = fresh.record(new_name, path)
|
|
1411
|
+
if index is None:
|
|
1412
|
+
continue
|
|
1413
|
+
fields = {"export_name": new_name, "source_file": path}
|
|
1414
|
+
if record is not None:
|
|
1415
|
+
fields.update(_without_none(record, keep=("ast_node_type",)))
|
|
1416
|
+
pmap.update(index, fields)
|
|
1417
|
+
files = len(per_file)
|
|
1418
|
+
total = manifest.get("total_export_changes") if isinstance(manifest, dict) else None
|
|
1419
|
+
return files, total if isinstance(total, int) else 0
|
|
1420
|
+
|
|
1421
|
+
|
|
1422
|
+
def _apply_docs_only(pmap: _Map, records: dict) -> tuple[int, int]:
|
|
1423
|
+
changed = [u for u in records.get("changed_urls") or [] if isinstance(u, str) and u]
|
|
1424
|
+
exports = [e for e in records.get("exports") or [] if isinstance(e, dict)]
|
|
1425
|
+
for url in changed:
|
|
1426
|
+
old = {_name_of(pmap.doc["entries"][i]): pmap.doc["entries"][i] for i in pmap.indexes_of(url)}
|
|
1427
|
+
pmap.remove(pmap.indexes_of(url))
|
|
1428
|
+
for record in exports:
|
|
1429
|
+
if record.get("url") != url or not _name_of(record):
|
|
1430
|
+
continue
|
|
1431
|
+
fields = {"export_name": _name_of(record), "export_type": record.get("export_type", record.get("type")),
|
|
1432
|
+
"params": record.get("params", record.get("parameters")),
|
|
1433
|
+
"return_type": record.get("return_type"), "source_file": url,
|
|
1434
|
+
"confidence": "T3", "signature_source": "T3"}
|
|
1435
|
+
carried = old.get(_name_of(record))
|
|
1436
|
+
entry = _rewrite(carried, _without_none(fields)) if carried else _new_entry(
|
|
1437
|
+
_without_none(fields), pmap.library)
|
|
1438
|
+
pmap.add(entry)
|
|
1439
|
+
return len(changed), len(exports)
|
|
1440
|
+
|
|
1441
|
+
|
|
1442
|
+
def _drift_note(drift: tuple | None) -> str:
|
|
1443
|
+
return f"drift override: HEAD {drift[0]} is not pinned {drift[1]}" if drift else ""
|
|
1444
|
+
|
|
1445
|
+
|
|
1446
|
+
def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, drift: tuple | None,
|
|
1447
|
+
warnings: list[str]) -> tuple[int, int] | list[dict]:
|
|
1448
|
+
"""Gap-driven mode: one write per verification record. A list return is the refusal."""
|
|
1449
|
+
verification = [r for r in records.get("verification") or [] if isinstance(r, dict) and _name_of(r)]
|
|
1450
|
+
reextracted = _Fresh(records.get("files") or [], None, None)
|
|
1451
|
+
merged = {}
|
|
1452
|
+
for record in _exports_of(merge_records, "--merge-records file") if merge_records is not None else []:
|
|
1453
|
+
if _name_of(record):
|
|
1454
|
+
merged.setdefault(_name_of(record), record)
|
|
1455
|
+
|
|
1456
|
+
def in_map(r: dict) -> bool:
|
|
1457
|
+
return bool(r.get("in_map")) or bool(r.get("map_entry"))
|
|
1458
|
+
|
|
1459
|
+
blocking = [{"export_name": _name_of(r), "severity": r.get("severity")} for r in verification
|
|
1460
|
+
if r.get("verification") == "unknown" and not in_map(r)
|
|
1461
|
+
and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT")
|
|
1462
|
+
and str(r.get("severity") or "").strip().lower() not in NON_BLOCKING]
|
|
1463
|
+
if blocking:
|
|
1464
|
+
return blocking
|
|
1465
|
+
note = _drift_note(drift)
|
|
1466
|
+
pinned_keys: set[tuple] = set()
|
|
1467
|
+
for r in verification:
|
|
1468
|
+
name, outcome, category = _name_of(r), r.get("verification"), r.get("gap_category")
|
|
1469
|
+
entry_file, entry_line = _location(r.get("map_entry"))
|
|
1470
|
+
index = pmap.find(name, entry_file, entry_line) if r.get("map_entry") else None
|
|
1471
|
+
lines = r.get("pinned_definition_lines")
|
|
1472
|
+
r5 = f"; definition lines per the test report: {lines}" if category == "MOVED_EXPORT" else ""
|
|
1473
|
+
if outcome == "rescoped":
|
|
1474
|
+
if index is not None:
|
|
1475
|
+
pmap.remove([index])
|
|
1476
|
+
elif outcome == "verified" and index is not None:
|
|
1477
|
+
if drift and category == "MOVED_EXPORT":
|
|
1478
|
+
warnings.append(f"provenance: {name}: verified at HEAD only ({note}{r5})")
|
|
1479
|
+
elif outcome == "moved" and index is not None:
|
|
1480
|
+
path, line = _location(r.get("new_location"))
|
|
1481
|
+
if path and line is not None:
|
|
1482
|
+
pmap.update(index, {"source_file": path, "source_line": line})
|
|
1483
|
+
elif outcome == "re-extracted":
|
|
1484
|
+
if r.get("reachability") == "internal-unreachable":
|
|
1485
|
+
continue # not public API: merge queued it as a metadata update
|
|
1486
|
+
path, line = _location(r.get("new_location"))
|
|
1487
|
+
record = reextracted.record(name, path) if path else None
|
|
1488
|
+
if record is None:
|
|
1489
|
+
warnings.append(f"provenance: {name}: re-extracted, but no record of it in the run's files")
|
|
1490
|
+
continue
|
|
1491
|
+
existing = pmap.find(name, path)
|
|
1492
|
+
if existing is None:
|
|
1493
|
+
pmap.add(_new_entry(record, pmap.library))
|
|
1494
|
+
else:
|
|
1495
|
+
pmap.update(existing, _without_none(record, keep=("ast_node_type",)))
|
|
1496
|
+
elif outcome in ("verified", "moved") and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1497
|
+
if r.get("reachability") == "internal-unreachable":
|
|
1498
|
+
continue
|
|
1499
|
+
where = r.get("source_citation") if outcome == "verified" else r.get("new_location")
|
|
1500
|
+
path, line = _location(where)
|
|
1501
|
+
if not path or line is None or (name, path) in pinned_keys or pmap.find(name, path) is not None:
|
|
1502
|
+
continue
|
|
1503
|
+
pinned_keys.add((name, path))
|
|
1504
|
+
extra = merged.get(name, {})
|
|
1505
|
+
pmap.add(_new_entry(_without_none({
|
|
1506
|
+
"export_name": name,
|
|
1507
|
+
"export_type": extra.get("export_type") or r.get("export_type"),
|
|
1508
|
+
"params": extra.get("params"), "return_type": extra.get("return_type"),
|
|
1509
|
+
"source_file": path, "source_line": line, **_labels("source-read", None)},
|
|
1510
|
+
keep=("ast_node_type",)), pmap.library))
|
|
1511
|
+
elif outcome == "unknown" and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1512
|
+
extra = merged.get(name, {})
|
|
1513
|
+
pmap.add(_new_entry(_without_none({
|
|
1514
|
+
"export_name": name, "export_type": extra.get("export_type") or r.get("export_type"),
|
|
1515
|
+
"params": extra.get("params"), "return_type": extra.get("return_type"),
|
|
1516
|
+
**_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library))
|
|
1517
|
+
else:
|
|
1518
|
+
if r.get("unknown_reason") == "drift-override":
|
|
1519
|
+
warnings.append(f"provenance: {name}: unknown ({note}; no line taken from HEAD{r5})")
|
|
1520
|
+
elif drift:
|
|
1521
|
+
warnings.append(f"provenance: {name}: {outcome} ({note}{r5})")
|
|
1522
|
+
else:
|
|
1523
|
+
warnings.append(f"provenance: {name}: {outcome}{r5}")
|
|
1524
|
+
return len(pmap.touched_files), len(verification)
|
|
1525
|
+
|
|
1526
|
+
|
|
1527
|
+
def _apply_files(pmap: _Map, compare: dict | None, new_files: dict | None, promoted: list | None,
|
|
1528
|
+
source_root: Path | None) -> dict:
|
|
1529
|
+
"""The file_entries[] changes of Category D and the promoted documents."""
|
|
1530
|
+
rows = pmap.doc.get("file_entries")
|
|
1531
|
+
if rows is None:
|
|
1532
|
+
rows = []
|
|
1533
|
+
if not isinstance(rows, list):
|
|
1534
|
+
raise ValueError("provenance `file_entries` must be an array")
|
|
1535
|
+
done = {"added": [], "updated": [], "removed": []}
|
|
1536
|
+
by_source = {_norm_path(r.get("source_file")): i for i, r in enumerate(rows) if isinstance(r, dict)}
|
|
1537
|
+
gone = set()
|
|
1538
|
+
for row in _get_list(compare or {}, "comparisons"):
|
|
1539
|
+
if not isinstance(row, dict):
|
|
1540
|
+
continue
|
|
1541
|
+
path, kind = _norm_path(row.get("source_file")), row.get("classification")
|
|
1542
|
+
index = by_source.get(path)
|
|
1543
|
+
if index is None:
|
|
1544
|
+
continue
|
|
1545
|
+
if kind == "MODIFIED_FILE" and isinstance(row.get("current_hash"), str):
|
|
1546
|
+
if rows[index].get("content_hash") != row["current_hash"]:
|
|
1547
|
+
rows[index] = {**rows[index], "content_hash": row["current_hash"]}
|
|
1548
|
+
done["updated"].append(path)
|
|
1549
|
+
elif kind == "DELETED_FILE":
|
|
1550
|
+
gone.add(index)
|
|
1551
|
+
done["removed"].append(path)
|
|
1552
|
+
rows = [r for i, r in enumerate(rows) if i not in gone]
|
|
1553
|
+
held = {_norm_path(r.get("source_file")) for r in rows if isinstance(r, dict)}
|
|
1554
|
+
for item in _get_list(new_files or {}, "new_files"):
|
|
1555
|
+
if not isinstance(item, dict):
|
|
1556
|
+
continue
|
|
1557
|
+
path, kind = _norm_path(item.get("source_file")), item.get("kind")
|
|
1558
|
+
if not path or path in held or kind not in ("script", "asset"):
|
|
1559
|
+
continue
|
|
1560
|
+
if source_root is None:
|
|
1561
|
+
raise ValueError("--new-files needs --source-root, to hash each new file")
|
|
1562
|
+
try:
|
|
1563
|
+
digest = _sha256_of(source_root / path)
|
|
1564
|
+
except OSError as exc:
|
|
1565
|
+
raise ValueError(f"cannot hash new file {path} under {source_root}: {exc}") from exc
|
|
1566
|
+
rows.append({"file_name": f"{kind}s/{Path(path).name}", "file_type": kind, "source_file": path,
|
|
1567
|
+
"confidence": "T1-low", "extraction_method": "file-copy", "content_hash": digest})
|
|
1568
|
+
held.add(path)
|
|
1569
|
+
done["added"].append(path)
|
|
1570
|
+
for doc in promoted or []:
|
|
1571
|
+
if not isinstance(doc, dict):
|
|
1572
|
+
continue
|
|
1573
|
+
path = _norm_path(doc.get("path"))
|
|
1574
|
+
if not path or path in held or not isinstance(doc.get("content_hash"), str):
|
|
1575
|
+
continue
|
|
1576
|
+
rows.append({"file_name": f"docs/authoritative/{path}", "file_type": "doc", "source_file": path,
|
|
1577
|
+
"content_hash": doc["content_hash"], "confidence": "T1-low",
|
|
1578
|
+
"extraction_method": "promoted-authoritative"})
|
|
1579
|
+
held.add(path)
|
|
1580
|
+
done["added"].append(path)
|
|
1581
|
+
if rows or "file_entries" in pmap.doc:
|
|
1582
|
+
pmap.doc["file_entries"] = rows
|
|
1583
|
+
return done
|
|
1584
|
+
|
|
1585
|
+
|
|
1586
|
+
def apply_update(*, update_type: str, provenance: dict | None, skill_name: str, manifest: dict | None = None,
|
|
1587
|
+
categories: dict | None = None, extraction: dict | None = None, details: dict | None = None,
|
|
1588
|
+
records: dict | None = None, merge_records: dict | None = None, compare: dict | None = None,
|
|
1589
|
+
new_files: dict | None = None, promoted: list | None = None, source_root: Path | None = None,
|
|
1590
|
+
generation_date: str, test_report_run_id: str | None, confidence_tier: str,
|
|
1591
|
+
manual_sections_preserved: int, source_commit: str | None = None,
|
|
1592
|
+
source_ref: str | None = None, drift: tuple | None = None) -> tuple[dict, dict]:
|
|
1593
|
+
"""(new map, summary) of one update (see "Apply" in the module docstring).
|
|
1594
|
+
|
|
1595
|
+
A refusal returns ({}, summary) with status "refused" and no map.
|
|
1596
|
+
Raises ValueError for inputs it cannot read.
|
|
1597
|
+
"""
|
|
1598
|
+
if update_type not in UPDATE_TYPES:
|
|
1599
|
+
raise ValueError(f"--update-type must be one of {', '.join(UPDATE_TYPES)}, got {update_type!r}")
|
|
1600
|
+
if provenance is None and update_type != "full":
|
|
1601
|
+
raise ValueError("--provenance-map is required unless --update-type is full (a degraded run)")
|
|
1602
|
+
records = records if isinstance(records, dict) else {}
|
|
1603
|
+
pmap = _Map(provenance, skill_name)
|
|
1604
|
+
warnings: list[str] = []
|
|
1605
|
+
mode = records.get("mode")
|
|
1606
|
+
if update_type == "gap-driven":
|
|
1607
|
+
outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings)
|
|
1608
|
+
if isinstance(outcome, list):
|
|
1609
|
+
return {}, {"status": "refused", "blocking_unresolved": outcome}
|
|
1610
|
+
files, exports = outcome
|
|
1611
|
+
elif mode == "docs-only":
|
|
1612
|
+
files, exports = _apply_docs_only(pmap, records)
|
|
1613
|
+
else:
|
|
1614
|
+
fresh = _Fresh(records.get("files") or [], extraction, details)
|
|
1615
|
+
if update_type == "full":
|
|
1616
|
+
for name, path in fresh.all_keys():
|
|
1617
|
+
index = pmap.find(name, path)
|
|
1618
|
+
record = fresh.record(name, path)
|
|
1619
|
+
if index is None:
|
|
1620
|
+
pmap.add(_new_entry(record, pmap.library))
|
|
1621
|
+
else:
|
|
1622
|
+
pmap.update(index, _without_none(record, keep=("ast_node_type",)))
|
|
1623
|
+
files = len({path for _name, path in fresh.all_keys()})
|
|
1624
|
+
exports = len(fresh.all_keys())
|
|
1625
|
+
else:
|
|
1626
|
+
files, exports = _apply_normal(pmap, manifest, categories, fresh, warnings)
|
|
1627
|
+
file_changes = _apply_files(pmap, compare, new_files, promoted, source_root)
|
|
1628
|
+
doc = pmap.doc
|
|
1629
|
+
for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
|
|
1630
|
+
if value is not None:
|
|
1631
|
+
doc[key] = value or None
|
|
1632
|
+
block = {
|
|
1633
|
+
"last_update": generation_date,
|
|
1634
|
+
"update_type": update_type,
|
|
1635
|
+
"test_report_run_id": test_report_run_id or None,
|
|
1636
|
+
"files_changed": files,
|
|
1637
|
+
"exports_affected": exports,
|
|
1638
|
+
"confidence_tier": confidence_tier,
|
|
1639
|
+
"manual_sections_preserved": manual_sections_preserved,
|
|
1640
|
+
}
|
|
1641
|
+
for key in UPDATE_BLOCK_KEYS:
|
|
1642
|
+
doc[key] = block[key]
|
|
1643
|
+
summary = {
|
|
1644
|
+
"status": "written",
|
|
1645
|
+
"entries": {"added": pmap.added, "updated": pmap.updated, "removed": pmap.removed},
|
|
1646
|
+
"file_entries": file_changes,
|
|
1647
|
+
"warnings": warnings,
|
|
1648
|
+
}
|
|
1649
|
+
return doc, summary
|
|
1650
|
+
|
|
1651
|
+
|
|
1652
|
+
def _write_json_atomic(path: Path, value) -> None:
|
|
1653
|
+
"""Write `value` as indented JSON through a temporary file beside `path` and one rename, retried on Windows,
|
|
1654
|
+
where a scanner may hold the target open for a moment."""
|
|
1655
|
+
tmp = path.with_name(f".{path.name}.skf-{os.getpid()}-tmp")
|
|
1656
|
+
try:
|
|
1657
|
+
tmp.write_bytes((json.dumps(value, indent=2, ensure_ascii=False) + "\n").encode("utf-8"))
|
|
1658
|
+
deadline = time.monotonic() + REPLACE_WAIT_SECONDS
|
|
1659
|
+
while True:
|
|
1660
|
+
try:
|
|
1661
|
+
os.replace(tmp, path)
|
|
1662
|
+
break
|
|
1663
|
+
except PermissionError:
|
|
1664
|
+
if os.name != "nt" or time.monotonic() >= deadline:
|
|
1665
|
+
raise
|
|
1666
|
+
time.sleep(0.05)
|
|
1667
|
+
except OSError:
|
|
1668
|
+
try:
|
|
1669
|
+
tmp.unlink()
|
|
1670
|
+
except OSError:
|
|
1671
|
+
pass
|
|
1672
|
+
raise
|
|
1673
|
+
|
|
1674
|
+
|
|
1675
|
+
# --------------------------------------------------------------------------
|
|
1676
|
+
# Records (update-skill re-extract.md §4, gap-driven.md §4a)
|
|
1677
|
+
# --------------------------------------------------------------------------
|
|
1678
|
+
|
|
1679
|
+
|
|
1680
|
+
# The fields a worker's patch may set: the ones no tool records, and the two
|
|
1681
|
+
# a patch sets only where the seed has none.
|
|
1682
|
+
PATCH_FIELDS = ("signature", "members", "docstring", "qmd_evidence")
|
|
1683
|
+
PATCH_IF_NULL = ("params", "return_type")
|
|
1684
|
+
_INVENTORY = None
|
|
1685
|
+
|
|
1686
|
+
|
|
1687
|
+
def _typed_param(param, language: object) -> str | None:
|
|
1688
|
+
"""One parameter as the provenance map writes it: skf-extraction-inventory.py's typed_param, loaded once from
|
|
1689
|
+
this folder, so create-skill and update-skill write one form."""
|
|
1690
|
+
global _INVENTORY
|
|
1691
|
+
if _INVENTORY is None:
|
|
1692
|
+
path = Path(__file__).resolve().parent / "skf-extraction-inventory.py"
|
|
1693
|
+
spec = importlib.util.spec_from_file_location("skf_extraction_inventory", path)
|
|
1694
|
+
if spec is None or spec.loader is None:
|
|
1695
|
+
raise ImportError(f"cannot load {path}")
|
|
1696
|
+
module = importlib.util.module_from_spec(spec)
|
|
1697
|
+
spec.loader.exec_module(module)
|
|
1698
|
+
_INVENTORY = module
|
|
1699
|
+
return _INVENTORY.typed_param(param, language if isinstance(language, str) else None)
|
|
1700
|
+
|
|
1701
|
+
|
|
1702
|
+
def _typed_params(value, language: object) -> list | None:
|
|
1703
|
+
if not isinstance(value, list):
|
|
1704
|
+
return None
|
|
1705
|
+
return [s for s in (_typed_param(p, language) for p in value) if s is not None]
|
|
1706
|
+
|
|
1707
|
+
|
|
1708
|
+
def _seed(record: dict, path: str, read_by_eye: bool) -> dict:
|
|
1709
|
+
"""The re-extraction record of one export from the runner's (or, read by eye, step 2's) record."""
|
|
1710
|
+
line = _line_of(record.get("source_line"))
|
|
1711
|
+
out = {
|
|
1712
|
+
"name": _name_of(record),
|
|
1713
|
+
"type": record.get("export_type", record.get("type")),
|
|
1714
|
+
"signature": record.get("signature"),
|
|
1715
|
+
"location": f"{path}:{line}" if line is not None else None,
|
|
1716
|
+
"params": _typed_params(record.get("params", record.get("parameters")), record.get("language")),
|
|
1717
|
+
"return_type": record.get("return_type"),
|
|
1718
|
+
}
|
|
1719
|
+
labels = _labels("source-read" if read_by_eye else record.get("extraction_method"), record.get("ast_node_type"))
|
|
1720
|
+
out.update({key: labels[key] for key in ("confidence", "extraction_method", "ast_node_type")})
|
|
1721
|
+
out["ast_recipe"] = None if read_by_eye else record.get("ast_recipe")
|
|
1722
|
+
return out
|
|
1723
|
+
|
|
1724
|
+
|
|
1725
|
+
def _patch_blocks(folder: Path) -> list[dict]:
|
|
1726
|
+
"""Every per-file block of the patch files in `folder` (a file holds one block or a list of them), in file
|
|
1727
|
+
name order. Raises ValueError."""
|
|
1728
|
+
if not folder.is_dir():
|
|
1729
|
+
raise ValueError(f"--patches {folder} is not a folder")
|
|
1730
|
+
blocks = []
|
|
1731
|
+
for path in sorted(folder.glob("*.json"), key=lambda p: p.name):
|
|
1732
|
+
doc = _load_json_file(path, "patch file")
|
|
1733
|
+
for block in doc if isinstance(doc, list) else [doc]:
|
|
1734
|
+
if not isinstance(block, dict) or not _norm_path(block.get("file_path")) \
|
|
1735
|
+
or not isinstance(block.get("exports"), list):
|
|
1736
|
+
raise ValueError(f"patch file {path} holds a block with no file_path or exports list")
|
|
1737
|
+
blocks.append(block)
|
|
1738
|
+
return blocks
|
|
1739
|
+
|
|
1740
|
+
|
|
1741
|
+
def build_records(extraction: dict | None, details: dict | None, files: list | None,
|
|
1742
|
+
patches: list[dict]) -> tuple[dict, dict]:
|
|
1743
|
+
"""(reextract-records.json, summary): each export's record seeded from the runner's and step 2's records, with
|
|
1744
|
+
the workers' patches merged in (see "Records" in the module docstring)."""
|
|
1745
|
+
runner, extra = {}, {}
|
|
1746
|
+
for source, doc, what in ((runner, extraction, "--extraction file"), (extra, details, "--export-details file")):
|
|
1747
|
+
for record in _exports_of(doc, what):
|
|
1748
|
+
key = (_name_of(record), _file_of(record))
|
|
1749
|
+
if all(key):
|
|
1750
|
+
source.setdefault(key, record)
|
|
1751
|
+
seeded: dict[tuple, dict] = {}
|
|
1752
|
+
languages: dict[tuple, object] = {}
|
|
1753
|
+
warnings = []
|
|
1754
|
+
|
|
1755
|
+
def fill_if_null(key: tuple, record: dict) -> None:
|
|
1756
|
+
"""Set params and return_type from `record` where the seed of `key` has none."""
|
|
1757
|
+
for field in PATCH_IF_NULL:
|
|
1758
|
+
value = record.get(field)
|
|
1759
|
+
if field == "params" and value is None:
|
|
1760
|
+
value = record.get("parameters")
|
|
1761
|
+
if seeded[key][field] is None and value is not None:
|
|
1762
|
+
seeded[key][field] = _typed_params(value, languages.get(key)) if field == "params" else value
|
|
1763
|
+
|
|
1764
|
+
for key, record in runner.items():
|
|
1765
|
+
seeded[key], languages[key] = _seed(record, key[1], read_by_eye=False), record.get("language")
|
|
1766
|
+
fill_if_null(key, extra.get(key, {}))
|
|
1767
|
+
for key, record in extra.items():
|
|
1768
|
+
if key in runner:
|
|
1769
|
+
continue
|
|
1770
|
+
if record.get("export_type", record.get("type")) is None:
|
|
1771
|
+
warnings.append(f"re-extract: {key[0]} in {key[1]}: an export-details record with no export_type the "
|
|
1772
|
+
f"runner does not hold; not added")
|
|
1773
|
+
continue
|
|
1774
|
+
seeded[key], languages[key] = _seed(record, key[1], read_by_eye=True), record.get("language")
|
|
1775
|
+
for block in patches:
|
|
1776
|
+
path = _norm_path(block.get("file_path"))
|
|
1777
|
+
for patch in block.get("exports") or []:
|
|
1778
|
+
key = (_name_of(patch) if isinstance(patch, dict) else None, path)
|
|
1779
|
+
if key not in seeded:
|
|
1780
|
+
warnings.append(f"re-extract: a patch names {key[0] or 'no export'} in {path}, which neither the "
|
|
1781
|
+
f"runner nor step 2 recorded; not added")
|
|
1782
|
+
continue
|
|
1783
|
+
for field in PATCH_FIELDS:
|
|
1784
|
+
if patch.get(field) is not None:
|
|
1785
|
+
seeded[key][field] = patch[field]
|
|
1786
|
+
fill_if_null(key, patch)
|
|
1787
|
+
paths = list(dict.fromkeys([p for p in (_norm_path(f) for f in files or []) if p]
|
|
1788
|
+
+ sorted({key[1] for key in seeded})))
|
|
1789
|
+
blocks = [{"file_path": path, "exports": [seeded[key] for key in sorted(seeded) if key[1] == path]}
|
|
1790
|
+
for path in paths]
|
|
1791
|
+
exports = [record for block in blocks for record in block["exports"]]
|
|
1792
|
+
summary = {
|
|
1793
|
+
"files_extracted": len(blocks),
|
|
1794
|
+
"exports_extracted": len(exports),
|
|
1795
|
+
"confidence_breakdown": {
|
|
1796
|
+
"T1": sum(1 for r in exports if r["confidence"] == "T1"),
|
|
1797
|
+
"T1-low": sum(1 for r in exports if r["confidence"] == "T1-low"),
|
|
1798
|
+
"T2": sum(1 for r in exports if r.get("qmd_evidence") is not None),
|
|
1799
|
+
},
|
|
1800
|
+
"warnings": warnings,
|
|
1801
|
+
}
|
|
1802
|
+
return {"mode": "normal", "files": blocks}, summary
|
|
1803
|
+
|
|
1804
|
+
|
|
1805
|
+
# --------------------------------------------------------------------------
|
|
1806
|
+
# Gap records (update-skill gap-driven.md §3, §4 and §4a)
|
|
1807
|
+
# --------------------------------------------------------------------------
|
|
1808
|
+
|
|
1809
|
+
|
|
1810
|
+
EXPORT_CATEGORIES = ("NEW_EXPORT", "MODIFIED_EXPORT", "MOVED_EXPORT", "DELETED_EXPORT")
|
|
1811
|
+
DOCUMENT_CATEGORIES = ("missing-export", "missing-type") # routed to §4a by category, never halting
|
|
1812
|
+
DRIFT_STATUSES = ("ok", "skipped", "overridden")
|
|
1813
|
+
# The drift gate's reasons, gap-driven.md §3: the first that fits.
|
|
1814
|
+
DRIFT_RESCOPE = "a public API recount from the tree (rule R1)"
|
|
1815
|
+
DRIFT_COMPLETENESS = "a line from the tree (rule R3)"
|
|
1816
|
+
DRIFT_NOT_IN_MAP = "a line and a signature from the tree"
|
|
1817
|
+
DRIFT_IN_MAP = "a signature from the tree"
|
|
1818
|
+
REACHABILITY = ("public", "internal-unreachable")
|
|
1819
|
+
VERIFIER_MISSING = "provenance: spot-checks not run: skf-verify-provenance-completeness.py is missing; re-install SKF"
|
|
1820
|
+
OUTCOMES = ("verified", "moved", "missing", "re-extracted", "rescoped", "unknown")
|
|
1821
|
+
_DEFINITION_LINES = re.compile(r"definition lines?\s*\(([\d,\s]+)\)", re.IGNORECASE)
|
|
1822
|
+
_VERIFIER = None
|
|
1823
|
+
|
|
1824
|
+
|
|
1825
|
+
def _verifier():
|
|
1826
|
+
"""skf-verify-provenance-completeness.py, loaded once from this folder for its definition-lines rules, or None
|
|
1827
|
+
when it cannot be loaded."""
|
|
1828
|
+
global _VERIFIER
|
|
1829
|
+
if _VERIFIER is None:
|
|
1830
|
+
path = Path(__file__).resolve().parent / "skf-verify-provenance-completeness.py"
|
|
1831
|
+
try:
|
|
1832
|
+
spec = importlib.util.spec_from_file_location("skf_verify_provenance_completeness", path)
|
|
1833
|
+
if spec is None or spec.loader is None:
|
|
1834
|
+
raise ImportError(f"cannot load {path}")
|
|
1835
|
+
module = importlib.util.module_from_spec(spec)
|
|
1836
|
+
spec.loader.exec_module(module)
|
|
1837
|
+
except (OSError, ImportError, SyntaxError):
|
|
1838
|
+
_VERIFIER = False
|
|
1839
|
+
else:
|
|
1840
|
+
_VERIFIER = module
|
|
1841
|
+
return _VERIFIER or None
|
|
1842
|
+
|
|
1843
|
+
|
|
1844
|
+
def _blocking(severity: object) -> bool:
|
|
1845
|
+
return str(severity or "").strip().lower() not in NON_BLOCKING
|
|
1846
|
+
|
|
1847
|
+
|
|
1848
|
+
def _gap_judgments(doc) -> dict:
|
|
1849
|
+
"""The answers gap-records reads from a --judgments file, checked. Raises ValueError."""
|
|
1850
|
+
if not isinstance(doc, dict) or not all(isinstance(v, dict) for v in doc.values()):
|
|
1851
|
+
raise ValueError("--judgments must map each gap id to an object of answers")
|
|
1852
|
+
for gid, answers in doc.items():
|
|
1853
|
+
line = answers.get("declaring_line")
|
|
1854
|
+
if "declaring_line" in answers and line is not None and (isinstance(line, bool) or not isinstance(line, int)):
|
|
1855
|
+
raise ValueError(f"--judgments: {gid}'s `declaring_line` must be a line number or null")
|
|
1856
|
+
if "reachability" in answers and answers["reachability"] not in REACHABILITY:
|
|
1857
|
+
raise ValueError(f"--judgments: {gid}'s `reachability` must be one of {', '.join(REACHABILITY)}")
|
|
1858
|
+
if "docstring" in answers and not isinstance(answers["docstring"], (str, type(None))):
|
|
1859
|
+
raise ValueError(f"--judgments: {gid}'s `docstring` must be a string or null")
|
|
1860
|
+
return doc
|
|
1861
|
+
|
|
1862
|
+
|
|
1863
|
+
def _drift_reason(entry: dict) -> str | None:
|
|
1864
|
+
"""The drift gate's reason an export-bearing entry needs the tree, or None for one that passes."""
|
|
1865
|
+
category = entry.get("change_category")
|
|
1866
|
+
if category == "DELETED_EXPORT":
|
|
1867
|
+
return DRIFT_RESCOPE
|
|
1868
|
+
if category not in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
1869
|
+
return None
|
|
1870
|
+
if entry.get("provenance_completeness"):
|
|
1871
|
+
return DRIFT_COMPLETENESS
|
|
1872
|
+
status = (entry.get("map_match") or {}).get("status")
|
|
1873
|
+
return DRIFT_IN_MAP if status in ("found", "ambiguous") else DRIFT_NOT_IN_MAP
|
|
1874
|
+
|
|
1875
|
+
|
|
1876
|
+
def _spot_check(root: Path | None, path: str | None, name: str, line: int | None, export_type: object,
|
|
1877
|
+
answers: dict) -> tuple[str, int | None, bool, str | None]:
|
|
1878
|
+
"""(outcome, the line a `moved` export moves to, whether it waits on a `declaring_line` answer, why the
|
|
1879
|
+
check failed)."""
|
|
1880
|
+
verifier = _verifier()
|
|
1881
|
+
if verifier is None or root is None or not path or line is None:
|
|
1882
|
+
return "unknown", None, False, None
|
|
1883
|
+
try:
|
|
1884
|
+
report = verifier.definition_lines_report(path, name, root, line,
|
|
1885
|
+
export_type if isinstance(export_type, str) else None)
|
|
1886
|
+
except Exception as exc: # noqa: BLE001 - a file it cannot read, or any verifier failure: this entry only
|
|
1887
|
+
return "unknown", None, False, f"{type(exc).__name__}: {exc}"
|
|
1888
|
+
check = report.get("line_check")
|
|
1889
|
+
if check == "file-missing":
|
|
1890
|
+
return "missing", None, False, None
|
|
1891
|
+
if check == "skipped-export-type":
|
|
1892
|
+
return "verified", None, False, None
|
|
1893
|
+
if check == "skipped-language":
|
|
1894
|
+
if "declaring_line" not in answers:
|
|
1895
|
+
return "unknown", None, True, None
|
|
1896
|
+
declared = answers["declaring_line"]
|
|
1897
|
+
if declared is None:
|
|
1898
|
+
return "unknown", None, False, None
|
|
1899
|
+
return ("verified", None, False, None) if declared == line else ("moved", declared, False, None)
|
|
1900
|
+
if report.get("line_is_definition"):
|
|
1901
|
+
return "verified", None, False, None
|
|
1902
|
+
defs = report.get("definition_lines") or []
|
|
1903
|
+
if len(defs) == 1 and defs[0] != line:
|
|
1904
|
+
return "moved", defs[0], False, None
|
|
1905
|
+
return "unknown", None, False, None
|
|
1906
|
+
|
|
1907
|
+
|
|
1908
|
+
def _pinned_lines(remediation: object) -> list[int] | None:
|
|
1909
|
+
"""The definition lines a provenance-line gap's remediation lists, `... definition line (5, 7) ...`."""
|
|
1910
|
+
m = _DEFINITION_LINES.search(remediation) if isinstance(remediation, str) else None
|
|
1911
|
+
lines = [int(n) for n in re.findall(r"\d+", m.group(1))] if m else []
|
|
1912
|
+
return lines or None
|
|
1913
|
+
|
|
1914
|
+
|
|
1915
|
+
def _where(path: object, line: object) -> str | None:
|
|
1916
|
+
file, number = _norm_path(path), _line_of(line)
|
|
1917
|
+
return f"{file}:{number}" if file and number is not None else None
|
|
1918
|
+
|
|
1919
|
+
|
|
1920
|
+
class _GapEntry:
|
|
1921
|
+
"""One export-bearing manifest entry as gap-records routes it."""
|
|
1922
|
+
|
|
1923
|
+
def __init__(self, entry: dict, answers: dict):
|
|
1924
|
+
self.entry = entry
|
|
1925
|
+
self.answers = answers
|
|
1926
|
+
self.name = _name_of(entry) or ""
|
|
1927
|
+
self.category = entry.get("change_category")
|
|
1928
|
+
match = entry.get("map_match") if isinstance(entry.get("map_match"), dict) else {}
|
|
1929
|
+
self.match_status = match.get("status") or "not-found"
|
|
1930
|
+
found = match.get("entry") if self.match_status == "found" and isinstance(match.get("entry"), dict) else None
|
|
1931
|
+
self.map_entry = {"source_file": found.get("source_file"), "source_line": found.get("source_line")} \
|
|
1932
|
+
if found else None
|
|
1933
|
+
self.map_export_type = found.get("export_type") if found else None
|
|
1934
|
+
self.in_map = bool(match.get("candidates"))
|
|
1935
|
+
citation = entry.get("source_citation")
|
|
1936
|
+
self.citation = citation if isinstance(citation, dict) else None
|
|
1937
|
+
self.recorded: tuple[str | None, int | None] = (None, None) # the file and line a check read
|
|
1938
|
+
self.outcome: str | None = None
|
|
1939
|
+
self.new_line: int | None = None
|
|
1940
|
+
self.unknown_reason: str | None = None
|
|
1941
|
+
self.reextract = False
|
|
1942
|
+
self.record: dict | None = None # the §4a record it matched
|
|
1943
|
+
self.record_path: str | None = None
|
|
1944
|
+
self.waits = False # on a declaring_line answer
|
|
1945
|
+
self.failure: str | None = None # why its spot-check failed
|
|
1946
|
+
|
|
1947
|
+
@property
|
|
1948
|
+
def blocking(self) -> bool:
|
|
1949
|
+
return _blocking(self.entry.get("severity"))
|
|
1950
|
+
|
|
1951
|
+
@property
|
|
1952
|
+
def paths(self) -> list[str]:
|
|
1953
|
+
return [p for p in (_norm_path(p) for p in self.entry.get("resolved_paths") or []) if p]
|
|
1954
|
+
|
|
1955
|
+
def check(self, root: Path | None, path: object, line: object, export_type: object) -> None:
|
|
1956
|
+
self.recorded = (_norm_path(path), _line_of(line))
|
|
1957
|
+
self.outcome, self.new_line, self.waits, self.failure = _spot_check(
|
|
1958
|
+
root, self.recorded[0], self.name, self.recorded[1], export_type, self.answers)
|
|
1959
|
+
|
|
1960
|
+
def route(self, root: Path | None, drift: bool) -> None:
|
|
1961
|
+
"""Spot-check the entry, or send it to targeted re-extraction (see "Gap records" in the module docstring)."""
|
|
1962
|
+
if self.category == "DELETED_EXPORT":
|
|
1963
|
+
self.outcome = "rescoped"
|
|
1964
|
+
if self.map_entry:
|
|
1965
|
+
self.recorded = (_norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"]))
|
|
1966
|
+
return
|
|
1967
|
+
if self.category == "MOVED_EXPORT":
|
|
1968
|
+
if self.map_entry is None:
|
|
1969
|
+
self.outcome = "unknown"
|
|
1970
|
+
return
|
|
1971
|
+
self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
|
|
1972
|
+
if drift and self.outcome == "moved":
|
|
1973
|
+
self.outcome, self.new_line, self.unknown_reason = "unknown", None, "drift-override"
|
|
1974
|
+
return
|
|
1975
|
+
if self.match_status == "found":
|
|
1976
|
+
self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
|
|
1977
|
+
return
|
|
1978
|
+
if self.match_status == "ambiguous":
|
|
1979
|
+
self.outcome = "unknown"
|
|
1980
|
+
return
|
|
1981
|
+
if self.citation:
|
|
1982
|
+
self.check(root, self.citation.get("file"), self.citation.get("line"), None)
|
|
1983
|
+
if self.waits or self.outcome in ("verified", "moved"):
|
|
1984
|
+
return
|
|
1985
|
+
if self.entry.get("provenance_completeness") and root is not None:
|
|
1986
|
+
self.reextract = True
|
|
1987
|
+
elif self.blocking:
|
|
1988
|
+
self.reextract = True
|
|
1989
|
+
elif self.entry.get("category") in DOCUMENT_CATEGORIES and self.paths:
|
|
1990
|
+
self.reextract = True
|
|
1991
|
+
else:
|
|
1992
|
+
self.outcome = "unknown"
|
|
1993
|
+
|
|
1994
|
+
def match(self, blocks: dict[str, list[dict]]) -> tuple[int, int]:
|
|
1995
|
+
"""Take the first export of the entry's name in the blocks of its resolved files; (files scanned, exports
|
|
1996
|
+
found in them)."""
|
|
1997
|
+
exports = 0
|
|
1998
|
+
for path in self.paths:
|
|
1999
|
+
for record in blocks.get(path, []):
|
|
2000
|
+
exports += 1
|
|
2001
|
+
if self.record is None and _name_of(record) == self.name:
|
|
2002
|
+
self.record, self.record_path = record, path
|
|
2003
|
+
if self.record is not None:
|
|
2004
|
+
self.outcome = "re-extracted"
|
|
2005
|
+
return len(self.paths), exports
|
|
2006
|
+
|
|
2007
|
+
@property
|
|
2008
|
+
def located(self) -> bool:
|
|
2009
|
+
return self.outcome in ("verified", "moved", "re-extracted")
|
|
2010
|
+
|
|
2011
|
+
@property
|
|
2012
|
+
def internal(self) -> bool:
|
|
2013
|
+
return self.category == "NEW_EXPORT" and self.answers.get("reachability") == "internal-unreachable"
|
|
2014
|
+
|
|
2015
|
+
def needs(self) -> list[str]:
|
|
2016
|
+
"""The answers the entry still waits on once routed."""
|
|
2017
|
+
want = []
|
|
2018
|
+
if self.category == "NEW_EXPORT" and self.located and "reachability" not in self.answers:
|
|
2019
|
+
want.append("reachability")
|
|
2020
|
+
if self.outcome == "re-extracted" and "docstring" not in self.answers:
|
|
2021
|
+
want.append("docstring")
|
|
2022
|
+
return want
|
|
2023
|
+
|
|
2024
|
+
def new_location(self) -> str | None:
|
|
2025
|
+
if self.outcome == "moved":
|
|
2026
|
+
return _where(self.recorded[0], self.new_line)
|
|
2027
|
+
if self.outcome == "re-extracted":
|
|
2028
|
+
return (self.record or {}).get("location")
|
|
2029
|
+
return None
|
|
2030
|
+
|
|
2031
|
+
def verification(self) -> dict:
|
|
2032
|
+
recorded = _where(*self.recorded)
|
|
2033
|
+
if self.outcome == "re-extracted":
|
|
2034
|
+
citation = self.new_location()
|
|
2035
|
+
elif self.outcome in ("verified", "moved", "missing", "rescoped") and recorded:
|
|
2036
|
+
citation = recorded
|
|
2037
|
+
else:
|
|
2038
|
+
citation = "unknown"
|
|
2039
|
+
return {
|
|
2040
|
+
"export_name": self.name,
|
|
2041
|
+
"gap_category": self.category,
|
|
2042
|
+
"severity": self.entry.get("severity"),
|
|
2043
|
+
"verification": self.outcome,
|
|
2044
|
+
"in_map": self.in_map,
|
|
2045
|
+
"map_entry": self.map_entry,
|
|
2046
|
+
"provenance_citation": citation,
|
|
2047
|
+
"source_citation": self.citation,
|
|
2048
|
+
"new_location": self.new_location(),
|
|
2049
|
+
"unknown_reason": self.unknown_reason,
|
|
2050
|
+
"pinned_definition_lines": _pinned_lines(self.entry.get("remediation"))
|
|
2051
|
+
if self.category == "MOVED_EXPORT" else None,
|
|
2052
|
+
"resolution_source": "remediation-paths" if self.outcome == "re-extracted" else None,
|
|
2053
|
+
"reachability": self.answers.get("reachability") if self.category == "NEW_EXPORT" and self.located
|
|
2054
|
+
else None,
|
|
2055
|
+
"export_type": None,
|
|
2056
|
+
}
|
|
2057
|
+
|
|
2058
|
+
def label(self, provenance: list[dict]) -> str:
|
|
2059
|
+
"""T1, T1-low or unlabeled: the confidence_breakdown bin of the entry."""
|
|
2060
|
+
if self.internal:
|
|
2061
|
+
return "unlabeled"
|
|
2062
|
+
method = None
|
|
2063
|
+
if self.outcome == "re-extracted":
|
|
2064
|
+
method = (self.record or {}).get("extraction_method")
|
|
2065
|
+
elif self.map_entry is not None:
|
|
2066
|
+
file, line = _norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"])
|
|
2067
|
+
for entry in provenance:
|
|
2068
|
+
if (_name_of(entry) == self.name and _file_of(entry) == file
|
|
2069
|
+
and _line_of(entry.get("source_line")) == line):
|
|
2070
|
+
method = entry.get("extraction_method")
|
|
2071
|
+
break
|
|
2072
|
+
elif self.located and self.category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
|
|
2073
|
+
method = "source-read" # a pinned cited export the map lacks: write.md §3 writes it source-read
|
|
2074
|
+
if method in ("ast-grep", "ast_bridge"):
|
|
2075
|
+
return "T1"
|
|
2076
|
+
if method in ("source-read", "source_reading"):
|
|
2077
|
+
return "T1-low"
|
|
2078
|
+
return "unlabeled"
|
|
2079
|
+
|
|
2080
|
+
|
|
2081
|
+
def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path | None, drift_status: str = "ok",
|
|
2082
|
+
judgments: dict | None = None, remediation: dict | None = None, plan: bool = False,
|
|
2083
|
+
tier: str | None = None) -> tuple[dict | None, dict]:
|
|
2084
|
+
"""(reextract-records.json or None, the summary): see "Gap records" in the module docstring."""
|
|
2085
|
+
if not isinstance(manifest, dict) or not isinstance(manifest.get("entries"), list):
|
|
2086
|
+
raise ValueError("the change manifest has no `entries` array: is it translate's output?")
|
|
2087
|
+
entries = [e for e in manifest["entries"] if isinstance(e, dict)]
|
|
2088
|
+
judgments = judgments or {}
|
|
2089
|
+
root = source_root if source_root is not None and source_root.is_dir() else None
|
|
2090
|
+
forwarded = [{"gap_id": e.get("gap_id"), "change_category": e.get("change_category")}
|
|
2091
|
+
for e in entries if e.get("change_category") not in EXPORT_CATEGORIES]
|
|
2092
|
+
routed = [_GapEntry(e, judgments.get(e.get("gap_id")) or {}) for e in entries
|
|
2093
|
+
if e.get("change_category") in EXPORT_CATEGORIES]
|
|
2094
|
+
drift = drift_status == "overridden"
|
|
2095
|
+
if drift:
|
|
2096
|
+
blocked = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "change_category": g.category,
|
|
2097
|
+
"severity": g.entry.get("severity"), "reason": _drift_reason(g.entry)}
|
|
2098
|
+
for g in routed if _drift_reason(g.entry)]
|
|
2099
|
+
if blocked:
|
|
2100
|
+
return None, {"status": "drift-blocked", "drift_blocked": blocked}
|
|
2101
|
+
bare = [g.name for g in routed if g.category == "DELETED_EXPORT" and not (
|
|
2102
|
+
isinstance(g.entry.get("rescope"), dict) and g.entry["rescope"].get("amendment")
|
|
2103
|
+
and g.entry["rescope"].get("exclude"))]
|
|
2104
|
+
if bare:
|
|
2105
|
+
return None, {"status": "blocked", "rescope_without_amendment": bare}
|
|
2106
|
+
warnings = []
|
|
2107
|
+
if root is not None and _verifier() is None:
|
|
2108
|
+
warnings.append(VERIFIER_MISSING)
|
|
2109
|
+
for g in routed:
|
|
2110
|
+
g.route(root, drift)
|
|
2111
|
+
warnings += [f"provenance: spot-check failed for {g.name}: {g.failure}" for g in routed if g.failure]
|
|
2112
|
+
waiting = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": ["declaring_line"],
|
|
2113
|
+
"file": g.recorded[0], "line": g.recorded[1]} for g in routed if g.waits]
|
|
2114
|
+
if waiting:
|
|
2115
|
+
return None, {"status": "needs-judgment", "needs_judgment": waiting, "warnings": warnings}
|
|
2116
|
+
to_scan = [g for g in routed if g.reextract]
|
|
2117
|
+
files = list(dict.fromkeys(p for g in to_scan for p in g.paths))
|
|
2118
|
+
if plan:
|
|
2119
|
+
return None, {"status": "planned", "reextract": {
|
|
2120
|
+
"entries": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
|
|
2121
|
+
"resolved_paths": g.paths} for g in to_scan], "files": files}, "warnings": warnings}
|
|
2122
|
+
blocks: dict[str, list[dict]] = {}
|
|
2123
|
+
scanned = 0
|
|
2124
|
+
if files:
|
|
2125
|
+
if remediation is None:
|
|
2126
|
+
raise ValueError("targeted re-extraction has files to match: pass --remediation-records, the output "
|
|
2127
|
+
"of `records` over them")
|
|
2128
|
+
if not isinstance(remediation, dict) or not isinstance(remediation.get("files"), list):
|
|
2129
|
+
raise ValueError("--remediation-records has no `files` array: is it the output of `records`?")
|
|
2130
|
+
for block in remediation["files"]:
|
|
2131
|
+
if isinstance(block, dict) and _norm_path(block.get("file_path")):
|
|
2132
|
+
scanned += 1
|
|
2133
|
+
blocks.setdefault(_norm_path(block["file_path"]), []).extend(
|
|
2134
|
+
r for r in block.get("exports") or [] if isinstance(r, dict))
|
|
2135
|
+
unresolved = []
|
|
2136
|
+
for g in to_scan:
|
|
2137
|
+
files_scanned, found = g.match(blocks)
|
|
2138
|
+
if g.record is not None:
|
|
2139
|
+
continue
|
|
2140
|
+
if not g.blocking and g.entry.get("category") in DOCUMENT_CATEGORIES:
|
|
2141
|
+
g.outcome = "unknown"
|
|
2142
|
+
continue
|
|
2143
|
+
unresolved.append({"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
|
|
2144
|
+
"remediation_paths": g.entry.get("remediation_paths") or [],
|
|
2145
|
+
"rejected_paths": g.entry.get("rejected_paths") or [],
|
|
2146
|
+
"files_scanned": files_scanned, "exports_found_in_scan": found})
|
|
2147
|
+
if unresolved:
|
|
2148
|
+
return None, {"status": "unresolved", "unresolved": unresolved, "warnings": warnings}
|
|
2149
|
+
needs = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": g.needs(),
|
|
2150
|
+
"file": _location(g.new_location())[0] if g.outcome != "verified" else g.recorded[0],
|
|
2151
|
+
"line": _location(g.new_location())[1] if g.outcome != "verified" else g.recorded[1]}
|
|
2152
|
+
for g in routed if g.needs()]
|
|
2153
|
+
if needs:
|
|
2154
|
+
return None, {"status": "needs-judgment", "needs_judgment": needs, "warnings": warnings}
|
|
2155
|
+
kept: dict[str, list[dict]] = {}
|
|
2156
|
+
for g in routed:
|
|
2157
|
+
if g.record is not None and not g.internal:
|
|
2158
|
+
record = dict(g.record)
|
|
2159
|
+
if g.answers.get("docstring") is not None:
|
|
2160
|
+
record["docstring"] = g.answers["docstring"]
|
|
2161
|
+
if record not in kept.setdefault(g.record_path, []):
|
|
2162
|
+
kept[g.record_path].append(record)
|
|
2163
|
+
breakdown = {"T1": 0, "T1-low": 0, "unlabeled": 0, "T2": 0}
|
|
2164
|
+
provenance_entries = _entries_of(provenance)
|
|
2165
|
+
for g in routed:
|
|
2166
|
+
breakdown[g.label(provenance_entries)] += 1
|
|
2167
|
+
breakdown["unlabeled"] += len(forwarded)
|
|
2168
|
+
records = {
|
|
2169
|
+
"mode": "gap-driven",
|
|
2170
|
+
"files_extracted": scanned,
|
|
2171
|
+
"exports_extracted": len(entries),
|
|
2172
|
+
"confidence_breakdown": breakdown,
|
|
2173
|
+
"verification": [g.verification() for g in routed],
|
|
2174
|
+
"files": [{"file_path": path, "exports": exports} for path, exports in kept.items()],
|
|
2175
|
+
}
|
|
2176
|
+
matched = [g for g in to_scan if g.record is not None]
|
|
2177
|
+
targeted = {"resolved_count": len(matched), "files_scanned": scanned,
|
|
2178
|
+
"exports_matched": len({(g.record_path, _name_of(g.record)) for g in matched}),
|
|
2179
|
+
"tier": tier} if to_scan else None
|
|
2180
|
+
summary = {
|
|
2181
|
+
"status": "written",
|
|
2182
|
+
"gap_count": len(entries),
|
|
2183
|
+
"files_extracted": scanned,
|
|
2184
|
+
"exports_extracted": len(entries),
|
|
2185
|
+
"confidence_breakdown": breakdown,
|
|
2186
|
+
"counts": {**{outcome: sum(1 for g in routed if g.outcome == outcome and not g.internal)
|
|
2187
|
+
for outcome in OUTCOMES}, "reclassified": sum(1 for g in routed if g.internal)},
|
|
2188
|
+
"reclassified": [g.name for g in routed if g.internal],
|
|
2189
|
+
"forwarded": forwarded,
|
|
2190
|
+
"targeted_reextraction": targeted,
|
|
2191
|
+
"warnings": warnings,
|
|
2192
|
+
}
|
|
2193
|
+
return records, summary
|
|
2194
|
+
|
|
2195
|
+
|
|
340
2196
|
# --------------------------------------------------------------------------
|
|
341
2197
|
# CLI
|
|
342
2198
|
# --------------------------------------------------------------------------
|
|
343
2199
|
|
|
344
2200
|
|
|
2201
|
+
def _category_c(args: argparse.Namespace, own: dict | None) -> dict | None:
|
|
2202
|
+
"""Category C from --category-c and --ccc-pairs, else the input's own
|
|
2203
|
+
(`own`, None when there is none). Raises ValueError."""
|
|
2204
|
+
if not args.category_c and not args.ccc_pairs:
|
|
2205
|
+
return own
|
|
2206
|
+
cat_c = dict(own or {})
|
|
2207
|
+
if args.category_c:
|
|
2208
|
+
doc = _load_helper_file(Path(args.category_c), "--category-c file", "category_c")
|
|
2209
|
+
if not isinstance(doc["category_c"], dict):
|
|
2210
|
+
raise ValueError(f"--category-c file {args.category_c} has no `category_c` object")
|
|
2211
|
+
cat_c = dict(doc["category_c"])
|
|
2212
|
+
if args.ccc_pairs:
|
|
2213
|
+
pairs = _load_json_file(Path(args.ccc_pairs), "--ccc-pairs file")
|
|
2214
|
+
if not isinstance(pairs, dict):
|
|
2215
|
+
raise ValueError(f"--ccc-pairs file {args.ccc_pairs} must hold a JSON object")
|
|
2216
|
+
for key in ("renamed_files", "renamed_exports"):
|
|
2217
|
+
cat_c[key] = list(_get_list(cat_c, key)) + [p for p in _get_list(pairs, key) if isinstance(p, dict)]
|
|
2218
|
+
return cat_c
|
|
2219
|
+
|
|
2220
|
+
|
|
2221
|
+
def _payload(args: argparse.Namespace) -> dict:
|
|
2222
|
+
"""The category JSON of a build or deletion-ratio call: the input with
|
|
2223
|
+
the helper files' slices assembled in. Raises ValueError."""
|
|
2224
|
+
payload = _load_input(Path(args.input) if args.input else None)
|
|
2225
|
+
cat_c = _category_c(args, payload.get("category_c") if isinstance(payload.get("category_c"), dict) else None)
|
|
2226
|
+
if cat_c is not None:
|
|
2227
|
+
payload = {**payload, "category_c": cat_c}
|
|
2228
|
+
category_a_doc = (
|
|
2229
|
+
_load_helper_file(Path(args.category_a), "--category-a file", "category_a") if args.category_a else None
|
|
2230
|
+
)
|
|
2231
|
+
diff = _load_helper_file(Path(args.category_b_diff), "--category-b-diff file", "removed") \
|
|
2232
|
+
if args.category_b_diff else None
|
|
2233
|
+
return assemble(payload, category_a_doc=category_a_doc, diff=diff)
|
|
2234
|
+
|
|
2235
|
+
|
|
2236
|
+
def docs_only_manifest(doc_hashes: dict) -> dict:
|
|
2237
|
+
"""A docs-only skill's change manifest from compare-hashes' output."""
|
|
2238
|
+
def urls(key: str) -> list[str]:
|
|
2239
|
+
return [e["url"] for e in _get_list(doc_hashes, key) if isinstance(e, dict) and isinstance(e.get("url"), str)]
|
|
2240
|
+
|
|
2241
|
+
changed, failed = urls("changed"), urls("fetch_failed")
|
|
2242
|
+
return {"mode": "docs-only", "no_changes": not changed, "changed_urls": changed, "fetch_failed": failed,
|
|
2243
|
+
"counts": {"docs_changed": len(changed), "docs_fetch_failed": len(failed)}}
|
|
2244
|
+
|
|
2245
|
+
|
|
2246
|
+
def _category_d(args: argparse.Namespace) -> dict | None:
|
|
2247
|
+
"""Category D from --file-compare (typed by --provenance-map) and --new-files, else None. Raises ValueError."""
|
|
2248
|
+
if not args.file_compare and not args.new_files:
|
|
2249
|
+
if args.provenance_map:
|
|
2250
|
+
raise ValueError("--provenance-map goes with --file-compare")
|
|
2251
|
+
return None
|
|
2252
|
+
if args.file_compare and not args.provenance_map:
|
|
2253
|
+
raise ValueError("--file-compare needs --provenance-map, which types each row by its file_entries[] row")
|
|
2254
|
+
compare = _load_helper_file(Path(args.file_compare), "--file-compare file", "comparisons") \
|
|
2255
|
+
if args.file_compare else None
|
|
2256
|
+
new_files = _load_helper_file(Path(args.new_files), "--new-files file", "new_files") if args.new_files else None
|
|
2257
|
+
provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file") if args.provenance_map else None
|
|
2258
|
+
if provenance is not None and not isinstance(provenance, dict):
|
|
2259
|
+
raise ValueError(f"--provenance-map file {args.provenance_map} must hold a JSON object")
|
|
2260
|
+
return category_d_from_files(compare, new_files, provenance)
|
|
2261
|
+
|
|
2262
|
+
|
|
345
2263
|
def _cmd_build(args: argparse.Namespace) -> int:
|
|
346
2264
|
try:
|
|
347
|
-
|
|
2265
|
+
if args.doc_hashes:
|
|
2266
|
+
if (args.input or args.category_a or args.category_b_diff or args.category_c or args.ccc_pairs
|
|
2267
|
+
or args.file_compare or args.new_files or args.provenance_map):
|
|
2268
|
+
raise ValueError("--doc-hashes takes no other input")
|
|
2269
|
+
doc_hashes = _load_helper_file(Path(args.doc_hashes), "--doc-hashes file", "changed")
|
|
2270
|
+
manifest = docs_only_manifest(doc_hashes)
|
|
2271
|
+
else:
|
|
2272
|
+
payload = _payload(args)
|
|
2273
|
+
cat_d = _category_d(args)
|
|
2274
|
+
if cat_d is not None:
|
|
2275
|
+
payload = {**payload, "category_d": cat_d}
|
|
2276
|
+
manifest = build_manifest(payload)
|
|
348
2277
|
except ValueError as exc:
|
|
349
2278
|
print(f"error: {exc}", file=sys.stderr)
|
|
350
2279
|
return 1
|
|
351
|
-
manifest = build_manifest(payload)
|
|
352
2280
|
json.dump(manifest, sys.stdout, indent=2)
|
|
353
2281
|
sys.stdout.write("\n")
|
|
354
2282
|
return 0
|
|
@@ -356,7 +2284,7 @@ def _cmd_build(args: argparse.Namespace) -> int:
|
|
|
356
2284
|
|
|
357
2285
|
def _cmd_deletion_ratio(args: argparse.Namespace) -> int:
|
|
358
2286
|
try:
|
|
359
|
-
payload =
|
|
2287
|
+
payload = _payload(args)
|
|
360
2288
|
except ValueError as exc:
|
|
361
2289
|
print(f"error: {exc}", file=sys.stderr)
|
|
362
2290
|
return 1
|
|
@@ -379,6 +2307,163 @@ def _cmd_deletion_ratio(args: argparse.Namespace) -> int:
|
|
|
379
2307
|
return 0
|
|
380
2308
|
|
|
381
2309
|
|
|
2310
|
+
def _optional_json(value: str | None, what: str):
|
|
2311
|
+
return _load_json_file(Path(value), what) if value else None
|
|
2312
|
+
|
|
2313
|
+
|
|
2314
|
+
def _emit(result: dict, output: str | None) -> None:
|
|
2315
|
+
if output:
|
|
2316
|
+
_write_json_atomic(Path(output), result)
|
|
2317
|
+
print(json.dumps({"status": "ok", "output": output}))
|
|
2318
|
+
else:
|
|
2319
|
+
json.dump(result, sys.stdout, indent=2)
|
|
2320
|
+
sys.stdout.write("\n")
|
|
2321
|
+
|
|
2322
|
+
|
|
2323
|
+
def _cmd_rename_candidates(args: argparse.Namespace) -> int:
|
|
2324
|
+
try:
|
|
2325
|
+
category_a_doc = _load_helper_file(Path(args.category_a), "--category-a file", "category_a")
|
|
2326
|
+
diff = _load_helper_file(Path(args.category_b_diff), "--category-b-diff file", "removed") \
|
|
2327
|
+
if args.category_b_diff else None
|
|
2328
|
+
provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file")
|
|
2329
|
+
result = rename_candidates(
|
|
2330
|
+
category_a_doc, diff, provenance,
|
|
2331
|
+
_optional_json(args.extraction, "--extraction file"),
|
|
2332
|
+
_optional_json(args.export_details, "--export-details file"),
|
|
2333
|
+
args.tier,
|
|
2334
|
+
source_root=Path(args.source_root) if args.source_root else None,
|
|
2335
|
+
sizes_commit=args.sizes_commit or None,
|
|
2336
|
+
)
|
|
2337
|
+
_emit(result, args.output)
|
|
2338
|
+
except ValueError as exc:
|
|
2339
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2340
|
+
return 1
|
|
2341
|
+
except OSError as exc:
|
|
2342
|
+
print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
|
|
2343
|
+
return 1
|
|
2344
|
+
return 0
|
|
2345
|
+
|
|
2346
|
+
|
|
2347
|
+
def _apply_categories(args: argparse.Namespace) -> dict | None:
|
|
2348
|
+
"""The category JSON apply reads: --input, with --category-c and --ccc-pairs in place of its category_c."""
|
|
2349
|
+
categories = _optional_json(args.input, "--input file")
|
|
2350
|
+
if categories is not None and not isinstance(categories, dict):
|
|
2351
|
+
raise ValueError("--input file must hold a JSON object")
|
|
2352
|
+
own = (categories or {}).get("category_c")
|
|
2353
|
+
cat_c = _category_c(args, own if isinstance(own, dict) else None)
|
|
2354
|
+
if cat_c is None:
|
|
2355
|
+
return categories
|
|
2356
|
+
return {**(categories or {}), "category_c": cat_c}
|
|
2357
|
+
|
|
2358
|
+
|
|
2359
|
+
def _cmd_apply(args: argparse.Namespace) -> int:
|
|
2360
|
+
if bool(args.drift_head) != bool(args.drift_pinned):
|
|
2361
|
+
print("error: --drift-head and --drift-pinned go together", file=sys.stderr)
|
|
2362
|
+
return 1
|
|
2363
|
+
try:
|
|
2364
|
+
promoted = _optional_json(args.promoted_docs, "--promoted-docs file")
|
|
2365
|
+
if promoted is not None and not isinstance(promoted, list):
|
|
2366
|
+
raise ValueError("--promoted-docs file must hold a JSON array")
|
|
2367
|
+
doc, summary = apply_update(
|
|
2368
|
+
update_type=args.update_type,
|
|
2369
|
+
provenance=_optional_json(args.provenance_map, "--provenance-map file"),
|
|
2370
|
+
skill_name=args.skill_name,
|
|
2371
|
+
manifest=_optional_json(args.manifest, "--manifest file"),
|
|
2372
|
+
categories=_apply_categories(args),
|
|
2373
|
+
extraction=_optional_json(args.extraction, "--extraction file"),
|
|
2374
|
+
details=_optional_json(args.export_details, "--export-details file"),
|
|
2375
|
+
records=_optional_json(args.reextract_records, "--reextract-records file"),
|
|
2376
|
+
merge_records=_optional_json(args.merge_records, "--merge-records file"),
|
|
2377
|
+
compare=_optional_json(args.file_compare, "--file-compare file"),
|
|
2378
|
+
new_files=_optional_json(args.new_files, "--new-files file"),
|
|
2379
|
+
promoted=promoted,
|
|
2380
|
+
source_root=Path(args.source_root) if args.source_root else None,
|
|
2381
|
+
generation_date=args.generation_date,
|
|
2382
|
+
test_report_run_id=args.test_report_run_id,
|
|
2383
|
+
confidence_tier=args.confidence_tier,
|
|
2384
|
+
manual_sections_preserved=args.manual_sections_preserved,
|
|
2385
|
+
source_commit=args.source_commit,
|
|
2386
|
+
source_ref=args.source_ref,
|
|
2387
|
+
drift=(args.drift_head, args.drift_pinned) if args.drift_head else None,
|
|
2388
|
+
)
|
|
2389
|
+
except ValueError as exc:
|
|
2390
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2391
|
+
return 1
|
|
2392
|
+
if summary["status"] == "refused":
|
|
2393
|
+
print(json.dumps(summary))
|
|
2394
|
+
return EXIT_REFUSED
|
|
2395
|
+
try:
|
|
2396
|
+
_write_json_atomic(Path(args.output), doc)
|
|
2397
|
+
except OSError as exc:
|
|
2398
|
+
print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
|
|
2399
|
+
return 1
|
|
2400
|
+
print(json.dumps({**summary, "map": args.output}))
|
|
2401
|
+
return 0
|
|
2402
|
+
|
|
2403
|
+
|
|
2404
|
+
def _cmd_records(args: argparse.Namespace) -> int:
|
|
2405
|
+
try:
|
|
2406
|
+
files = _optional_json(args.files_from, "--files-from file")
|
|
2407
|
+
if files is not None and not (isinstance(files, list) and all(isinstance(f, str) for f in files)):
|
|
2408
|
+
raise ValueError(f"--files-from file {args.files_from} must hold a JSON array of paths")
|
|
2409
|
+
records, summary = build_records(
|
|
2410
|
+
_optional_json(args.extraction, "--extraction file"),
|
|
2411
|
+
_optional_json(args.export_details, "--export-details file"),
|
|
2412
|
+
files,
|
|
2413
|
+
_patch_blocks(Path(args.patches)) if args.patches else [],
|
|
2414
|
+
)
|
|
2415
|
+
_write_json_atomic(Path(args.output), records)
|
|
2416
|
+
except ValueError as exc:
|
|
2417
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2418
|
+
return 1
|
|
2419
|
+
except OSError as exc:
|
|
2420
|
+
print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
|
|
2421
|
+
return 1
|
|
2422
|
+
print(json.dumps({"status": "written", "output": args.output, **summary}))
|
|
2423
|
+
return 0
|
|
2424
|
+
|
|
2425
|
+
|
|
2426
|
+
def _cmd_gap_records(args: argparse.Namespace) -> int:
|
|
2427
|
+
try:
|
|
2428
|
+
if args.plan and args.output:
|
|
2429
|
+
raise ValueError("--plan writes no records: drop -o")
|
|
2430
|
+
if not args.plan and not args.output:
|
|
2431
|
+
raise ValueError("-o is required without --plan")
|
|
2432
|
+
if args.files_out and not args.plan:
|
|
2433
|
+
raise ValueError("--files-out goes with --plan")
|
|
2434
|
+
if bool(args.evidence) != bool(args.tier):
|
|
2435
|
+
raise ValueError("--evidence and --tier go together")
|
|
2436
|
+
judgments = _gap_judgments(_load_json_file(Path(args.judgments), "--judgments file")) \
|
|
2437
|
+
if args.judgments else {}
|
|
2438
|
+
records, summary = gap_records(
|
|
2439
|
+
_load_json_file(Path(args.manifest), "--manifest file"),
|
|
2440
|
+
_load_json_file(Path(args.provenance_map), "--provenance-map file"),
|
|
2441
|
+
source_root=Path(args.source_root) if args.source_root else None,
|
|
2442
|
+
drift_status=args.drift_status,
|
|
2443
|
+
judgments=judgments,
|
|
2444
|
+
remediation=_optional_json(args.remediation_records, "--remediation-records file"),
|
|
2445
|
+
plan=args.plan,
|
|
2446
|
+
tier=args.tier,
|
|
2447
|
+
)
|
|
2448
|
+
if summary["status"] == "planned" and args.files_out:
|
|
2449
|
+
_write_json_atomic(Path(args.files_out), summary["reextract"]["files"])
|
|
2450
|
+
if records is not None:
|
|
2451
|
+
_write_json_atomic(Path(args.output), records)
|
|
2452
|
+
summary = {"status": summary["status"], "output": args.output,
|
|
2453
|
+
**{k: v for k, v in summary.items() if k != "status"}}
|
|
2454
|
+
if args.evidence and summary["targeted_reextraction"] is not None:
|
|
2455
|
+
with open(args.evidence, "a", encoding="utf-8", newline="\n") as fh:
|
|
2456
|
+
fh.write(json.dumps({"targeted_reextraction": summary["targeted_reextraction"]}) + "\n")
|
|
2457
|
+
except ValueError as exc:
|
|
2458
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2459
|
+
return 1
|
|
2460
|
+
except OSError as exc:
|
|
2461
|
+
print(f"error: cannot write: {exc}", file=sys.stderr)
|
|
2462
|
+
return 1
|
|
2463
|
+
print(json.dumps(summary, indent=2))
|
|
2464
|
+
return 0
|
|
2465
|
+
|
|
2466
|
+
|
|
382
2467
|
def _build_parser() -> argparse.ArgumentParser:
|
|
383
2468
|
parser = argparse.ArgumentParser(
|
|
384
2469
|
prog="skf-build-change-manifest",
|
|
@@ -390,23 +2475,117 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
390
2475
|
sub = parser.add_subparsers(dest="cmd", required=True)
|
|
391
2476
|
|
|
392
2477
|
p_build = sub.add_parser("build", help="aggregate categories into manifest")
|
|
393
|
-
p_build.add_argument(
|
|
394
|
-
|
|
395
|
-
|
|
2478
|
+
p_build.add_argument("--doc-hashes", metavar="FILE",
|
|
2479
|
+
help="a docs-only skill: skf-detect-docs.py compare-hashes output, the only input")
|
|
2480
|
+
p_build.add_argument("--file-compare", metavar="FILE",
|
|
2481
|
+
help="skf-hash-content.py compare output: Category D's modified and deleted files")
|
|
2482
|
+
p_build.add_argument("--new-files", metavar="FILE", help="skf-new-file-diff.py output: Category D's new files")
|
|
2483
|
+
p_build.add_argument("--provenance-map", metavar="FILE",
|
|
2484
|
+
help="the map --file-compare compared: each row's file_type")
|
|
396
2485
|
p_build.set_defaults(func=_cmd_build)
|
|
397
2486
|
|
|
398
2487
|
p_ratio = sub.add_parser(
|
|
399
2488
|
"deletion-ratio",
|
|
400
2489
|
help="compute §2.2 trigger; requires --provenance-map",
|
|
401
2490
|
)
|
|
402
|
-
p_ratio.add_argument(
|
|
403
|
-
"--input", default=None, help="path to JSON input (default: stdin)"
|
|
404
|
-
)
|
|
405
2491
|
p_ratio.add_argument(
|
|
406
2492
|
"--provenance-map", required=True, help="path to provenance-map.json"
|
|
407
2493
|
)
|
|
408
2494
|
p_ratio.set_defaults(func=_cmd_deletion_ratio)
|
|
409
2495
|
|
|
2496
|
+
p_rename = sub.add_parser(
|
|
2497
|
+
"rename-candidates",
|
|
2498
|
+
help="Category C by fixed rules: renamed files and renamed exports",
|
|
2499
|
+
)
|
|
2500
|
+
p_rename.add_argument("--category-a", required=True, metavar="FILE",
|
|
2501
|
+
help="skf-classify-changed-files.py classify output (deleted and added files)")
|
|
2502
|
+
p_rename.add_argument("--category-b-diff", metavar="FILE",
|
|
2503
|
+
help="skf-structural-diff.py output (removed and added exports)")
|
|
2504
|
+
p_rename.add_argument("--provenance-map", required=True, metavar="FILE",
|
|
2505
|
+
help="the provenance map: the deleted files' exports")
|
|
2506
|
+
p_rename.add_argument("--extraction", metavar="FILE", help="the recipe runner's output: the added files' exports")
|
|
2507
|
+
p_rename.add_argument("--export-details", metavar="FILE",
|
|
2508
|
+
help="the workers' export details: params, return types and exports read by eye")
|
|
2509
|
+
p_rename.add_argument("--tier", required=True, choices=TIERS, help="the forge tier, which picks the rules")
|
|
2510
|
+
p_rename.add_argument("--source-root", metavar="DIR", help="the source tree: the added files' sizes")
|
|
2511
|
+
p_rename.add_argument("--sizes-commit", metavar="COMMIT",
|
|
2512
|
+
help="the commit the deleted files' sizes are read at (git cat-file -s)")
|
|
2513
|
+
p_rename.add_argument("-o", "--output", metavar="FILE", help="write the JSON here instead of stdout")
|
|
2514
|
+
p_rename.set_defaults(func=_cmd_rename_candidates)
|
|
2515
|
+
|
|
2516
|
+
p_apply = sub.add_parser("apply", help="write the provenance map an update leaves")
|
|
2517
|
+
p_apply.add_argument("--update-type", required=True, choices=UPDATE_TYPES,
|
|
2518
|
+
help="incremental (normal mode), gap-driven, or full (degraded mode)")
|
|
2519
|
+
p_apply.add_argument("--provenance-map", metavar="FILE", help="the map the update started from")
|
|
2520
|
+
p_apply.add_argument("--manifest", metavar="FILE", help="the change manifest (build's output)")
|
|
2521
|
+
p_apply.add_argument("--input", metavar="FILE", help="the category JSON: category_c's renamed exports")
|
|
2522
|
+
p_apply.add_argument("--extraction", metavar="FILE", help="the recipe runner's output")
|
|
2523
|
+
p_apply.add_argument("--export-details", metavar="FILE", help="the workers' export details")
|
|
2524
|
+
p_apply.add_argument("--reextract-records", metavar="FILE", help="step 3's records")
|
|
2525
|
+
p_apply.add_argument("--merge-records", metavar="FILE", help="merge's records of the exports no fresh record holds")
|
|
2526
|
+
p_apply.add_argument("--file-compare", metavar="FILE", help="skf-hash-content.py compare output")
|
|
2527
|
+
p_apply.add_argument("--new-files", metavar="FILE", help="skf-new-file-diff.py output")
|
|
2528
|
+
p_apply.add_argument("--promoted-docs", metavar="FILE", help="the documents step 2 promoted, with their hashes")
|
|
2529
|
+
p_apply.add_argument("--source-root", metavar="DIR", help="the source tree: the new files' hashes")
|
|
2530
|
+
p_apply.add_argument("--skill-name", required=True, help="the skill name: source_library of a new entry")
|
|
2531
|
+
p_apply.add_argument("--generation-date", required=True, help="last_update: the generation_date written")
|
|
2532
|
+
p_apply.add_argument("--test-report-run-id", default=None, help="the run id of the test report applied")
|
|
2533
|
+
p_apply.add_argument("--confidence-tier", required=True, help="the forge tier the update ran at")
|
|
2534
|
+
p_apply.add_argument("--manual-sections-preserved", required=True, type=int,
|
|
2535
|
+
help="the [MANUAL] blocks the post-merge check found intact")
|
|
2536
|
+
p_apply.add_argument("--source-commit", default=None, help="the map's source_commit (empty: null)")
|
|
2537
|
+
p_apply.add_argument("--source-ref", default=None, help="the map's source_ref (empty: null)")
|
|
2538
|
+
p_apply.add_argument("--drift-head", default=None, help="under the drift override: HEAD's short SHA")
|
|
2539
|
+
p_apply.add_argument("--drift-pinned", default=None, help="under the drift override: the pinned short SHA")
|
|
2540
|
+
p_apply.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the new map")
|
|
2541
|
+
p_apply.set_defaults(func=_cmd_apply)
|
|
2542
|
+
|
|
2543
|
+
p_records = sub.add_parser("records", help="write step 3's re-extraction records from the runner's and the "
|
|
2544
|
+
"workers' files")
|
|
2545
|
+
p_records.add_argument("--extraction", metavar="FILE", help="the recipe runner's output")
|
|
2546
|
+
p_records.add_argument("--export-details", metavar="FILE",
|
|
2547
|
+
help="step 2's export details: params and return types, and the exports read by eye")
|
|
2548
|
+
p_records.add_argument("--files-from", metavar="FILE",
|
|
2549
|
+
help="the files step 3 extracts (a JSON array), each a block even with no export")
|
|
2550
|
+
p_records.add_argument("--patches", metavar="DIR", help="the folder of the workers' per-file patches")
|
|
2551
|
+
p_records.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the records")
|
|
2552
|
+
p_records.set_defaults(func=_cmd_records)
|
|
2553
|
+
|
|
2554
|
+
p_gap = sub.add_parser("gap-records", help="spot-check, route and record a gap-driven repair's manifest entries")
|
|
2555
|
+
p_gap.add_argument("--manifest", required=True, metavar="FILE", help="skf-parse-gaps.py translate's manifest")
|
|
2556
|
+
p_gap.add_argument("--provenance-map", required=True, metavar="FILE", help="the map the repair starts from")
|
|
2557
|
+
p_gap.add_argument("--source-root", metavar="DIR", help="the source tree the spot-checks read")
|
|
2558
|
+
p_gap.add_argument("--drift-status", choices=DRIFT_STATUSES, default="ok",
|
|
2559
|
+
help="the workspace drift check's status; overridden runs the drift gate")
|
|
2560
|
+
p_gap.add_argument("--judgments", metavar="FILE", help="the answers to its needs_judgment[], keyed by gap id")
|
|
2561
|
+
p_gap.add_argument("--remediation-records", metavar="FILE",
|
|
2562
|
+
help="`records` output over the files targeted re-extraction scanned")
|
|
2563
|
+
p_gap.add_argument("--plan", action="store_true",
|
|
2564
|
+
help="print the answers it needs and the files targeted re-extraction scans; write no records")
|
|
2565
|
+
p_gap.add_argument("--files-out", metavar="FILE", help="with --plan: write those files here, a JSON array")
|
|
2566
|
+
p_gap.add_argument("--evidence", metavar="FILE",
|
|
2567
|
+
help="append the targeted_reextraction record to this JSON-lines file when it ran")
|
|
2568
|
+
p_gap.add_argument("--tier", help="with --evidence: the forge tier the record names")
|
|
2569
|
+
p_gap.add_argument("-o", "--output", metavar="FILE", help="where to write the records (without --plan)")
|
|
2570
|
+
p_gap.set_defaults(func=_cmd_gap_records)
|
|
2571
|
+
|
|
2572
|
+
for p in (p_build, p_ratio):
|
|
2573
|
+
p.add_argument(
|
|
2574
|
+
"--input", default=None, help="path to JSON input (default: stdin)"
|
|
2575
|
+
)
|
|
2576
|
+
p.add_argument(
|
|
2577
|
+
"--category-a", metavar="FILE",
|
|
2578
|
+
help="skf-classify-changed-files.py classify output: the Category A slice and its moves",
|
|
2579
|
+
)
|
|
2580
|
+
p.add_argument(
|
|
2581
|
+
"--category-b-diff", metavar="FILE",
|
|
2582
|
+
help="skf-structural-diff.py output over the modified files: the Category B slice",
|
|
2583
|
+
)
|
|
2584
|
+
for p in (p_build, p_ratio, p_apply):
|
|
2585
|
+
p.add_argument("--category-c", metavar="FILE", help="rename-candidates output: the Category C slice")
|
|
2586
|
+
p.add_argument("--ccc-pairs", metavar="FILE",
|
|
2587
|
+
help="the CCC check's pairs, {renamed_files, renamed_exports}, added to Category C")
|
|
2588
|
+
|
|
410
2589
|
return parser
|
|
411
2590
|
|
|
412
2591
|
|