bmad-module-skill-forge 2.2.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +49 -49
- package/docs/404.md +10 -8
- package/docs/_data/pinned.yaml +22 -13
- package/docs/_internal/STABILITY.md +58 -35
- package/docs/agents.md +39 -53
- package/docs/architecture.md +65 -51
- package/docs/bmad-synergy.md +53 -51
- package/docs/campaign.md +127 -75
- package/docs/concepts.md +34 -28
- package/docs/examples.md +70 -68
- package/docs/forge-auto.md +34 -31
- package/docs/getting-started.md +118 -57
- package/docs/how-it-works.md +25 -21
- package/docs/index.md +15 -15
- package/docs/skill-model.md +70 -59
- package/docs/troubleshooting.md +143 -18
- package/docs/verifying-a-skill.md +87 -57
- package/docs/why-skf.md +24 -22
- package/docs/workflows.md +274 -110
- package/package.json +18 -10
- package/src/forger/preferences.yaml +8 -1
- package/src/knowledge/agentskills-spec.md +2 -0
- package/src/knowledge/ccc-bridge.md +40 -24
- package/src/knowledge/confidence-tiers.md +9 -5
- package/src/knowledge/overview.md +3 -3
- package/src/knowledge/progressive-capability.md +8 -8
- package/src/knowledge/provenance-tracking.md +9 -5
- package/src/knowledge/qmd-registry.md +3 -3
- package/src/knowledge/skf-knowledge-index.csv +2 -2
- package/src/knowledge/skill-lifecycle.md +18 -11
- package/src/knowledge/split-body-strategy.md +0 -3
- package/src/knowledge/tool-resolution.md +4 -2
- package/src/knowledge/version-paths.md +101 -15
- package/src/module-help.csv +3 -3
- package/src/module.yaml +10 -0
- package/src/shared/data/ast-grep-recipes.yaml +608 -0
- package/src/shared/data/ide-context-files.json +30 -0
- package/src/shared/health-check.md +31 -59
- package/src/shared/references/description-guard-protocol.md +12 -9
- package/src/shared/references/feasibility-report-schema.md +19 -13
- package/src/shared/references/headless-gate-convention.md +55 -3
- package/src/shared/references/output-contract-schema.md +19 -8
- package/src/shared/references/pipeline-contracts.md +94 -29
- package/src/shared/references/tessl-review.md +77 -0
- package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
- package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
- package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
- package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
- package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
- package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
- package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
- package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
- package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
- package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
- package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
- package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
- package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +233 -64
- package/src/shared/scripts/skf-build-change-manifest.py +2753 -28
- package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
- package/src/shared/scripts/skf-check-unit-records.py +184 -0
- package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
- package/src/shared/scripts/skf-classify-changed-files.py +772 -0
- package/src/shared/scripts/skf-comention-pairs.py +1333 -123
- package/src/shared/scripts/skf-compare-file-hashes.py +267 -105
- package/src/shared/scripts/skf-count-imports.py +1867 -0
- package/src/shared/scripts/skf-count-tokens.py +1 -1
- package/src/shared/scripts/skf-derive-assembly-shape.py +21 -3
- package/src/shared/scripts/skf-description-guard.py +99 -11
- package/src/shared/scripts/skf-detect-docs.py +694 -2
- package/src/shared/scripts/skf-detect-language.py +486 -107
- package/src/shared/scripts/skf-detect-registry.py +887 -0
- package/src/shared/scripts/skf-detect-scripts-assets.py +197 -55
- package/src/shared/scripts/skf-detect-tools.py +209 -27
- package/src/shared/scripts/skf-detect-workspaces.py +506 -30
- package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
- package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
- package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
- package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
- package/src/shared/scripts/skf-extract-public-api.py +5247 -96
- package/src/shared/scripts/skf-extraction-inventory.py +1280 -0
- package/src/shared/scripts/skf-extraction-snapshot.py +664 -0
- package/src/shared/scripts/skf-fetch-temporal.py +674 -0
- package/src/shared/scripts/skf-find-cycles.py +97 -29
- package/src/shared/scripts/skf-find-test-report.py +476 -0
- package/src/shared/scripts/skf-forge-tier-rw.py +469 -205
- package/src/shared/scripts/skf-github-fetch.py +365 -0
- package/src/shared/scripts/skf-github-probe.py +700 -0
- package/src/shared/scripts/skf-hash-content.py +180 -20
- package/src/shared/scripts/skf-language-corpora.py +1 -1
- package/src/shared/scripts/skf-load-provenance.py +161 -27
- package/src/shared/scripts/skf-manifest-ops.py +1 -1
- package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
- package/src/shared/scripts/skf-merge-doc-urls.py +143 -20
- package/src/shared/scripts/skf-names-present.py +221 -0
- package/src/shared/scripts/skf-pair-intersect.py +119 -28
- package/src/shared/scripts/skf-parse-gaps.py +1384 -0
- package/src/shared/scripts/skf-preapply.py +1 -1
- package/src/shared/scripts/skf-preflight.py +176 -28
- package/src/shared/scripts/skf-promote-staged.py +430 -0
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +64 -8
- package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
- package/src/shared/scripts/skf-quick-batch.py +774 -0
- package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
- package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
- package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
- package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
- package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
- package/src/shared/scripts/skf-resolve-authoritative-files.py +155 -14
- package/src/shared/scripts/skf-resolve-package.py +523 -106
- package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
- package/src/shared/scripts/skf-run-lock.py +577 -0
- package/src/shared/scripts/skf-scan-manifests.py +998 -57
- package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
- package/src/shared/scripts/skf-severity-classify.py +712 -108
- package/src/shared/scripts/skf-shape-detect.py +298 -204
- package/src/shared/scripts/skf-shard-body.py +9 -8
- package/src/shared/scripts/skf-skill-inventory.py +2412 -89
- package/src/shared/scripts/skf-skills-module.py +472 -0
- package/src/shared/scripts/skf-source-tree.py +1796 -0
- package/src/shared/scripts/skf-structural-diff.py +787 -121
- package/src/shared/scripts/skf-tessl-review.py +706 -0
- package/src/shared/scripts/skf-update-active-symlink.py +120 -67
- package/src/shared/scripts/skf-update-run-state.py +624 -0
- package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
- package/src/shared/scripts/skf-validate-brief-schema.py +23 -2
- package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
- package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
- package/src/shared/scripts/skf-validate-output.py +712 -64
- package/src/shared/scripts/skf-validate-pins.py +1 -1
- package/src/shared/scripts/skf-verify-no-trace.py +116 -10
- package/src/shared/scripts/skf-verify-provenance-completeness.py +3945 -152
- package/src/shared/scripts/skf-write-skill-brief.py +406 -25
- package/src/shared/tool-requirements.yaml +201 -0
- package/src/skf-analyze-source/SKILL.md +41 -36
- package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
- package/src/skf-analyze-source/customize.toml +36 -24
- package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
- package/src/skf-analyze-source/references/continue.md +2 -13
- package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
- package/src/skf-analyze-source/references/generate-briefs.md +121 -55
- package/src/skf-analyze-source/references/headless-contract.md +42 -15
- package/src/skf-analyze-source/references/identify-units.md +96 -110
- package/src/skf-analyze-source/references/init.md +73 -71
- package/src/skf-analyze-source/references/map-and-detect.md +123 -105
- package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
- package/src/skf-analyze-source/references/recommend.md +51 -70
- package/src/skf-analyze-source/references/scan-project.md +60 -84
- package/src/skf-analyze-source/references/scan-root.md +39 -0
- package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
- package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
- package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
- package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
- package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
- package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
- package/src/skf-audit-skill/SKILL.md +43 -51
- package/src/skf-audit-skill/assets/drift-report-template.md +28 -3
- package/src/skf-audit-skill/customize.toml +41 -20
- package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
- package/src/skf-audit-skill/references/doc-drift.md +141 -0
- package/src/skf-audit-skill/references/headless-contract.md +29 -0
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +149 -210
- package/src/skf-audit-skill/references/re-index.md +103 -110
- package/src/skf-audit-skill/references/report.md +96 -33
- package/src/skf-audit-skill/references/semantic-diff.md +34 -25
- package/src/skf-audit-skill/references/severity-classify.md +79 -65
- package/src/skf-audit-skill/references/severity-rules.md +15 -6
- package/src/skf-audit-skill/references/structural-diff.md +80 -90
- package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
- package/src/skf-audit-skill/scripts/render-drift-tables.py +712 -0
- package/src/skf-brief-skill/SKILL.md +37 -26
- package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +18 -25
- package/src/skf-brief-skill/assets/skill-brief-schema.md +30 -23
- package/src/skf-brief-skill/customize.toml +48 -20
- package/src/skf-brief-skill/references/analyze-target.md +144 -127
- package/src/skf-brief-skill/references/confirm-brief.md +11 -9
- package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
- package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
- package/src/skf-brief-skill/references/gather-intent.md +87 -182
- package/src/skf-brief-skill/references/headless-args.md +84 -16
- package/src/skf-brief-skill/references/health-check.md +1 -1
- package/src/skf-brief-skill/references/invocation-contract.md +18 -13
- package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
- package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
- package/src/skf-brief-skill/references/scope-definition.md +123 -47
- package/src/skf-brief-skill/references/step-auto-brief.md +26 -100
- package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
- package/src/skf-brief-skill/references/version-resolution.md +7 -10
- package/src/skf-brief-skill/references/write-brief.md +66 -57
- package/src/skf-campaign/SKILL.md +31 -34
- package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
- package/src/skf-campaign/customize.toml +47 -16
- package/src/skf-campaign/references/campaign-contracts.md +39 -29
- package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
- package/src/skf-campaign/references/health-check.md +7 -16
- package/src/skf-campaign/references/step-01-setup.md +46 -84
- package/src/skf-campaign/references/step-02-strategy.md +31 -30
- package/src/skf-campaign/references/step-03-pins.md +26 -18
- package/src/skf-campaign/references/step-04-provenance.md +24 -23
- package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
- package/src/skf-campaign/references/step-06-batch.md +41 -34
- package/src/skf-campaign/references/step-07-capstone.md +40 -25
- package/src/skf-campaign/references/step-08-verify.md +28 -29
- package/src/skf-campaign/references/step-09-refine.md +29 -30
- package/src/skf-campaign/references/step-10-export.md +48 -36
- package/src/skf-campaign/references/step-11-maintenance.md +31 -34
- package/src/skf-campaign/references/step-resume.md +37 -68
- package/src/skf-campaign/scripts/campaign-deps.py +57 -18
- package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
- package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
- package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
- package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
- package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
- package/src/skf-campaign/scripts/campaign-report.py +204 -37
- package/src/skf-campaign/scripts/campaign-state.py +1128 -0
- package/src/skf-campaign/scripts/campaign-status.py +10 -74
- package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
- package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
- package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
- package/src/skf-campaign/templates/campaign-report-template.md +5 -1
- package/src/skf-campaign/templates/kickoff-template.md +0 -14
- package/src/skf-create-skill/SKILL.md +27 -20
- package/src/skf-create-skill/assets/compile-assembly-rules.md +11 -8
- package/src/skf-create-skill/assets/skill-sections.md +21 -11
- package/src/skf-create-skill/customize.toml +20 -8
- package/src/skf-create-skill/references/authoritative-files-protocol.md +36 -58
- package/src/skf-create-skill/references/batch-mode.md +81 -0
- package/src/skf-create-skill/references/ccc-index-check.md +27 -0
- package/src/skf-create-skill/references/compile.md +119 -70
- package/src/skf-create-skill/references/component-extraction.md +113 -91
- package/src/skf-create-skill/references/enrich.md +41 -23
- package/src/skf-create-skill/references/entry-points-by-hand.md +37 -0
- package/src/skf-create-skill/references/extract.md +204 -162
- package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +11 -11
- package/src/skf-create-skill/references/extraction-patterns.md +588 -201
- package/src/skf-create-skill/references/generate-artifacts.md +145 -93
- package/src/skf-create-skill/references/health-check.md +1 -1
- package/src/skf-create-skill/references/load-brief.md +53 -36
- package/src/skf-create-skill/references/relabel-rule.md +12 -0
- package/src/skf-create-skill/references/report.md +37 -57
- package/src/skf-create-skill/references/source-resolution-protocols.md +58 -169
- package/src/skf-create-skill/references/step-auto-shard.md +20 -93
- package/src/skf-create-skill/references/step-doc-rot.md +44 -60
- package/src/skf-create-skill/references/step-doc-sources.md +28 -19
- package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
- package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
- package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
- package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
- package/src/skf-create-skill/references/validate.md +173 -99
- package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
- package/src/skf-create-stack-skill/SKILL.md +29 -56
- package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
- package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
- package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
- package/src/skf-create-stack-skill/customize.toml +23 -12
- package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
- package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
- package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
- package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
- package/src/skf-create-stack-skill/references/generate-output.md +138 -128
- package/src/skf-create-stack-skill/references/init.md +90 -32
- package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
- package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
- package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
- package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
- package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
- package/src/skf-create-stack-skill/references/report.md +39 -33
- package/src/skf-create-stack-skill/references/validate.md +20 -39
- package/src/skf-drop-skill/SKILL.md +28 -51
- package/src/skf-drop-skill/customize.toml +44 -30
- package/src/skf-drop-skill/references/execute.md +83 -146
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +64 -0
- package/src/skf-drop-skill/references/report.md +46 -30
- package/src/skf-drop-skill/references/select.md +122 -124
- package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
- package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
- package/src/skf-export-skill/SKILL.md +40 -62
- package/src/skf-export-skill/assets/managed-section-format.md +14 -50
- package/src/skf-export-skill/assets/snippet-format.md +3 -1
- package/src/skf-export-skill/customize.toml +42 -13
- package/src/skf-export-skill/references/generate-snippet.md +54 -46
- package/src/skf-export-skill/references/health-check.md +2 -0
- package/src/skf-export-skill/references/invocation-contract.md +15 -0
- package/src/skf-export-skill/references/load-skill.md +88 -62
- package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
- package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
- package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
- package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
- package/src/skf-export-skill/references/package.md +15 -73
- package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
- package/src/skf-export-skill/references/result-envelope.md +31 -16
- package/src/skf-export-skill/references/summary.md +41 -17
- package/src/skf-export-skill/references/token-report.md +4 -12
- package/src/skf-export-skill/references/update-context.md +118 -237
- package/src/skf-forger/SKILL.md +61 -37
- package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
- package/src/skf-forger/customize.toml +43 -0
- package/src/skf-forger/references/pipeline-mode.md +73 -18
- package/src/skf-forger/scripts/forge-status.py +299 -0
- package/src/skf-forger/scripts/parse-pipeline.py +322 -54
- package/src/skf-forger/scripts/pipeline-gate.py +428 -0
- package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
- package/src/skf-quick-skill/SKILL.md +54 -53
- package/src/skf-quick-skill/assets/skill-template.md +4 -2
- package/src/skf-quick-skill/customize.toml +51 -20
- package/src/skf-quick-skill/references/batch-mode.md +74 -21
- package/src/skf-quick-skill/references/compile.md +30 -31
- package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
- package/src/skf-quick-skill/references/finalize.md +69 -18
- package/src/skf-quick-skill/references/halt-contract.md +41 -29
- package/src/skf-quick-skill/references/health-check.md +5 -3
- package/src/skf-quick-skill/references/quick-extract.md +114 -45
- package/src/skf-quick-skill/references/registry-resolution.md +9 -64
- package/src/skf-quick-skill/references/resolve-target.md +116 -66
- package/src/skf-quick-skill/references/write-and-validate.md +79 -40
- package/src/skf-refine-architecture/SKILL.md +43 -53
- package/src/skf-refine-architecture/customize.toml +49 -20
- package/src/skf-refine-architecture/references/compile.md +97 -54
- package/src/skf-refine-architecture/references/exit-codes.md +30 -0
- package/src/skf-refine-architecture/references/finding-storage.md +21 -0
- package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
- package/src/skf-refine-architecture/references/improvements.md +14 -20
- package/src/skf-refine-architecture/references/init.md +98 -57
- package/src/skf-refine-architecture/references/issue-detection.md +32 -39
- package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
- package/src/skf-refine-architecture/references/report.md +65 -54
- package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
- package/src/skf-rename-skill/SKILL.md +19 -54
- package/src/skf-rename-skill/customize.toml +32 -26
- package/src/skf-rename-skill/references/execute.md +230 -184
- package/src/skf-rename-skill/references/exit-codes.md +7 -7
- package/src/skf-rename-skill/references/health-check.md +1 -7
- package/src/skf-rename-skill/references/invocation-contract.md +25 -0
- package/src/skf-rename-skill/references/report.md +63 -52
- package/src/skf-rename-skill/references/select.md +197 -115
- package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
- package/src/skf-setup/SKILL.md +26 -26
- package/src/skf-setup/customize.toml +14 -10
- package/src/skf-setup/references/auto-index.md +50 -63
- package/src/skf-setup/references/ccc-index.md +29 -90
- package/src/skf-setup/references/detect-and-tier.md +58 -67
- package/src/skf-setup/references/health-check.md +2 -0
- package/src/skf-setup/references/invocation-contract.md +14 -0
- package/src/skf-setup/references/report.md +43 -166
- package/src/skf-setup/references/tier-rules.md +5 -5
- package/src/skf-setup/references/write-config.md +34 -57
- package/src/skf-test-skill/SKILL.md +17 -57
- package/src/skf-test-skill/assets/output-section-formats.md +33 -34
- package/src/skf-test-skill/customize.toml +27 -13
- package/src/skf-test-skill/references/coherence-check.md +134 -62
- package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
- package/src/skf-test-skill/references/coverage-check.md +209 -264
- package/src/skf-test-skill/references/detect-mode.md +5 -5
- package/src/skf-test-skill/references/external-validators.md +94 -82
- package/src/skf-test-skill/references/health-check.md +6 -7
- package/src/skf-test-skill/references/init.md +164 -91
- package/src/skf-test-skill/references/invocation-contract.md +66 -0
- package/src/skf-test-skill/references/migration-section-rules.md +5 -2
- package/src/skf-test-skill/references/report.md +164 -142
- package/src/skf-test-skill/references/score.md +80 -107
- package/src/skf-test-skill/references/scoring-rules.md +42 -155
- package/src/skf-test-skill/references/source-access-protocol.md +18 -94
- package/src/skf-test-skill/references/step-hard-gate.md +51 -33
- package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
- package/src/skf-test-skill/scripts/build-result-context.py +380 -0
- package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
- package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
- package/src/skf-test-skill/scripts/compute-score.py +223 -45
- package/src/skf-test-skill/scripts/gap-ledger.py +1666 -0
- package/src/skf-test-skill/scripts/hard-gate.py +203 -0
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +1198 -0
- package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
- package/src/skf-test-skill/scripts/reconcile-coverage.py +547 -79
- package/src/skf-test-skill/scripts/score-signatures.py +479 -0
- package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
- package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +133 -56
- package/src/skf-test-skill/templates/test-report-template.md +21 -15
- package/src/skf-update-skill/SKILL.md +38 -25
- package/src/skf-update-skill/customize.toml +27 -12
- package/src/skf-update-skill/references/detect-changes.md +253 -223
- package/src/skf-update-skill/references/gap-driven.md +336 -0
- package/src/skf-update-skill/references/health-check.md +31 -5
- package/src/skf-update-skill/references/init.md +254 -84
- package/src/skf-update-skill/references/invocation-contract.md +12 -0
- package/src/skf-update-skill/references/manual-section-rules.md +6 -23
- package/src/skf-update-skill/references/merge.md +182 -49
- package/src/skf-update-skill/references/re-extract.md +53 -262
- package/src/skf-update-skill/references/report.md +102 -35
- package/src/skf-update-skill/references/write.md +357 -155
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +183 -46
- package/src/skf-verify-stack/SKILL.md +23 -42
- package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
- package/src/skf-verify-stack/customize.toml +55 -22
- package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
- package/src/skf-verify-stack/references/coverage.md +67 -64
- package/src/skf-verify-stack/references/exit-codes.md +21 -8
- package/src/skf-verify-stack/references/init.md +107 -62
- package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
- package/src/skf-verify-stack/references/integrations.md +106 -65
- package/src/skf-verify-stack/references/report.md +59 -42
- package/src/skf-verify-stack/references/requirements.md +32 -9
- package/src/skf-verify-stack/references/synthesize.md +53 -39
- package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
- package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
- package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
- package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
- package/tools/cli/commands/install.js +11 -4
- package/tools/cli/commands/status.js +15 -3
- package/tools/cli/commands/update.js +11 -4
- package/tools/cli/lib/tool-check.js +320 -0
- package/tools/cli/lib/version-check.js +22 -9
- package/tools/cli/skf-cli.js +4 -1
- package/.gitattributes +0 -16
- package/.nvmrc +0 -1
- package/docs/_internal/RELEASING.md +0 -679
- package/release-audits/v1.0.0-launch-audit.md +0 -728
- package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
- package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
- package/src/skf-campaign/manifest.yaml +0 -15
- package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
- package/src/skf-create-skill/references/ecosystem-check.md +0 -74
- package/src/skf-drop-skill/references/headless-contract.md +0 -18
- package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
- package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
- package/src/skf-update-skill/references/validate.md +0 -124
- package/tools/validate-docs-drift.js +0 -233
- package/tools/validate-docs-links.js +0 -665
|
@@ -1,48 +1,26 @@
|
|
|
1
1
|
# /// script
|
|
2
|
-
# requires-python = ">=3.
|
|
2
|
+
# requires-python = ">=3.11"
|
|
3
3
|
# dependencies = ["pyyaml"]
|
|
4
4
|
# ///
|
|
5
|
-
"""SKF Merge CCC Exclusions —
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
references/ccc-index.md`
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
the
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
**/{skills_output_folder} Generated skill files
|
|
26
|
-
**/{forge_data_folder} Compilation workspace
|
|
27
|
-
|
|
28
|
-
Validation rules for the 2 conditional values (PR #248):
|
|
29
|
-
|
|
30
|
-
- REJECT empty or whitespace-only values
|
|
31
|
-
→ would produce `**/`, matching every path → ccc indexes nothing
|
|
32
|
-
- REJECT values starting with `/`, `~/`, or `./`
|
|
33
|
-
→ produces `**//abs/path`, `**/~/x`, or `**/./rel` — malformed glob
|
|
34
|
-
- REJECT values containing glob meta-characters (`*`, `?`, `[`)
|
|
35
|
-
→ interpolation collides with the surrounding pattern syntax
|
|
36
|
-
- REJECT values containing unresolved `{project-root}`-style placeholders
|
|
37
|
-
(any `{` or `}` character) — produces `**/{project-root}/x`, which
|
|
38
|
-
cocoindex accepts as a literal-segment glob and silently never matches.
|
|
39
|
-
This is the bug from issue #293: the step file was passing template
|
|
40
|
-
strings straight through.
|
|
41
|
-
|
|
42
|
-
Rejected values are SKIPPED (the pattern is not added) and a warning
|
|
43
|
-
is appended to the output. The 4 always-include patterns are applied
|
|
44
|
-
unconditionally — config-value rejection cannot disable indexing
|
|
45
|
-
entirely.
|
|
5
|
+
"""SKF Merge CCC Exclusions — prepare .cocoindex_code/settings.yml and keep
|
|
6
|
+
the SKF exclusion patterns in it current.
|
|
7
|
+
|
|
8
|
+
One invocation from setup step 1b (`src/skf-setup/references/ccc-index.md`)
|
|
9
|
+
owns every decision about the project's ccc settings file, so none of it
|
|
10
|
+
lives in step prose:
|
|
11
|
+
|
|
12
|
+
1. run `ccc init -f` when settings.yml is missing, and rebuild a file that
|
|
13
|
+
lacks the ccc default exclusions;
|
|
14
|
+
2. validate the two folder values from `_bmad/skf/config.yaml`;
|
|
15
|
+
3. exclude each folder whole, or entry by entry when it also holds
|
|
16
|
+
content SKF did not generate; leave out a folder with no SKF output,
|
|
17
|
+
and leave alone a folder the user already excluded;
|
|
18
|
+
4. merge the SKF patterns and prune the SKF-owned patterns the current
|
|
19
|
+
config no longer produces;
|
|
20
|
+
5. warn when `/.cocoindex_code/` is not gitignored;
|
|
21
|
+
6. return one `index_action` telling the step whether to run `ccc index`,
|
|
22
|
+
and with --build-index act on it: run `ccc index` itself, read the
|
|
23
|
+
file count from `ccc status` and stamp the time (Index build, below).
|
|
46
24
|
|
|
47
25
|
CLI — invoke via `uv run` so the PEP 723 PyYAML dependency declared
|
|
48
26
|
above is auto-resolved on first call and cached. `docs/getting-started.md`
|
|
@@ -53,43 +31,432 @@ on a fresh interpreter:
|
|
|
53
31
|
uv run skf-merge-ccc-exclusions.py \\
|
|
54
32
|
--project-root /abs/path \\
|
|
55
33
|
--skills-output-folder skills \\
|
|
56
|
-
--forge-data-folder _bmad-output/forge-data
|
|
34
|
+
--forge-data-folder _bmad-output/forge-data \\
|
|
35
|
+
--prior-state-from /abs/path/_bmad/_memory/forger-sidecar/forge-tier.yaml \\
|
|
36
|
+
--index-fresh false \\
|
|
37
|
+
--skip-index false
|
|
38
|
+
|
|
39
|
+
uv run skf-merge-ccc-exclusions.py \\
|
|
40
|
+
--project-root /abs/path \\
|
|
41
|
+
--config /abs/path/_bmad/skf/config.yaml \\
|
|
42
|
+
--prior-state-from /abs/path/_bmad/_memory/forger-sidecar/forge-tier.yaml \\
|
|
43
|
+
--index-fresh false --skip-index false \\
|
|
44
|
+
--build-index --result-to /abs/run/ccc-exclusions.json
|
|
45
|
+
|
|
46
|
+
Flags:
|
|
47
|
+
|
|
48
|
+
--project-root required; the directory holding .cocoindex_code/
|
|
49
|
+
--skills-output-folder raw skills_output_folder config value, forwarded
|
|
50
|
+
verbatim (`{project-root}/...` is resolved here)
|
|
51
|
+
--forge-data-folder raw forge_data_folder config value, same handling
|
|
52
|
+
--config instead of the two folder flags: the
|
|
53
|
+
`_bmad/skf/config.yaml` to read both raw values
|
|
54
|
+
from, so no caller forwards them by hand (a key
|
|
55
|
+
the file lacks reads as an empty value)
|
|
56
|
+
--prior-state-from forge-tier.yaml; its ccc_index.exclude_patterns is
|
|
57
|
+
the record of the patterns SKF owns
|
|
58
|
+
--index-fresh true|false (case-insensitive); the prior index is
|
|
59
|
+
still fresh, so an unchanged settings.yml keeps it
|
|
60
|
+
--skip-index true|false; the setup run opted out of indexing
|
|
61
|
+
--defer-index a due index build waits for a later run: setup
|
|
62
|
+
passes it when the run missed its --require-tier,
|
|
63
|
+
which ends tier_failure (Index decision, below)
|
|
64
|
+
--no-ccc-init never run ccc init (tests and diagnostics)
|
|
65
|
+
--build-index setup mode only: act on index_action and add the
|
|
66
|
+
`index` result (Index build, below)
|
|
67
|
+
--result-to also write the JSON result, or the JSON error of a
|
|
68
|
+
run that fails, to this file, so the caller's
|
|
69
|
+
next steps read it from there; with
|
|
70
|
+
--build-index it is written before `ccc index`
|
|
71
|
+
runs too (Index build, below)
|
|
72
|
+
--clone-root instead of --project-root: an SKF workspace clone
|
|
73
|
+
(clone mode, below)
|
|
74
|
+
--include-ext clone mode only, repeatable: a file extension whose
|
|
75
|
+
files ccc must index (`ex`, `.ex`)
|
|
76
|
+
|
|
77
|
+
Clone mode (create-skill step 3 and step 7) prepares the settings.yml of an
|
|
78
|
+
SKF workspace clone, a source repository rather than an SKF project, so the
|
|
79
|
+
list merges its step prose used to make by hand live here too:
|
|
80
|
+
|
|
81
|
+
uv run skf-merge-ccc-exclusions.py --clone-root /abs/clone \\
|
|
82
|
+
[--include-ext ex --include-ext exs]
|
|
83
|
+
|
|
84
|
+
1. run `ccc init -f` when settings.yml is missing, so the clone gets a
|
|
85
|
+
project of its own instead of an enclosing one's;
|
|
86
|
+
2. quote each bare `- *...` list item an earlier hand edit left, which
|
|
87
|
+
YAML reads as an alias, so ccc can load the file again;
|
|
88
|
+
3. append each standard build and dependency exclusion (CLONE_EXCLUDES)
|
|
89
|
+
that exclude_patterns lacks, keeping every entry already there;
|
|
90
|
+
4. with --include-ext, append `**/*.<ext>` to include_patterns for each
|
|
91
|
+
extension no include_patterns entry matches, reading each entry as
|
|
92
|
+
ccc's glob library does.
|
|
93
|
+
|
|
94
|
+
A file with no exclude_patterns list, or no include_patterns list, gets
|
|
95
|
+
nothing added to that list: a list written from scratch replaces the ccc
|
|
96
|
+
defaults. Clone mode never removes an entry, never rebuilds the file, and
|
|
97
|
+
takes none of the folder, record or index flags of setup mode. Its writes
|
|
98
|
+
go through a temporary file of their own, so two runs merging the same
|
|
99
|
+
clone never tear the file; they add the same entries.
|
|
100
|
+
|
|
101
|
+
Settings states (target = {project-root}/.cocoindex_code/settings.yml):
|
|
102
|
+
|
|
103
|
+
missing `ccc init -f` creates it (ccc_init "created"). With
|
|
104
|
+
--no-ccc-init, or when ccc init leaves no file (ccc_init
|
|
105
|
+
"failed"), settings_ready is false and index_action is "fail":
|
|
106
|
+
a later `ccc index` would auto-initialise at the enclosing git
|
|
107
|
+
root and index it without the SKF exclusions. When the created
|
|
108
|
+
file has no exclude_patterns list, SKF adds nothing to it.
|
|
109
|
+
rebuild the file is empty, has no `exclude_patterns` key, or has
|
|
110
|
+
neither an `include_patterns` key nor `**/.cocoindex_code` in
|
|
111
|
+
its exclusions — every file ccc writes has both, so such a file
|
|
112
|
+
suppresses the ccc defaults (hidden folders, node_modules, the
|
|
113
|
+
ccc database). The original is moved aside to
|
|
114
|
+
`settings.yml.skf-repair`, `ccc init -f` writes a fresh file,
|
|
115
|
+
and the result is the fresh file plus every original top-level
|
|
116
|
+
key plus every original exclude entry not already there
|
|
117
|
+
(ccc_init "rebuilt"). The backup is removed only after the
|
|
118
|
+
write succeeds; a leftover backup found on startup is restored
|
|
119
|
+
and the repair runs again. If ccc init produces no usable file,
|
|
120
|
+
the original bytes are restored, ccc_init is "failed", and no
|
|
121
|
+
SKF change is applied that run; a fresh prior index is kept, but
|
|
122
|
+
no new index is built from that file (index_action "keep" or
|
|
123
|
+
"fail"). With --no-ccc-init the rebuild is skipped with a
|
|
124
|
+
warning and the file is left alone.
|
|
125
|
+
ready any other file is reconciled in place (ccc_init "not_needed").
|
|
126
|
+
|
|
127
|
+
The script never writes settings.yml from scratch. `ccc init -f` is the
|
|
128
|
+
only creator: `-f` merely skips ccc's parent-marker check, so one call
|
|
129
|
+
covers the git-root, nested-folder and repair cases, and on an existing
|
|
130
|
+
file ccc prints "Project already initialized." and changes nothing. At the
|
|
131
|
+
top of a git checkout ccc also adds `/.cocoindex_code/` to `.gitignore`;
|
|
132
|
+
SKF never writes `.gitignore` itself.
|
|
133
|
+
|
|
134
|
+
Patterns SKF owns:
|
|
57
135
|
|
|
58
|
-
|
|
136
|
+
**/_bmad SKF framework module (workflows, agents, knowledge)
|
|
137
|
+
**/_bmad-output Build output artifacts
|
|
138
|
+
**/.claude Claude Code configuration
|
|
139
|
+
**/_skf-learn SKF learning materials
|
|
140
|
+
{skills_output_folder} Generated skill files (default `skills`)
|
|
141
|
+
{forge_data_folder} Compilation workspace
|
|
142
|
+
|
|
143
|
+
The four `**/` patterns are always produced. The two folder patterns are
|
|
144
|
+
the project-relative value itself, anchored to the project root: ccc reads
|
|
145
|
+
`skills` as that one root folder, while `**/skills` would also drop every
|
|
146
|
+
nested folder with the same name.
|
|
147
|
+
|
|
148
|
+
A folder that also holds content SKF did not generate gets per-entry
|
|
149
|
+
patterns instead of the bare value, each anchored the same way:
|
|
150
|
+
|
|
151
|
+
{folder}/<entry> each SKF-owned first-level directory or
|
|
152
|
+
root file, named as it is on disk
|
|
153
|
+
{folder}/<fixed entry> always, present or not: `_batch` and
|
|
154
|
+
`.export-manifest.json` (skills),
|
|
155
|
+
`_campaign` and `improvement-queue` (forge)
|
|
156
|
+
{folder}/<stem>-result*.json always for the `export-skill` and
|
|
157
|
+
`drop-skill` stems (skills), and for the
|
|
158
|
+
stem of any other SKF result at the root
|
|
159
|
+
{folder}/<prefix>* forge only, one per SKF report prefix
|
|
160
|
+
|
|
161
|
+
Glob characters in an entry name are written as one-character classes
|
|
162
|
+
(`[x]` becomes `[[]x[]]`); a backslash escape is never used, because ccc's
|
|
163
|
+
matcher reads a backslash as `/` on Windows. A name holding `'`, `\\`, a
|
|
164
|
+
control character, U+FFFD or an undecodable byte cannot go through a
|
|
165
|
+
quoted shell payload, so that entry stays indexed with a warning. No
|
|
166
|
+
pattern is ever `{folder}/*` or a `!` negation: ccc applies a negation
|
|
167
|
+
against every pattern, its own defaults included, and a `!{folder}/...`
|
|
168
|
+
also cancels a bare `{folder}` pattern.
|
|
169
|
+
|
|
170
|
+
A `!` entry of the user's cancels an SKF pattern the same way. ccc walks
|
|
171
|
+
into an excluded folder when a `!` entry matches the folder, matches the
|
|
172
|
+
one child name ccc probes it with (in practice a wildcard child: `!*/*`
|
|
173
|
+
and `!**/skills/*` re-open `skills`, `!*/x` does not), or starts with
|
|
174
|
+
`{folder}/`, and then indexes everything below it that no other pattern
|
|
175
|
+
excludes. So once the patterns are merged, each folder with the bare
|
|
176
|
+
value or per-entry patterns gets one warning naming the entries that
|
|
177
|
+
re-open one of its patterns, when ccc, reading the exclude_patterns this
|
|
178
|
+
run writes, walks down to a file of SKF output git lists below that
|
|
179
|
+
pattern (or, with none there yet, into the child it probes). Result and
|
|
180
|
+
report families are not checked. Inside a hidden folder ccc's own `**/.*`
|
|
181
|
+
still excludes that output, so an entry there warns only when it
|
|
182
|
+
re-opens every level down to it, and the bare value's warning says all
|
|
183
|
+
SKF output is back only when every SKF entry there is reached. A folder
|
|
184
|
+
inside an always-excluded one never switches to per-entry patterns, so
|
|
185
|
+
its warning offers no folder of the user's own there. A backslash in an
|
|
186
|
+
entry is read as ccc's glob library reads it on the machine the helper
|
|
187
|
+
runs on: `/` on Windows, an escape elsewhere. The warning repeats on
|
|
188
|
+
every run while the entry stays, including after a folder that held
|
|
189
|
+
other content goes back to the bare value; SKF never removes the entry.
|
|
190
|
+
|
|
191
|
+
Folder values are normalized first: surrounding whitespace is stripped,
|
|
192
|
+
`\\` becomes `/`, a leading `{project-root}/` is dropped, and `./`, `//`
|
|
193
|
+
and a trailing `/` collapse. A normalized value is then refused, with a
|
|
194
|
+
warning naming the key and the file to fix, when it is:
|
|
195
|
+
|
|
196
|
+
- empty or `.` (would exclude the whole repository)
|
|
197
|
+
- absolute or anchored: a leading `/`, `~` or `./`, a drive letter, or
|
|
198
|
+
a trailing `/` (absolute paths are never rewritten)
|
|
199
|
+
- carrying a `..` segment (outside the project)
|
|
200
|
+
- starting with `!` (ccc reads it as a negation)
|
|
201
|
+
- carrying `*`, `?`, `[`, `]` or `\\` (ccc reads them as glob syntax)
|
|
202
|
+
- carrying `{` or `}` (an unresolved template placeholder)
|
|
203
|
+
- carrying a control character (does not survive a quoted payload)
|
|
204
|
+
- carrying `'` (breaks a single-quoted payload)
|
|
205
|
+
|
|
206
|
+
A refused value is left out; the four `**/` patterns still merge.
|
|
207
|
+
|
|
208
|
+
Folder check: for each accepted folder that exists, the script lists it
|
|
209
|
+
with `git --literal-pathspecs ls-files -z --cached --others
|
|
210
|
+
--exclude-per-directory=.gitignore -- <folder>`: tracked files plus
|
|
211
|
+
untracked files no `.gitignore` excludes, the only ignore files ccc reads
|
|
212
|
+
(`.git/info/exclude` and a global excludes file hide nothing from ccc, so
|
|
213
|
+
they hide nothing here either). A tracked file deleted from the worktree
|
|
214
|
+
is dropped. The paths are grouped by first-level entry. A group is SKF
|
|
215
|
+
output by the same rule the workflows use before they write into, move or
|
|
216
|
+
delete a skill's folders (skf-skill-inventory.py):
|
|
217
|
+
|
|
218
|
+
skills it is `_batch` or its name contains `.skf-`, or its
|
|
219
|
+
`metadata.json` or a `<v>/<n>/metadata.json` carries an SKF
|
|
220
|
+
marker (generated_by, tool_versions.skf, or skill_type with
|
|
221
|
+
forge_tier / confidence_tier); a linked group never is. The
|
|
222
|
+
versioned layout, an `active` pointer, a manifest key or a
|
|
223
|
+
`*-result*.json` alone do not count, so a module's skills that an
|
|
224
|
+
earlier SKF moved into the versioned layout stay indexed
|
|
225
|
+
forge its name contains `.skf-`, it is `_campaign` or
|
|
226
|
+
`improvement-queue`, or it holds a `skill-brief.yaml*`,
|
|
227
|
+
`.brief-draft.json` or `*-result*.json` file directly, or a
|
|
228
|
+
provenance map, evidence report, extraction rules or
|
|
229
|
+
`*-result*.json` file directly in a folder of it; a linked group,
|
|
230
|
+
folder or file never counts. Like the skills rule it reads the
|
|
231
|
+
group on disk, so a file `.gitignore` hides still counts
|
|
232
|
+
|
|
233
|
+
Root files named `*-result*.json`, `.export-manifest.json` (skills) or
|
|
234
|
+
with an SKF report prefix (forge) are SKF output. Neutral entries never
|
|
235
|
+
count: `.gitkeep`, `.gitignore`, `.gitattributes`, `.DS_Store`,
|
|
236
|
+
`Thumbs.db`, `desktop.ini` and README files, plus any other dot-named
|
|
237
|
+
entry when `**/.*` (the ccc default) is in exclude_patterns and no entry
|
|
238
|
+
there starts with `!`. Everything else is foreign. A group that is, or
|
|
239
|
+
contains, the other configured SKF folder counts as SKF output but gets no
|
|
240
|
+
pattern of its own: that folder is reconciled separately, deepest first.
|
|
241
|
+
When both settings name the same folder, an entry either kind accepts is
|
|
242
|
+
SKF output. A submodule or nested repository is classified too: when the
|
|
243
|
+
folder itself is one, its own files are listed from inside it, and one
|
|
244
|
+
deeper down is a group of its own.
|
|
245
|
+
|
|
246
|
+
Each folder gets one verdict:
|
|
247
|
+
|
|
248
|
+
user the folder value is already in exclude_patterns and SKF does
|
|
249
|
+
not own it (see the record below): no pattern, no warning
|
|
250
|
+
bare no foreign group, and either SKF output or no other root file
|
|
251
|
+
(loose notes beside SKF output are tolerated): the bare value
|
|
252
|
+
entries foreign entries next to SKF output, or a folder nested inside
|
|
253
|
+
it that is itself `entries` or `left_out`: the per-entry
|
|
254
|
+
patterns, with a note naming the foreign entries when there
|
|
255
|
+
are any
|
|
256
|
+
left_out foreign entries and no SKF output: nothing, with a warning
|
|
257
|
+
naming the setting to change
|
|
258
|
+
|
|
259
|
+
The check is skipped, giving the bare value, silently when git is missing,
|
|
260
|
+
the project is not a git repository, the folder does not exist, or an
|
|
261
|
+
always-included `**/` pattern already covers the folder; it is skipped
|
|
262
|
+
with a warning when git times out or fails in any other way (for example
|
|
263
|
+
a repository git refuses as unsafe). Every git and ccc child runs with
|
|
264
|
+
the git location variables (GIT_DIR, GIT_INDEX_FILE, ...) removed from
|
|
265
|
+
its environment, so a git hook's index never leaks into the check; git
|
|
266
|
+
also runs with LC_ALL=C so its messages are recognised in any locale.
|
|
267
|
+
|
|
268
|
+
Ownership record and pruning: `ccc_index.exclude_patterns` in the
|
|
269
|
+
forge-tier.yaml passed through --prior-state-from is the set of patterns
|
|
270
|
+
SKF owned after its last reconcile, per-entry patterns included. A
|
|
271
|
+
recorded pattern the current run does not produce is removed from
|
|
272
|
+
settings.yml, so a dropped or renamed skill loses its pattern and a folder
|
|
273
|
+
that changes verdict swaps its bare value for per-entry patterns or back;
|
|
274
|
+
entries SKF never recorded (user entries and the ccc defaults) are never
|
|
275
|
+
removed, and the four `**/` patterns are always produced so are never
|
|
276
|
+
pruned. A record is legacy when forge-tier.yaml exists, the record names
|
|
277
|
+
no folder pattern (it holds at most the four `**/` patterns), and it is
|
|
278
|
+
empty or settings.yml still holds a `**/{value}` form of an accepted
|
|
279
|
+
folder value; those `**/{value}` forms then count as owned, so installs
|
|
280
|
+
migrate to the anchored forms. A `**/skills` added by the user after a
|
|
281
|
+
folder pattern is recorded survives, and with no forge-tier.yaml at all
|
|
282
|
+
SKF owns nothing yet. A record is trusted when forge-tier.yaml exists and
|
|
283
|
+
the record is not legacy: with a trusted record, a folder value already in
|
|
284
|
+
exclude_patterns that the record does not hold is the user's (verdict
|
|
285
|
+
`user`), unless the record holds `{value}/...` per-entry patterns and none
|
|
286
|
+
of them is still in exclude_patterns. SKF then swapped them for the bare
|
|
287
|
+
value in a run that stopped before its record was saved, so the value is
|
|
288
|
+
SKF's and is classified again. Without a trusted record the folder is
|
|
289
|
+
still classified: an all-SKF folder adopts the bare value already there,
|
|
290
|
+
and a folder with foreign entries keeps it untouched with a one-time
|
|
291
|
+
warning (the next run has a trusted record). When a folder value is refused, pruning is blocked
|
|
292
|
+
for the whole run and the recorded patterns that would have been removed
|
|
293
|
+
stay (warning): a config typo must not silently un-exclude real output. If
|
|
294
|
+
that run also used the legacy fallback, effective_patterns is null so the
|
|
295
|
+
record stays folder-free and the next valid run can still migrate.
|
|
296
|
+
|
|
297
|
+
Index decision: "fail" when no settings.yml exists; otherwise "skip"
|
|
298
|
+
under --skip-index true (with a warning when settings.yml changed);
|
|
299
|
+
otherwise, after a failed rebuild, "keep" when the prior index is fresh
|
|
300
|
+
and "fail" when it is not; otherwise "index" when settings.yml changed or
|
|
301
|
+
the prior index is not fresh, or "defer" in its place under --defer-index;
|
|
302
|
+
otherwise "keep". --defer-index changes only a due build: a fresh index
|
|
303
|
+
is still kept, its record carried forward, which --skip-index would null.
|
|
304
|
+
|
|
305
|
+
After a reconcile, `/.cocoindex_code/` coverage is checked with
|
|
306
|
+
`git check-ignore -q --no-index -- .cocoindex_code/target_sqlite.db`; an
|
|
307
|
+
uncovered path adds a warning. `gitignore_updated` compares the
|
|
308
|
+
`.gitignore` bytes before and after the run (only ccc init writes it).
|
|
309
|
+
|
|
310
|
+
Index build (--build-index): the script acts on index_action itself and
|
|
311
|
+
returns the index state the run ends with, so no step reads `ccc status`
|
|
312
|
+
or types a timestamp:
|
|
313
|
+
|
|
314
|
+
keep status "fresh"; last_indexed and file_count carry over from the
|
|
315
|
+
prior forge-tier.yaml (--prior-state-from)
|
|
316
|
+
skip status "skipped"
|
|
317
|
+
defer status "skipped", as for skip: the `ccc index` run that "index"
|
|
318
|
+
would make waits for a later run, so the next setup run builds it
|
|
319
|
+
fail status "failed", with not_ready_reason as failed_reason
|
|
320
|
+
index `ccc index` runs in the project root (up to CCC_INDEX_TIMEOUT_SEC).
|
|
321
|
+
Then `ccc status` gives the file count. While it prints an
|
|
322
|
+
`Indexing in progress:` line, `ccc index` runs again (it waits
|
|
323
|
+
for the running pass, then makes a quick incremental one), up to
|
|
324
|
+
CCC_INDEX_RERUNS times; a pass still running after that adds a
|
|
325
|
+
warning and the count read last. Status "created", stamped with
|
|
326
|
+
this script's UTC clock, when ccc indexed at least one file;
|
|
327
|
+
status "failed", with the reason, when `ccc index` or `ccc status`
|
|
328
|
+
fails, the status shows no file count, or the count is 0.
|
|
329
|
+
Before `ccc index` starts, --result-to already holds the merge
|
|
330
|
+
result with the index "failed" ("ccc index did not finish"),
|
|
331
|
+
so a call its host stops during a long index still leaves this
|
|
332
|
+
run's exclusion record for the next steps; the final result
|
|
333
|
+
replaces it.
|
|
334
|
+
|
|
335
|
+
indexed_path is the --project-root value exactly as given, which is what
|
|
336
|
+
skf-detect-tools.py compares the next run's --project-root with.
|
|
337
|
+
|
|
338
|
+
Output (single JSON document on stdout, ASCII only):
|
|
59
339
|
|
|
60
340
|
{
|
|
61
341
|
"status": "ok",
|
|
62
|
-
"version": "
|
|
63
|
-
"
|
|
64
|
-
"
|
|
65
|
-
"
|
|
66
|
-
|
|
67
|
-
"
|
|
68
|
-
|
|
69
|
-
"
|
|
342
|
+
"version": "v2",
|
|
343
|
+
"settings_yml_path": "/abs/path/.cocoindex_code/settings.yml",
|
|
344
|
+
"settings_yml_existed": bool, after crash recovery, before init
|
|
345
|
+
"ccc_init": "not_needed" | "created" | "rebuilt"
|
|
346
|
+
| "failed",
|
|
347
|
+
"settings_ready": bool, false only when no usable
|
|
348
|
+
settings.yml exists
|
|
349
|
+
"not_ready_reason": str | null, why index_action is
|
|
350
|
+
"fail" (null otherwise)
|
|
351
|
+
"patterns_added": int,
|
|
352
|
+
"patterns_added_list": [str], produced patterns not in the
|
|
353
|
+
pre-run list
|
|
354
|
+
"patterns_removed": int,
|
|
355
|
+
"patterns_removed_list": [str], SKF-owned patterns pruned
|
|
356
|
+
"patterns_already_present": int,
|
|
357
|
+
"effective_patterns": [str] | null, sorted SKF-owned set to
|
|
358
|
+
record, per-entry patterns
|
|
359
|
+
included (copy every entry
|
|
360
|
+
exactly); null when nothing was
|
|
361
|
+
reconciled, or a legacy-record run
|
|
362
|
+
with a refused value (keep the
|
|
363
|
+
old record)
|
|
364
|
+
"written": bool, settings.yml changed this run
|
|
365
|
+
(ccc init create, rebuild, or
|
|
366
|
+
SKF edit)
|
|
367
|
+
"gitignore_updated": bool,
|
|
368
|
+
"index_action": "index" | "keep" | "skip" | "defer"
|
|
369
|
+
| "fail",
|
|
370
|
+
"index": only with --build-index:
|
|
371
|
+
{"status": "fresh" | "created" | "skipped"
|
|
372
|
+
| "failed",
|
|
373
|
+
"indexed_path": str | null,
|
|
374
|
+
"last_indexed": str | null, ISO-8601 UTC
|
|
375
|
+
"file_count": int | null,
|
|
376
|
+
"failed_reason": str | null},
|
|
377
|
+
"warnings": [str] refused values and other notes for
|
|
378
|
+
the report, a user `!` entry that
|
|
379
|
+
cancels an SKF pattern included
|
|
70
380
|
}
|
|
71
381
|
|
|
72
|
-
|
|
73
|
-
re-run — no I/O wasted). Non-zero `warnings` does NOT prevent the
|
|
74
|
-
write — always-include patterns still merge.
|
|
382
|
+
Clone mode output (single JSON document on stdout, ASCII only):
|
|
75
383
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
384
|
+
{
|
|
385
|
+
"status": "ok",
|
|
386
|
+
"version": "v2",
|
|
387
|
+
"settings_yml_path": "/abs/clone/.cocoindex_code/settings.yml",
|
|
388
|
+
"settings_yml_existed": bool,
|
|
389
|
+
"ccc_init": "not_needed" | "created" | "failed",
|
|
390
|
+
"settings_ready": bool, false only when no settings.yml
|
|
391
|
+
exists after the ccc init
|
|
392
|
+
"not_ready_reason": str | null,
|
|
393
|
+
"patterns_added": int,
|
|
394
|
+
"patterns_added_list": [str], standard exclusions appended
|
|
395
|
+
"patterns_already_present": int,
|
|
396
|
+
"includes_added_list": [str], `**/*.<ext>` entries appended
|
|
397
|
+
"includes_covered_list": [str], extensions an include_patterns
|
|
398
|
+
entry already matched
|
|
399
|
+
"written": bool, settings.yml changed this run
|
|
400
|
+
"warnings": [str]
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
Payload safety: setup's steps once embedded these strings in
|
|
404
|
+
single-quoted `echo '...'` payloads, where dash's `echo` rewrites
|
|
405
|
+
backslash escapes; they now read the --result-to file, and the strings
|
|
406
|
+
stay safe for any caller that still quotes them. So every warning,
|
|
407
|
+
`not_ready_reason` and error message has `'` replaced by a
|
|
408
|
+
backtick, `\\` by `/`, and control characters and lone surrogates by `?`;
|
|
409
|
+
folder values carrying `'`, `\\` or a control character are refused, and
|
|
410
|
+
an entry name carrying one gets no pattern. No pattern SKF produces, and
|
|
411
|
+
so no effective_patterns entry, holds any of them. Data fields such as
|
|
412
|
+
`settings_yml_path` are emitted as they are.
|
|
413
|
+
|
|
414
|
+
Paths in messages: when a warning or `not_ready_reason` names the
|
|
415
|
+
project root in SKF's own words, it writes the literal `{project-root}`
|
|
416
|
+
placeholder, never the absolute path. Setup's step 4 banner renders a
|
|
417
|
+
warning's placeholder like its own `{project-root}` paths, and the
|
|
418
|
+
envelope keeps it verbatim. The unresolved-placeholder refusal names the
|
|
419
|
+
placeholder this script resolves in words ("the project-root
|
|
420
|
+
placeholder"), so a `{project-root}` in SKF's own words always means the
|
|
421
|
+
project root. Output and errors quoted from `ccc init` or git can carry
|
|
422
|
+
paths of their own, the project root included. Error messages on stderr
|
|
423
|
+
name absolute paths.
|
|
424
|
+
|
|
425
|
+
Writes to settings.yml use temp + fsync + rename (mirrors
|
|
426
|
+
skf-atomic-write.py), emit ASCII-only YAML (non-ASCII escaped, as ccc
|
|
427
|
+
writes it — ccc reads the file with the platform default encoding), and
|
|
428
|
+
drop YAML comments; ccc ignores them and keeps unknown top-level keys,
|
|
429
|
+
which are preserved. Concurrent writers must
|
|
430
|
+
coordinate via external `flock` (the typical pattern for shared-file
|
|
431
|
+
mutation in this module).
|
|
79
432
|
|
|
80
433
|
Exit codes:
|
|
81
|
-
0 success
|
|
82
|
-
1
|
|
83
|
-
|
|
434
|
+
0 success, including "not ready" (warnings stay on the success path)
|
|
435
|
+
1 --project-root (or --clone-root) is not a directory; settings.yml
|
|
436
|
+
cannot be parsed, is not a mapping, or has an exclude_patterns (clone
|
|
437
|
+
mode: or include_patterns) that is neither a list nor null (JSON error
|
|
438
|
+
on stderr)
|
|
439
|
+
2 usage error (including a non-boolean --index-fresh / --skip-index, an
|
|
440
|
+
--include-ext that is not a plain extension, and a setup flag or
|
|
441
|
+
--include-ext given in the other mode), a write, move-aside or restore
|
|
442
|
+
failure, or any unexpected internal error (always JSON on stderr, never
|
|
443
|
+
a traceback)
|
|
84
444
|
"""
|
|
85
445
|
|
|
86
446
|
from __future__ import annotations
|
|
87
447
|
|
|
88
448
|
import argparse
|
|
449
|
+
import functools
|
|
89
450
|
import json
|
|
90
451
|
import os
|
|
452
|
+
import re
|
|
453
|
+
import shutil
|
|
454
|
+
import subprocess
|
|
91
455
|
import sys
|
|
92
|
-
|
|
456
|
+
import unicodedata
|
|
457
|
+
from datetime import datetime, timezone
|
|
458
|
+
from pathlib import Path, PurePosixPath
|
|
459
|
+
from typing import NamedTuple
|
|
93
460
|
|
|
94
461
|
import yaml
|
|
95
462
|
|
|
@@ -100,29 +467,178 @@ ALWAYS_INCLUDE = (
|
|
|
100
467
|
"**/.claude",
|
|
101
468
|
"**/_skf-learn",
|
|
102
469
|
)
|
|
103
|
-
GLOB_META_CHARS = set("*?[")
|
|
470
|
+
GLOB_META_CHARS = set("*?[]\\")
|
|
104
471
|
PLACEHOLDER_CHARS = set("{}")
|
|
472
|
+
CCC_SELF_EXCLUDE = "**/.cocoindex_code"
|
|
473
|
+
CCC_INIT_TIMEOUT_SEC = 75
|
|
474
|
+
# A first index of a large codebase can take many minutes.
|
|
475
|
+
CCC_INDEX_TIMEOUT_SEC = 3600
|
|
476
|
+
CCC_STATUS_TIMEOUT_SEC = 75
|
|
477
|
+
CCC_INDEX_RERUNS = 3
|
|
478
|
+
# The index failed_reason --result-to holds while `ccc index` runs.
|
|
479
|
+
INDEX_UNFINISHED = "ccc index did not finish"
|
|
480
|
+
CCC_IN_PROGRESS = "Indexing in progress:"
|
|
481
|
+
CCC_FILES_RE = re.compile(r"^\s*Files:\s*(\d+)\s*$", re.MULTILINE)
|
|
482
|
+
GIT_TIMEOUT_SEC = 10
|
|
483
|
+
SAMPLE_SIZE = 3
|
|
484
|
+
INDEX_ACTIONS = ("index", "keep", "skip", "defer", "fail")
|
|
485
|
+
# Keep identical to GIT_LOCATION_VARS in skf-check-workspace-drift.py.
|
|
486
|
+
GIT_LOCATION_VARS = (
|
|
487
|
+
"GIT_DIR",
|
|
488
|
+
"GIT_WORK_TREE",
|
|
489
|
+
"GIT_INDEX_FILE",
|
|
490
|
+
"GIT_PREFIX",
|
|
491
|
+
"GIT_COMMON_DIR",
|
|
492
|
+
"GIT_OBJECT_DIRECTORY",
|
|
493
|
+
"GIT_ALTERNATE_OBJECT_DIRECTORIES",
|
|
494
|
+
"GIT_NAMESPACE",
|
|
495
|
+
)
|
|
496
|
+
# Keep identical to RESULT_JSON_RE in skf-skill-inventory.py
|
|
497
|
+
# (test/test-skf-skill-inventory.py pins the copies).
|
|
498
|
+
RESULT_JSON_RE = re.compile(r"^([a-z0-9][a-z0-9-]*)-result(-latest|-\d[^/]*)?\.json$")
|
|
499
|
+
README_RE = re.compile(r"^readme(\..+)?$", re.IGNORECASE)
|
|
500
|
+
NEUTRAL_ROOT_FILES = frozenset({".gitkeep", ".gitignore", ".gitattributes", ".DS_Store",
|
|
501
|
+
"Thumbs.db", "desktop.ini"})
|
|
502
|
+
SKF_FIXED_ENTRIES = {
|
|
503
|
+
"skills": ("_batch", ".export-manifest.json"),
|
|
504
|
+
"forge": ("_campaign", "improvement-queue"),
|
|
505
|
+
}
|
|
506
|
+
SKF_ROOT_RESULT_STEMS = {"skills": ("export-skill", "drop-skill"), "forge": ()}
|
|
507
|
+
# Keep identical to SKF_GENERATORS in skf-skill-inventory.py
|
|
508
|
+
# (test/test-skf-skill-inventory.py pins the copies).
|
|
509
|
+
SKF_GENERATORS = frozenset({"quick-skill", "create-skill", "create-stack-skill"})
|
|
510
|
+
FORGE_ROOT_PREFIXES = (
|
|
511
|
+
"analyze-source-",
|
|
512
|
+
"feasibility-report-",
|
|
513
|
+
"verify-stack-result-",
|
|
514
|
+
"ra-state-",
|
|
515
|
+
"refine-architecture-result-",
|
|
516
|
+
"refined-architecture-",
|
|
517
|
+
)
|
|
518
|
+
# Keep identical to FORGE_GROUP_DIRS in skf-skill-inventory.py
|
|
519
|
+
# (test/test-skf-skill-inventory.py pins the copies).
|
|
520
|
+
FORGE_GROUP_DIRS = frozenset({"_campaign", "improvement-queue"})
|
|
521
|
+
# Keep identical to FORGE_VERSION_ANCHORS in skf-skill-inventory.py
|
|
522
|
+
# (test/test-skf-skill-inventory.py pins the copies).
|
|
523
|
+
FORGE_VERSION_ANCHORS = frozenset({"provenance-map.json", "evidence-report.md", "extraction-rules.yaml"})
|
|
524
|
+
FOLDER_KEYS = (("skills_output_folder", "skills"), ("forge_data_folder", "forge"))
|
|
525
|
+
# Glob characters written as one-character classes in per-entry patterns,
|
|
526
|
+
# as ccc's glob library escapes them; its backslash escaping is off on Windows.
|
|
527
|
+
CCC_CLASS_ESCAPES = frozenset("*?[]{}")
|
|
528
|
+
|
|
529
|
+
SETTINGS_DIR = ".cocoindex_code"
|
|
530
|
+
SETTINGS_NAME = "settings.yml"
|
|
531
|
+
BACKUP_NAME = "settings.yml.skf-repair"
|
|
532
|
+
FIX_HINT = "fix the value in {project-root}/_bmad/skf/config.yaml"
|
|
533
|
+
|
|
534
|
+
# Clone mode: the build and dependency folders create-skill keeps out of the
|
|
535
|
+
# index of a workspace clone. They are generic artifact patterns, not SKF
|
|
536
|
+
# paths, in the `**/name` form (ccc matches it at any depth; a trailing-slash
|
|
537
|
+
# form such as `build/` matches nothing).
|
|
538
|
+
CLONE_EXCLUDES = (
|
|
539
|
+
"**/node_modules",
|
|
540
|
+
"**/dist",
|
|
541
|
+
"**/build",
|
|
542
|
+
"**/.git",
|
|
543
|
+
"**/vendor",
|
|
544
|
+
"**/__pycache__",
|
|
545
|
+
"**/.cache",
|
|
546
|
+
"**/.next",
|
|
547
|
+
"**/.nuxt",
|
|
548
|
+
"**/target",
|
|
549
|
+
"**/out",
|
|
550
|
+
"**/.venv",
|
|
551
|
+
"**/.tox",
|
|
552
|
+
)
|
|
553
|
+
# An extension --include-ext takes: letters, digits, `_`, `+` and `-` only,
|
|
554
|
+
# so `**/*.<ext>` is never a wider glob than one file type.
|
|
555
|
+
EXTENSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_+-]*$")
|
|
556
|
+
# A nested file with a given extension, matched against include_patterns.
|
|
557
|
+
INCLUDE_PROBE = "skf-probe/skf-probe.{ext}"
|
|
558
|
+
# A bare `- *...` list item: YAML reads the value as an alias. The value ends
|
|
559
|
+
# before trailing blanks or a comment (a `#` after a blank), which stay after it.
|
|
560
|
+
BARE_GLOB_ITEM_RE = re.compile(r"^([ \t]*-[ \t]+)(\*[^'\"\n]*?)((?:[ \t]+#[^\n]*)?|[ \t]*)$", re.MULTILINE)
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
class HelperError(Exception):
|
|
564
|
+
"""A handled failure that ends the run with a JSON error and exit `code`."""
|
|
565
|
+
|
|
566
|
+
def __init__(self, code: int, message: str):
|
|
567
|
+
super().__init__(message)
|
|
568
|
+
self.code = code
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _payload_safe(text) -> str:
|
|
572
|
+
"""Make text safe for a single-quoted `echo '...'` payload.
|
|
573
|
+
|
|
574
|
+
`'` becomes a backtick, a backslash becomes `/`, and control characters
|
|
575
|
+
and lone surrogates become `?`: dash's `echo` rewrites backslash
|
|
576
|
+
escapes, and a surrogate cannot be encoded.
|
|
577
|
+
"""
|
|
578
|
+
return "".join(
|
|
579
|
+
"`" if ch == "'" else "/" if ch == "\\" else
|
|
580
|
+
"?" if ord(ch) < 0x20 or ord(ch) == 0x7F or 0xD800 <= ord(ch) <= 0xDFFF else ch
|
|
581
|
+
for ch in str(text))
|
|
105
582
|
|
|
106
583
|
|
|
107
584
|
def _die(code: int, message: str) -> None:
|
|
108
|
-
print(json.dumps({"status": "error", "message": message}), file=sys.stderr)
|
|
585
|
+
print(json.dumps({"status": "error", "message": _payload_safe(message)}), file=sys.stderr)
|
|
109
586
|
sys.exit(code)
|
|
110
587
|
|
|
111
588
|
|
|
112
|
-
def
|
|
113
|
-
|
|
114
|
-
payload.setdefault("version", "v1")
|
|
115
|
-
print(json.dumps(payload))
|
|
589
|
+
def _child_env() -> dict:
|
|
590
|
+
"""os.environ without the git location variables, plus PYTHONUTF8=1.
|
|
116
591
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
592
|
+
A git hook (for example `git commit -a` running the test suite) exports
|
|
593
|
+
an absolute GIT_INDEX_FILE; inherited by `git -C <other repo>`, it would
|
|
594
|
+
make git read the hook's index instead of the project's.
|
|
595
|
+
"""
|
|
596
|
+
env = {k: v for k, v in os.environ.items() if k not in GIT_LOCATION_VARS}
|
|
597
|
+
env["PYTHONUTF8"] = "1"
|
|
598
|
+
return env
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def _resolve_outside_cwd(command: str) -> str | None:
|
|
602
|
+
"""shutil.which with a CWD-shim guard. Returns the resolved path or None.
|
|
603
|
+
|
|
604
|
+
shutil.which on Windows searches the current directory ahead of PATH,
|
|
605
|
+
and CWD here is the repo under analysis — a bare-name lookup resolving
|
|
606
|
+
into CWD would execute a repo-planted shim (e.g. ccc.cmd). Such a
|
|
607
|
+
resolution is treated as not-found. Explicit paths supplied by callers
|
|
608
|
+
(containing a separator) are honored as-is. Keep identical to the
|
|
609
|
+
sibling guards in skf-detect-tools.py,
|
|
610
|
+
skf-qmd-classify-collections.py, skf-ccc-git-hygiene.py,
|
|
611
|
+
skf-source-tree.py, skf-tessl-review.py and
|
|
612
|
+
skf-verify-provenance-completeness.py.
|
|
613
|
+
"""
|
|
614
|
+
resolved = shutil.which(command)
|
|
615
|
+
if resolved is None:
|
|
616
|
+
return None
|
|
617
|
+
if os.sep in command or (os.altsep and os.altsep in command):
|
|
618
|
+
return resolved
|
|
619
|
+
resolved_dir = os.path.dirname(resolved)
|
|
620
|
+
if resolved_dir:
|
|
621
|
+
cwd = os.path.normcase(os.path.abspath(os.getcwd()))
|
|
622
|
+
if os.path.normcase(os.path.abspath(resolved_dir)) == cwd:
|
|
623
|
+
return None
|
|
624
|
+
return resolved
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def _atomic_write(target: Path, content: str, unique: bool = False) -> None:
|
|
628
|
+
"""Crash-safe write via temp + fsync + rename. Mirrors skf-atomic-write.py.
|
|
629
|
+
|
|
630
|
+
With `unique`, the temp file is one of this call's own (clone mode,
|
|
631
|
+
where two runs can write the same file at once), else `<name>.skf-tmp`.
|
|
632
|
+
Raises HelperError(2) on any OS failure; the temp file is removed.
|
|
633
|
+
"""
|
|
121
634
|
tmp = target.with_name(target.name + ".skf-tmp")
|
|
635
|
+
if unique:
|
|
636
|
+
tmp = target.with_name(f"{target.name}.{os.getpid()}.{os.urandom(4).hex()}.skf-tmp")
|
|
122
637
|
# O_BINARY (Windows only; 0 elsewhere) suppresses the text-mode \n -> \r\n
|
|
123
638
|
# translation that would otherwise corrupt verbatim writes on Windows.
|
|
124
639
|
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0)
|
|
125
640
|
try:
|
|
641
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
126
642
|
fd = os.open(tmp, flags, 0o644)
|
|
127
643
|
try:
|
|
128
644
|
os.write(fd, content.encode("utf-8"))
|
|
@@ -131,75 +647,153 @@ def _atomic_write(target: Path, content: str) -> None:
|
|
|
131
647
|
os.close(fd)
|
|
132
648
|
os.replace(tmp, target)
|
|
133
649
|
except OSError as e:
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
650
|
+
try:
|
|
651
|
+
tmp.unlink()
|
|
652
|
+
except OSError:
|
|
653
|
+
pass
|
|
654
|
+
raise HelperError(2, f"atomic write failed for {target}: {e}")
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
# ─── External commands (git, ccc) ───────────────────────────────────────────
|
|
140
658
|
|
|
141
659
|
|
|
142
|
-
|
|
660
|
+
def _git(root: Path, *args: str, literal: bool = False) -> tuple[int, bytes, str] | None:
|
|
661
|
+
"""Run `git -C root <args>`. Return (returncode, stdout bytes, stderr text).
|
|
143
662
|
|
|
663
|
+
Returns None when git cannot be resolved or fails to start, and
|
|
664
|
+
(-1, b"", "") on timeout. `literal=True` adds `--literal-pathspecs` so a
|
|
665
|
+
folder value such as `:odd` is never read as pathspec magic — only for
|
|
666
|
+
`ls-files`: `check-ignore` rejects that flag. git runs with LC_ALL=C so
|
|
667
|
+
its "not a git repository" message can be recognised in any locale.
|
|
668
|
+
"""
|
|
669
|
+
exe = _resolve_outside_cwd("git")
|
|
670
|
+
if exe is None:
|
|
671
|
+
return None
|
|
672
|
+
argv = [exe, *(["--literal-pathspecs"] if literal else []), "-C", str(root), *args]
|
|
673
|
+
env = _child_env()
|
|
674
|
+
env["LC_ALL"] = "C"
|
|
675
|
+
try:
|
|
676
|
+
result = subprocess.run(
|
|
677
|
+
argv,
|
|
678
|
+
stdin=subprocess.DEVNULL,
|
|
679
|
+
capture_output=True,
|
|
680
|
+
env=env,
|
|
681
|
+
timeout=GIT_TIMEOUT_SEC,
|
|
682
|
+
)
|
|
683
|
+
except subprocess.TimeoutExpired:
|
|
684
|
+
return -1, b"", ""
|
|
685
|
+
except (OSError, ValueError):
|
|
686
|
+
return None
|
|
687
|
+
return result.returncode, result.stdout, result.stderr.decode("utf-8", errors="replace")
|
|
144
688
|
|
|
145
|
-
def resolve_repo_relative(raw_value: str, project_root: Path) -> str:
|
|
146
|
-
"""Normalize a config value like '{project-root}/skills' to 'skills'.
|
|
147
689
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
- `<segments>` (already relative) → returns as-is, stripped
|
|
151
|
-
- Other input → returns the stripped value (validate_config_value
|
|
152
|
-
will catch absolute paths, placeholders, glob-meta).
|
|
690
|
+
def run_ccc_init(root: Path) -> tuple[bool, str]:
|
|
691
|
+
"""Run `ccc init -f` in root. Return (exit code was 0, condensed output).
|
|
153
692
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
693
|
+
The caller judges success by whether settings.yml exists afterwards,
|
|
694
|
+
not by the exit code. Output is stdout plus stderr, decoded as UTF-8,
|
|
695
|
+
whitespace-collapsed and cut to 300 characters.
|
|
696
|
+
"""
|
|
697
|
+
exe = _resolve_outside_cwd("ccc")
|
|
698
|
+
if exe is None:
|
|
699
|
+
return False, "ccc was not found on PATH"
|
|
700
|
+
try:
|
|
701
|
+
result = subprocess.run(
|
|
702
|
+
[exe, "init", "-f"],
|
|
703
|
+
cwd=str(root),
|
|
704
|
+
stdin=subprocess.DEVNULL,
|
|
705
|
+
capture_output=True,
|
|
706
|
+
env=_child_env(),
|
|
707
|
+
timeout=CCC_INIT_TIMEOUT_SEC,
|
|
708
|
+
)
|
|
709
|
+
except subprocess.TimeoutExpired:
|
|
710
|
+
return False, f"ccc init timed out after {CCC_INIT_TIMEOUT_SEC}s"
|
|
711
|
+
except (OSError, ValueError) as e:
|
|
712
|
+
return False, f"ccc init could not start: {e}"
|
|
713
|
+
output = (result.stdout + result.stderr).decode("utf-8", errors="replace")
|
|
714
|
+
return result.returncode == 0, " ".join(output.split())[:300]
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
# ─── Config-value normalization and validation ─────────────────────────────
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def resolve_repo_relative(raw_value, project_root: Path) -> str:
|
|
721
|
+
"""Normalize a config value like '{project-root}/skills/' to 'skills'.
|
|
722
|
+
|
|
723
|
+
Strips whitespace, turns `\\` into `/`, drops a leading
|
|
724
|
+
`{project-root}/` (`{project-root}` alone gives ""), and collapses
|
|
725
|
+
`./`, `//` and a trailing `/`. Absolute paths and `..` segments are
|
|
726
|
+
kept so validate_config_value can refuse them.
|
|
157
727
|
"""
|
|
158
728
|
if raw_value is None:
|
|
159
729
|
return ""
|
|
160
|
-
value = str(raw_value).strip()
|
|
730
|
+
value = str(raw_value).strip().replace("\\", "/")
|
|
161
731
|
if value.startswith("{project-root}/"):
|
|
162
732
|
value = value[len("{project-root}/"):]
|
|
163
733
|
elif value == "{project-root}":
|
|
164
734
|
value = ""
|
|
165
|
-
return value
|
|
735
|
+
return PurePosixPath(value).as_posix() if value else ""
|
|
166
736
|
|
|
167
737
|
|
|
168
|
-
def validate_config_value(key: str, raw_value
|
|
169
|
-
"""Validate a
|
|
738
|
+
def validate_config_value(key: str, raw_value) -> tuple[str | None, str | None]:
|
|
739
|
+
"""Validate a folder value used as a root-anchored ccc exclusion.
|
|
170
740
|
|
|
171
741
|
Returns (cleaned_value, warning_or_None). When cleaned_value is None
|
|
172
|
-
the value was rejected and the caller
|
|
742
|
+
the value was rejected and the caller must not add the pattern. The
|
|
743
|
+
first failing rule wins.
|
|
173
744
|
"""
|
|
174
|
-
if raw_value is None or
|
|
745
|
+
if raw_value is None or str(raw_value).strip() in ("", "."):
|
|
175
746
|
return None, (
|
|
176
747
|
f"{key} is empty or whitespace-only; refused for ccc exclusion because "
|
|
177
|
-
f"
|
|
178
|
-
f"
|
|
179
|
-
f"value in {{project-root}}/_bmad/skf/config.yaml"
|
|
748
|
+
f"an empty pattern would exclude the entire repository from indexing; "
|
|
749
|
+
f"{FIX_HINT}"
|
|
180
750
|
)
|
|
181
751
|
|
|
182
752
|
value = str(raw_value).strip()
|
|
183
753
|
|
|
184
|
-
if value.startswith("/") or
|
|
754
|
+
if value.startswith(("/", "~", "./")) or re.match(r"^[A-Za-z]:", value) or value.endswith("/"):
|
|
185
755
|
return None, (
|
|
186
756
|
f"{key} is an absolute or anchored path; refused for ccc exclusion "
|
|
187
|
-
f"because
|
|
188
|
-
f"
|
|
757
|
+
f"because SKF exclusions are relative to the project root; "
|
|
758
|
+
f"{FIX_HINT} to a repo-relative path"
|
|
759
|
+
)
|
|
760
|
+
|
|
761
|
+
if ".." in PurePosixPath(value).parts:
|
|
762
|
+
return None, (
|
|
763
|
+
f"{key} contains a .. segment; refused for ccc exclusion because the "
|
|
764
|
+
f"folder must sit inside the project; {FIX_HINT}"
|
|
765
|
+
)
|
|
766
|
+
|
|
767
|
+
if value.startswith("!"):
|
|
768
|
+
return None, (
|
|
769
|
+
f"{key} starts with !, which ccc reads as a negation; refused for ccc "
|
|
770
|
+
f"exclusion; {FIX_HINT}"
|
|
189
771
|
)
|
|
190
772
|
|
|
191
773
|
if any(ch in GLOB_META_CHARS for ch in value):
|
|
192
774
|
return None, (
|
|
193
|
-
f"{key} contains glob meta-character (*, ?, [
|
|
194
|
-
f"exclusion because
|
|
195
|
-
f"
|
|
775
|
+
f"{key} contains a glob meta-character (*, ?, [, ] or a backslash); "
|
|
776
|
+
f"refused for ccc exclusion because ccc would read it as pattern "
|
|
777
|
+
f"syntax; {FIX_HINT}"
|
|
196
778
|
)
|
|
197
779
|
|
|
198
780
|
if any(ch in PLACEHOLDER_CHARS for ch in value):
|
|
199
781
|
return None, (
|
|
200
782
|
f"{key} contains an unresolved template placeholder ({{ or }}); "
|
|
201
783
|
f"refused for ccc exclusion because the step is supposed to forward "
|
|
202
|
-
f"the raw config value and let this script resolve
|
|
784
|
+
f"the raw config value and let this script resolve the project-root placeholder"
|
|
785
|
+
)
|
|
786
|
+
|
|
787
|
+
if any(ord(ch) < 0x20 or ord(ch) == 0x7F for ch in value):
|
|
788
|
+
return None, (
|
|
789
|
+
f"{key} contains a control character; refused for ccc exclusion "
|
|
790
|
+
f"because it does not survive a quoted shell payload; {FIX_HINT}"
|
|
791
|
+
)
|
|
792
|
+
|
|
793
|
+
if "'" in value:
|
|
794
|
+
return None, (
|
|
795
|
+
f"{key} contains a single quote, which breaks a single-quoted shell payload; "
|
|
796
|
+
f"refused for ccc exclusion; {FIX_HINT}"
|
|
203
797
|
)
|
|
204
798
|
|
|
205
799
|
return value, None
|
|
@@ -210,161 +804,1651 @@ def validate_config_value(key: str, raw_value: str) -> tuple[str | None, str | N
|
|
|
210
804
|
|
|
211
805
|
def assemble_patterns(skills_output_folder: str, forge_data_folder: str
|
|
212
806
|
) -> tuple[list[str], list[str]]:
|
|
213
|
-
"""Return (
|
|
807
|
+
"""Return (patterns, warnings) without touching the filesystem.
|
|
214
808
|
|
|
215
|
-
|
|
216
|
-
|
|
809
|
+
Patterns are the 4 unconditional ones plus each folder value that
|
|
810
|
+
passed validation, anchored as-is (a value shared by both folders
|
|
811
|
+
appears once). The merge itself also runs the collision check, which
|
|
812
|
+
this pure helper does not.
|
|
217
813
|
"""
|
|
218
814
|
patterns = list(ALWAYS_INCLUDE)
|
|
219
815
|
warnings: list[str] = []
|
|
816
|
+
for (key, _kind), raw in zip(FOLDER_KEYS, (skills_output_folder, forge_data_folder)):
|
|
817
|
+
cleaned, warn = validate_config_value(key, raw)
|
|
818
|
+
if cleaned is None:
|
|
819
|
+
warnings.append(warn)
|
|
820
|
+
elif cleaned not in patterns:
|
|
821
|
+
patterns.append(cleaned)
|
|
822
|
+
return patterns, warnings
|
|
220
823
|
|
|
221
|
-
cleaned, warn = validate_config_value("skills_output_folder", skills_output_folder)
|
|
222
|
-
if cleaned is not None:
|
|
223
|
-
patterns.append(f"**/{cleaned}")
|
|
224
|
-
elif warn:
|
|
225
|
-
warnings.append(warn)
|
|
226
824
|
|
|
227
|
-
|
|
228
|
-
if cleaned is not None:
|
|
229
|
-
patterns.append(f"**/{cleaned}")
|
|
230
|
-
elif warn:
|
|
231
|
-
warnings.append(warn)
|
|
825
|
+
# ─── Folder check ───────────────────────────────────────────────────────────
|
|
232
826
|
|
|
233
|
-
return patterns, warnings
|
|
234
827
|
|
|
828
|
+
# Only `.gitignore` files: ccc reads no other ignore file, so an entry that
|
|
829
|
+
# `.git/info/exclude` or a global excludes file hides is still indexed.
|
|
830
|
+
_LS_FILES_ARGS = ("ls-files", "-z", "--cached", "--others",
|
|
831
|
+
"--exclude-per-directory=.gitignore")
|
|
235
832
|
|
|
236
|
-
# ─── settings.yml read / merge / render ─────────────────────────────────────
|
|
237
833
|
|
|
834
|
+
def _ls_files(cwd: Path, *pathspec: str) -> tuple[list[str] | None, str | None]:
|
|
835
|
+
"""Run `git ls-files` in cwd. Return (records, problem).
|
|
836
|
+
|
|
837
|
+
records None means the listing is unavailable. problem is None for a
|
|
838
|
+
silent skip (git missing, not a git repository) and otherwise a short
|
|
839
|
+
description for a warning (timeout, any other git failure such as a
|
|
840
|
+
repository git refuses as unsafe).
|
|
841
|
+
"""
|
|
842
|
+
res = _git(cwd, *_LS_FILES_ARGS, *(("--", *pathspec) if pathspec else ()), literal=True)
|
|
843
|
+
if res is None:
|
|
844
|
+
return None, None
|
|
845
|
+
returncode, stdout, stderr = res
|
|
846
|
+
if returncode == -1:
|
|
847
|
+
return None, "git ls-files timed out"
|
|
848
|
+
if returncode != 0:
|
|
849
|
+
if "not a git repository" in stderr.lower():
|
|
850
|
+
return None, None
|
|
851
|
+
detail = next((line.strip() for line in stderr.splitlines() if line.strip()),
|
|
852
|
+
f"exit code {returncode}")
|
|
853
|
+
return None, f"git ls-files failed ({detail[:200]})"
|
|
854
|
+
# surrogateescape keeps an undecodable name equal to the name
|
|
855
|
+
# os.listdir returns, so the existence check and the disk match still
|
|
856
|
+
# find it; _unwritable then refuses it a pattern.
|
|
857
|
+
return [r.decode("utf-8", errors="surrogateescape") for r in stdout.split(b"\0") if r], None
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
def list_folder_paths(root: Path, folder: str) -> tuple[list[tuple[str, ...]] | None, str | None]:
|
|
861
|
+
"""Return (paths relative to folder, problem).
|
|
862
|
+
|
|
863
|
+
Lists tracked files plus untracked files no `.gitignore` excludes (the
|
|
864
|
+
only ignore files ccc reads); a tracked file deleted from the worktree
|
|
865
|
+
is dropped, since ccc cannot see it. A missing folder gives ([], None).
|
|
866
|
+
None paths mean the check is skipped; problem (see _ls_files) says
|
|
867
|
+
whether that deserves a warning. When git reports the folder itself as
|
|
868
|
+
one entry — a submodule or a nested repository, which ccc indexes like
|
|
869
|
+
any other folder — its own files are listed from inside it. A nested
|
|
870
|
+
repository deeper down comes back as a single directory entry, which
|
|
871
|
+
the classifier treats as a group.
|
|
872
|
+
"""
|
|
873
|
+
if not (root / folder).is_dir():
|
|
874
|
+
return [], None
|
|
875
|
+
records, problem = _ls_files(root, folder)
|
|
876
|
+
if records is None:
|
|
877
|
+
return None, problem
|
|
878
|
+
depth = len(PurePosixPath(folder).parts)
|
|
879
|
+
cwd = root
|
|
880
|
+
if any(len(PurePosixPath(r).parts) <= depth for r in records):
|
|
881
|
+
records, problem = _ls_files(root / folder)
|
|
882
|
+
if records is None:
|
|
883
|
+
return None, problem
|
|
884
|
+
depth = 0
|
|
885
|
+
cwd = root / folder
|
|
886
|
+
found: set[tuple[str, ...]] = set()
|
|
887
|
+
for record in records:
|
|
888
|
+
if not os.path.lexists(cwd / record):
|
|
889
|
+
continue
|
|
890
|
+
parts = PurePosixPath(record).parts[depth:]
|
|
891
|
+
if parts:
|
|
892
|
+
found.add(parts)
|
|
893
|
+
return sorted(found), None
|
|
238
894
|
|
|
239
|
-
def read_settings_yml(path: Path) -> tuple[dict, bool]:
|
|
240
|
-
"""Return (parsed_dict, existed_before).
|
|
241
895
|
|
|
242
|
-
|
|
243
|
-
|
|
896
|
+
# Keep identical to _has_skf_metadata in skf-skill-inventory.py
|
|
897
|
+
# (test/test-skf-skill-inventory.py pins the copies).
|
|
898
|
+
def _has_skf_metadata(path: Path) -> bool:
|
|
899
|
+
"""True when a flat skill's metadata.json carries an SKF marker."""
|
|
900
|
+
try:
|
|
901
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
902
|
+
except (OSError, ValueError):
|
|
903
|
+
return False
|
|
904
|
+
if not isinstance(data, dict):
|
|
905
|
+
return False
|
|
906
|
+
generated_by = data.get("generated_by")
|
|
907
|
+
if isinstance(generated_by, str) and generated_by in SKF_GENERATORS:
|
|
908
|
+
return True
|
|
909
|
+
tool_versions = data.get("tool_versions")
|
|
910
|
+
if isinstance(tool_versions, dict) and "skf" in tool_versions:
|
|
911
|
+
return True
|
|
912
|
+
return (data.get("skill_type") in ("single", "individual", "stack")
|
|
913
|
+
and ("forge_tier" in data or "confidence_tier" in data))
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
# Keep identical to _is_link_or_junction in skf-atomic-write.py
|
|
917
|
+
# (test/test-skf-skill-inventory.py pins the copies).
|
|
918
|
+
def _is_link_or_junction(p: Path) -> bool:
|
|
919
|
+
"""True for POSIX symlinks AND Windows junctions/symlinks.
|
|
920
|
+
|
|
921
|
+
`Path.is_symlink()` is False for Windows junctions; os.readlink succeeds
|
|
922
|
+
for both symlinks and junctions (since CPython 3.8 on Windows). A regular
|
|
923
|
+
directory raises OSError on readlink, which is the signal we want to
|
|
924
|
+
refuse replacement. On Windows, any other reparse point (a cloud-sync
|
|
925
|
+
placeholder, a deduplicated file, an app execution alias) raises
|
|
926
|
+
ValueError: it does not redirect to another path, so it is not a link.
|
|
927
|
+
"""
|
|
928
|
+
if p.is_symlink():
|
|
929
|
+
return True
|
|
930
|
+
if not p.exists() and not p.is_symlink():
|
|
931
|
+
return False
|
|
932
|
+
try:
|
|
933
|
+
os.readlink(p)
|
|
934
|
+
return True
|
|
935
|
+
except (OSError, ValueError):
|
|
936
|
+
return False
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
# Keep identical to _is_marked_version in skf-skill-inventory.py
|
|
940
|
+
# (test/test-skf-skill-inventory.py pins the copies).
|
|
941
|
+
def _is_marked_version(version_dir: Path, name: str) -> bool:
|
|
942
|
+
"""True when `version_dir/name/metadata.json` carries an SKF marker.
|
|
943
|
+
|
|
944
|
+
SKF never links a version folder or the package inside one: a linked
|
|
945
|
+
one is not its output, whatever the metadata behind the link says.
|
|
946
|
+
"""
|
|
947
|
+
package = version_dir / name
|
|
948
|
+
return (not _is_link_or_junction(version_dir) and not _is_link_or_junction(package)
|
|
949
|
+
and _has_skf_metadata(package / "metadata.json"))
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
# Keep identical to _has_skf_evidence in skf-skill-inventory.py
|
|
953
|
+
# (test/test-skf-skill-inventory.py pins the copies).
|
|
954
|
+
def _has_skf_evidence(group_dir: Path, name: str) -> bool:
|
|
955
|
+
"""True when skill group `group_dir` holds evidence that SKF generated it.
|
|
956
|
+
|
|
957
|
+
Evidence is the `_batch` name, `.skf-` in the name, a marked root
|
|
958
|
+
`metadata.json`, or a marked version folder (`.skf-` staging folders
|
|
959
|
+
aside). The versioned layout, an `active` link, a manifest key or a
|
|
960
|
+
result file are not evidence on their own, and a linked group holds none.
|
|
961
|
+
"""
|
|
962
|
+
if _is_link_or_junction(group_dir):
|
|
963
|
+
return False
|
|
964
|
+
if name == "_batch" or ".skf-" in name:
|
|
965
|
+
return True
|
|
966
|
+
if _has_skf_metadata(group_dir / "metadata.json"):
|
|
967
|
+
return True
|
|
968
|
+
try:
|
|
969
|
+
children = list(group_dir.iterdir())
|
|
970
|
+
except OSError:
|
|
971
|
+
return False
|
|
972
|
+
return any(".skf-" not in child.name and child.is_dir() and _is_marked_version(child, name)
|
|
973
|
+
for child in children)
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
# Keep identical to _has_forge_evidence in skf-skill-inventory.py
|
|
977
|
+
# (test/test-skf-skill-inventory.py pins the copies).
|
|
978
|
+
def _has_forge_evidence(group_dir: Path, name: str) -> bool:
|
|
979
|
+
"""True when forge group `group_dir` holds evidence that SKF generated it.
|
|
980
|
+
|
|
981
|
+
Evidence is `.skf-` in the name, SKF's own `_campaign` or
|
|
982
|
+
`improvement-queue` folder, a skill brief (`skill-brief.yaml*` or
|
|
983
|
+
`.brief-draft.json`) or a `*-result*.json` file directly in the group, or
|
|
984
|
+
a provenance map, evidence report, extraction rules or `*-result*.json`
|
|
985
|
+
file directly in a folder of the group (`.skf-` staging folders aside).
|
|
986
|
+
A linked group, folder or file is never evidence.
|
|
987
|
+
"""
|
|
988
|
+
if _is_link_or_junction(group_dir):
|
|
989
|
+
return False
|
|
990
|
+
if ".skf-" in name or name in FORGE_GROUP_DIRS:
|
|
991
|
+
return True
|
|
992
|
+
try:
|
|
993
|
+
children = list(group_dir.iterdir())
|
|
994
|
+
except OSError:
|
|
995
|
+
return False
|
|
996
|
+
for child in children:
|
|
997
|
+
entry = child.name
|
|
998
|
+
if _is_link_or_junction(child):
|
|
999
|
+
continue
|
|
1000
|
+
if child.is_file():
|
|
1001
|
+
if (entry.startswith("skill-brief.yaml") or entry == ".brief-draft.json"
|
|
1002
|
+
or RESULT_JSON_RE.match(entry)):
|
|
1003
|
+
return True
|
|
1004
|
+
elif child.is_dir() and ".skf-" not in entry:
|
|
1005
|
+
try:
|
|
1006
|
+
inner = list(child.iterdir())
|
|
1007
|
+
except OSError:
|
|
1008
|
+
continue
|
|
1009
|
+
if any((f.name in FORGE_VERSION_ANCHORS or RESULT_JSON_RE.match(f.name))
|
|
1010
|
+
and not _is_link_or_junction(f) and f.is_file() for f in inner):
|
|
1011
|
+
return True
|
|
1012
|
+
return False
|
|
1013
|
+
|
|
1014
|
+
|
|
1015
|
+
def _skf_group(base: Path, kind: str, group: str) -> bool:
|
|
1016
|
+
"""True when first-level entry `group` under the folder is SKF output of `kind`.
|
|
1017
|
+
|
|
1018
|
+
Both kinds follow the rule the workflows use before they write into,
|
|
1019
|
+
move or delete a skill's folders (skf-skill-inventory.py), read from the
|
|
1020
|
+
group on disk, so ccc excludes exactly the groups the workflows treat as
|
|
1021
|
+
SKF output.
|
|
1022
|
+
"""
|
|
1023
|
+
if kind == "skills":
|
|
1024
|
+
return _has_skf_evidence(base / group, group)
|
|
1025
|
+
return _has_forge_evidence(base / group, group)
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
def _skf_root_file(kind: str, name: str) -> bool:
|
|
1029
|
+
"""True when a file directly under the folder is SKF output."""
|
|
1030
|
+
return bool(
|
|
1031
|
+
".skf-" in name
|
|
1032
|
+
or RESULT_JSON_RE.match(name)
|
|
1033
|
+
or (kind == "skills" and name == ".export-manifest.json")
|
|
1034
|
+
or (kind == "forge" and name.startswith(FORGE_ROOT_PREFIXES))
|
|
1035
|
+
)
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
def _holds_other_folder(folder: str, group: str, other_folders) -> bool:
|
|
1039
|
+
"""True when folder/group is, or contains, another configured SKF folder."""
|
|
1040
|
+
group_parts = PurePosixPath(folder, group).parts
|
|
1041
|
+
return any(PurePosixPath(other).parts[:len(group_parts)] == group_parts
|
|
1042
|
+
for other in other_folders)
|
|
1043
|
+
|
|
1044
|
+
|
|
1045
|
+
class FolderScan(NamedTuple):
|
|
1046
|
+
"""What classify_folder found in one configured folder."""
|
|
1047
|
+
|
|
1048
|
+
foreign: list[str] # the entries SKF did not generate that decide the verdict
|
|
1049
|
+
owned_groups: list[str] # first-level directories SKF generated
|
|
1050
|
+
owned_files: list[str] # root files SKF generated
|
|
1051
|
+
has_output: bool # any SKF output, a group holding the other folder included
|
|
1052
|
+
listed: frozenset # every first-level name git listed
|
|
1053
|
+
|
|
1054
|
+
|
|
1055
|
+
def classify_folder(root: Path, folder: str, kind, paths, other_folders=(),
|
|
1056
|
+
hidden_neutral: bool = False) -> FolderScan:
|
|
1057
|
+
"""Classify the first-level entries of a configured folder.
|
|
1058
|
+
|
|
1059
|
+
`kind` is "skills" or "forge", or a collection of both when the two
|
|
1060
|
+
folder values are the same folder; an entry is SKF output when any of
|
|
1061
|
+
the kinds accepts it. `paths` comes from list_folder_paths. A group
|
|
1062
|
+
that is, or contains, another configured SKF folder (`other_folders`)
|
|
1063
|
+
is SKF output but gets no pattern of its own: that folder is reconciled
|
|
1064
|
+
separately. A single-segment entry that is a directory on disk (a
|
|
1065
|
+
nested repository or submodule) is a group, not a root file.
|
|
1066
|
+
`foreign`: any foreign group gives the foreign groups, as `name/`, then
|
|
1067
|
+
the other root files; otherwise other root files count only when the
|
|
1068
|
+
folder holds no SKF output at all. Neutral entries never count; with
|
|
1069
|
+
`hidden_neutral` (ccc's `**/.*` default is active) neither does any
|
|
1070
|
+
other dot-named entry that is not SKF output.
|
|
1071
|
+
"""
|
|
1072
|
+
kinds = {kind} if isinstance(kind, str) else set(kind)
|
|
1073
|
+
base = root / folder
|
|
1074
|
+
groups: dict[str, list[tuple[str, ...]]] = {}
|
|
1075
|
+
other_root: list[str] = []
|
|
1076
|
+
owned_files: list[str] = []
|
|
1077
|
+
for rel in paths:
|
|
1078
|
+
if len(rel) == 1 and not (base / rel[0]).is_dir():
|
|
1079
|
+
name = rel[0]
|
|
1080
|
+
if any(_skf_root_file(k, name) for k in kinds):
|
|
1081
|
+
owned_files.append(name)
|
|
1082
|
+
elif not (name in NEUTRAL_ROOT_FILES or README_RE.match(name)
|
|
1083
|
+
or (hidden_neutral and name.startswith("."))):
|
|
1084
|
+
other_root.append(name)
|
|
1085
|
+
continue
|
|
1086
|
+
members = groups.setdefault(rel[0], [])
|
|
1087
|
+
if len(rel) > 1:
|
|
1088
|
+
members.append(rel)
|
|
1089
|
+
foreign_groups: list[str] = []
|
|
1090
|
+
owned_groups: list[str] = []
|
|
1091
|
+
holds_other = False
|
|
1092
|
+
for group in sorted(groups):
|
|
1093
|
+
if _holds_other_folder(folder, group, other_folders):
|
|
1094
|
+
holds_other = True
|
|
1095
|
+
elif any(_skf_group(base, k, group) for k in kinds):
|
|
1096
|
+
owned_groups.append(group)
|
|
1097
|
+
elif hidden_neutral and group.startswith("."):
|
|
1098
|
+
continue
|
|
1099
|
+
else:
|
|
1100
|
+
foreign_groups.append(group + "/")
|
|
1101
|
+
has_output = bool(owned_files or owned_groups or holds_other)
|
|
1102
|
+
if foreign_groups:
|
|
1103
|
+
foreign = foreign_groups + other_root
|
|
1104
|
+
elif other_root and not has_output:
|
|
1105
|
+
foreign = other_root
|
|
1106
|
+
else:
|
|
1107
|
+
foreign = []
|
|
1108
|
+
listed = frozenset(groups) | frozenset(owned_files) | frozenset(other_root) | frozenset(
|
|
1109
|
+
rel[0] for rel in paths if len(rel) == 1)
|
|
1110
|
+
return FolderScan(foreign, owned_groups, sorted(owned_files), has_output, listed)
|
|
1111
|
+
|
|
1112
|
+
|
|
1113
|
+
def ccc_literal(name: str) -> str:
|
|
1114
|
+
"""Return `name` as a ccc glob that matches only itself.
|
|
1115
|
+
|
|
1116
|
+
Each glob character becomes a one-character class, as the escape
|
|
1117
|
+
function of ccc's glob library (globset) does. Backslash escapes are
|
|
1118
|
+
never used: globset turns them off on Windows and reads a backslash
|
|
1119
|
+
there as `/`.
|
|
1120
|
+
"""
|
|
1121
|
+
return "".join(f"[{ch}]" if ch in CCC_CLASS_ESCAPES else ch for ch in name)
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def _unwritable(name: str) -> bool:
|
|
1125
|
+
"""True when a name cannot go into a pattern.
|
|
1126
|
+
|
|
1127
|
+
A `'` breaks a single-quoted shell payload, a backslash or a control
|
|
1128
|
+
character does not survive `echo` in dash, a replacement character
|
|
1129
|
+
stands for a name that could not be decoded, and a surrogate (bytes on
|
|
1130
|
+
disk that are not UTF-8) makes ccc fail on the whole pattern list.
|
|
1131
|
+
"""
|
|
1132
|
+
return any(ch in "'\\\ufffd" or ord(ch) < 0x20 or ord(ch) == 0x7F
|
|
1133
|
+
or 0xD800 <= ord(ch) <= 0xDFFF for ch in name)
|
|
1134
|
+
|
|
1135
|
+
|
|
1136
|
+
def _norm(name: str) -> str:
|
|
1137
|
+
return unicodedata.normalize("NFC", name).casefold()
|
|
1138
|
+
|
|
1139
|
+
|
|
1140
|
+
def _on_disk(name: str, disk: list[str] | None, listed=frozenset()) -> list[str]:
|
|
1141
|
+
"""The names ccc sees for `name`: itself when listed on disk, else the
|
|
1142
|
+
entries equal to it up to Unicode normalization and case (macOS,
|
|
1143
|
+
Windows), else none (the entry is gone). No listing: `name` itself."""
|
|
1144
|
+
if disk is None or name in disk:
|
|
1145
|
+
return [name]
|
|
1146
|
+
key = _norm(name)
|
|
1147
|
+
return [d for d in disk if d not in listed and _norm(d) == key]
|
|
1148
|
+
|
|
1149
|
+
|
|
1150
|
+
def entry_patterns(folder: str, kind, owned_groups, owned_files, disk, listed=frozenset()
|
|
1151
|
+
) -> tuple[list[str], list[str]]:
|
|
1152
|
+
"""Return (patterns, skipped names) that exclude the SKF entries one by one.
|
|
1153
|
+
|
|
1154
|
+
Each pattern is `{folder}/` plus one entry name as it is on disk (`disk`,
|
|
1155
|
+
from os.listdir; None when unreadable), glob characters written as
|
|
1156
|
+
classes. The fixed entries and result stems of each kind, and every
|
|
1157
|
+
forge report prefix, are always included, so output SKF writes before
|
|
1158
|
+
the next setup run is covered. Timestamped root results share one
|
|
1159
|
+
`{folder}/{stem}-result*.json` and forge reports one `{folder}/{prefix}*`.
|
|
1160
|
+
Never a `!` negation: ccc applies it to every pattern, its own defaults
|
|
1161
|
+
included. Skipped names are SKF entries `_unwritable` refuses.
|
|
1162
|
+
"""
|
|
1163
|
+
kinds = {kind} if isinstance(kind, str) else set(kind)
|
|
1164
|
+
patterns: set[str] = set()
|
|
1165
|
+
skipped: set[str] = set()
|
|
1166
|
+
exact = list(owned_groups)
|
|
1167
|
+
for kind_ in kinds:
|
|
1168
|
+
exact.extend(SKF_FIXED_ENTRIES[kind_])
|
|
1169
|
+
for stem in SKF_ROOT_RESULT_STEMS[kind_]:
|
|
1170
|
+
patterns.add(f"{folder}/{stem}-result*.json")
|
|
1171
|
+
if kind_ == "forge":
|
|
1172
|
+
patterns.update(f"{folder}/{prefix}*" for prefix in FORGE_ROOT_PREFIXES)
|
|
1173
|
+
for name in owned_files:
|
|
1174
|
+
prefix = next((p for p in FORGE_ROOT_PREFIXES if name.startswith(p)), None)
|
|
1175
|
+
if "forge" in kinds and prefix:
|
|
1176
|
+
patterns.add(f"{folder}/{prefix}*")
|
|
1177
|
+
continue
|
|
1178
|
+
m = RESULT_JSON_RE.match(name)
|
|
1179
|
+
if m:
|
|
1180
|
+
patterns.add(f"{folder}/{m.group(1)}-result*.json")
|
|
1181
|
+
continue
|
|
1182
|
+
exact.append(name)
|
|
1183
|
+
fixed = {n for k in kinds for n in SKF_FIXED_ENTRIES[k]}
|
|
1184
|
+
for name in dict.fromkeys(exact):
|
|
1185
|
+
if name in fixed and (disk is None or name not in disk):
|
|
1186
|
+
patterns.add(f"{folder}/{ccc_literal(name)}")
|
|
1187
|
+
continue
|
|
1188
|
+
if _unwritable(name):
|
|
1189
|
+
skipped.add(name)
|
|
1190
|
+
continue
|
|
1191
|
+
for real in _on_disk(name, disk, listed):
|
|
1192
|
+
if _unwritable(real):
|
|
1193
|
+
skipped.add(real)
|
|
1194
|
+
else:
|
|
1195
|
+
patterns.add(f"{folder}/{ccc_literal(real)}")
|
|
1196
|
+
return sorted(patterns), sorted(skipped)
|
|
1197
|
+
|
|
1198
|
+
|
|
1199
|
+
# ─── settings.yml and the ownership record ─────────────────────────────────
|
|
1200
|
+
|
|
1201
|
+
|
|
1202
|
+
def load_settings(path: Path) -> dict:
|
|
1203
|
+
"""Parse settings.yml. An empty file gives {}.
|
|
1204
|
+
|
|
1205
|
+
Raises HelperError(1) when the file cannot be read or parsed, is not a
|
|
1206
|
+
mapping, or has an exclude_patterns that is neither a list nor null.
|
|
244
1207
|
"""
|
|
245
|
-
if not path.exists():
|
|
246
|
-
return {}, False
|
|
247
1208
|
try:
|
|
248
1209
|
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
249
|
-
except yaml.YAMLError as e:
|
|
250
|
-
|
|
1210
|
+
except (yaml.YAMLError, ValueError) as e:
|
|
1211
|
+
raise HelperError(1, f"failed to parse {path}: {e}")
|
|
1212
|
+
except OSError as e:
|
|
1213
|
+
raise HelperError(1, f"failed to read {path}: {e}")
|
|
251
1214
|
if data is None:
|
|
252
|
-
return {}
|
|
1215
|
+
return {}
|
|
253
1216
|
if not isinstance(data, dict):
|
|
254
|
-
|
|
255
|
-
|
|
1217
|
+
raise HelperError(1, f"expected mapping at top of {path}, got {type(data).__name__}")
|
|
1218
|
+
excludes = data.get("exclude_patterns")
|
|
1219
|
+
if excludes is not None and not isinstance(excludes, list):
|
|
1220
|
+
raise HelperError(1, f"exclude_patterns in {path} is not a list "
|
|
1221
|
+
f"(got {type(excludes).__name__})")
|
|
1222
|
+
return data
|
|
256
1223
|
|
|
257
1224
|
|
|
258
|
-
def
|
|
259
|
-
|
|
260
|
-
|
|
1225
|
+
def needs_rebuild(data: dict) -> bool:
|
|
1226
|
+
"""True when the file lacks the ccc defaults and must be rebuilt.
|
|
1227
|
+
|
|
1228
|
+
Every file ccc writes has both `exclude_patterns` (including
|
|
1229
|
+
`**/.cocoindex_code`) and `include_patterns`.
|
|
1230
|
+
"""
|
|
1231
|
+
if "exclude_patterns" not in data:
|
|
1232
|
+
return True
|
|
1233
|
+
excludes = [str(p) for p in (data.get("exclude_patterns") or [])]
|
|
1234
|
+
return "include_patterns" not in data and CCC_SELF_EXCLUDE not in excludes
|
|
1235
|
+
|
|
1236
|
+
|
|
1237
|
+
def read_owned_record(path, warnings: list[str]) -> list[str]:
|
|
1238
|
+
"""Return `ccc_index.exclude_patterns` from forge-tier.yaml, or [].
|
|
1239
|
+
|
|
1240
|
+
A missing path or file, or a non-list value, gives [] silently; an
|
|
1241
|
+
unreadable or unparseable file gives [] plus a warning.
|
|
1242
|
+
"""
|
|
1243
|
+
if path is None:
|
|
1244
|
+
return []
|
|
1245
|
+
path = Path(path)
|
|
1246
|
+
if not path.is_file():
|
|
1247
|
+
return []
|
|
1248
|
+
try:
|
|
1249
|
+
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
1250
|
+
except (OSError, ValueError, yaml.YAMLError):
|
|
1251
|
+
warnings.append("could not read the SKF exclusion record in forge-tier.yaml; "
|
|
1252
|
+
"treated it as empty")
|
|
1253
|
+
return []
|
|
1254
|
+
ccc = data.get("ccc_index") if isinstance(data, dict) else None
|
|
1255
|
+
record = ccc.get("exclude_patterns") if isinstance(ccc, dict) else None
|
|
1256
|
+
return [str(p) for p in record] if isinstance(record, list) else []
|
|
1257
|
+
|
|
1258
|
+
|
|
1259
|
+
def plan_exclusions(existing: list[str], produced: list[str], owned_prior, prune_allowed: bool
|
|
1260
|
+
) -> tuple[list[str], list[str], list[str]]:
|
|
1261
|
+
"""Return (merged, added, removed).
|
|
1262
|
+
|
|
1263
|
+
Removes the entries of `existing` that SKF owned and no longer produces
|
|
1264
|
+
(only when `prune_allowed`), keeps every other entry in order, and
|
|
1265
|
+
appends the produced patterns not already kept.
|
|
1266
|
+
"""
|
|
1267
|
+
removable = (set(owned_prior) - set(produced)) if prune_allowed else set()
|
|
1268
|
+
kept = [e for e in existing if e not in removable]
|
|
1269
|
+
removed: list[str] = []
|
|
1270
|
+
for e in existing:
|
|
1271
|
+
if e in removable and e not in removed:
|
|
1272
|
+
removed.append(e)
|
|
1273
|
+
added: list[str] = []
|
|
1274
|
+
for p in produced:
|
|
1275
|
+
if p not in kept and p not in added:
|
|
1276
|
+
added.append(p)
|
|
1277
|
+
return kept + added, added, removed
|
|
1278
|
+
|
|
1279
|
+
|
|
1280
|
+
def decide_index_action(ready: bool, skip_index: bool, written: bool, index_fresh: bool,
|
|
1281
|
+
usable: bool = True, defer: bool = False) -> str:
|
|
1282
|
+
"""Return one of INDEX_ACTIONS for the step to act on.
|
|
1283
|
+
|
|
1284
|
+
`usable` is False when settings.yml exists but still lacks the ccc
|
|
1285
|
+
defaults (a failed rebuild): a fresh index is kept, but no new index is
|
|
1286
|
+
built from that file. `defer` turns a due build ("index") into "defer".
|
|
1287
|
+
"""
|
|
1288
|
+
if not ready:
|
|
1289
|
+
return "fail"
|
|
1290
|
+
if skip_index:
|
|
1291
|
+
return "skip"
|
|
1292
|
+
if not usable:
|
|
1293
|
+
return "keep" if index_fresh else "fail"
|
|
1294
|
+
if written or not index_fresh:
|
|
1295
|
+
return "defer" if defer else "index"
|
|
1296
|
+
return "keep"
|
|
1297
|
+
|
|
1298
|
+
|
|
1299
|
+
# ─── Merge ──────────────────────────────────────────────────────────────────
|
|
1300
|
+
|
|
1301
|
+
|
|
1302
|
+
def _read_bytes_or_none(path: Path) -> bytes | None:
|
|
1303
|
+
try:
|
|
1304
|
+
return path.read_bytes() if path.is_file() else None
|
|
1305
|
+
except OSError:
|
|
1306
|
+
return None
|
|
1307
|
+
|
|
1308
|
+
|
|
1309
|
+
def _plural(n: int, one: str, many: str) -> str:
|
|
1310
|
+
return f"{n} {one if n == 1 else many}"
|
|
261
1311
|
|
|
262
|
-
|
|
263
|
-
|
|
1312
|
+
|
|
1313
|
+
def _output_hint(output: str) -> str:
|
|
1314
|
+
return output or "no output"
|
|
1315
|
+
|
|
1316
|
+
|
|
1317
|
+
def _prepare_settings(root: Path, target: Path, backup: Path, allow_ccc_init: bool,
|
|
1318
|
+
state: dict, warnings: list[str]) -> dict | None:
|
|
1319
|
+
"""Bring settings.yml to a reconcilable state.
|
|
1320
|
+
|
|
1321
|
+
Returns the parsed data to reconcile, or None when this run must not
|
|
1322
|
+
touch the SKF patterns (not ready, rebuild skipped or failed, or ccc
|
|
1323
|
+
wrote no exclude_patterns list).
|
|
1324
|
+
"""
|
|
1325
|
+
if not state["existed"]:
|
|
1326
|
+
if not allow_ccc_init:
|
|
1327
|
+
state.update(ready=False,
|
|
1328
|
+
reason="settings.yml is missing and --no-ccc-init was given")
|
|
1329
|
+
return None
|
|
1330
|
+
_ok, output = run_ccc_init(root)
|
|
1331
|
+
if not target.is_file():
|
|
1332
|
+
state.update(ready=False, ccc_init="failed",
|
|
1333
|
+
reason=("ccc init did not create .cocoindex_code/settings.yml "
|
|
1334
|
+
f"({_output_hint(output)})"))
|
|
1335
|
+
return None
|
|
1336
|
+
state.update(ccc_init="created", written=True)
|
|
1337
|
+
data = load_settings(target)
|
|
1338
|
+
if not isinstance(data.get("exclude_patterns"), list):
|
|
1339
|
+
warnings.append("ccc init wrote no exclude_patterns list, so SKF left its "
|
|
1340
|
+
"exclusions out rather than replace the ccc defaults")
|
|
1341
|
+
return None
|
|
1342
|
+
return data
|
|
1343
|
+
|
|
1344
|
+
data = load_settings(target)
|
|
1345
|
+
if not needs_rebuild(data):
|
|
1346
|
+
return data
|
|
1347
|
+
if not allow_ccc_init:
|
|
1348
|
+
warnings.append("settings.yml lacks the ccc default exclusions; rebuild skipped "
|
|
1349
|
+
"because --no-ccc-init was given")
|
|
1350
|
+
return None
|
|
1351
|
+
|
|
1352
|
+
original = data
|
|
1353
|
+
try:
|
|
1354
|
+
os.replace(target, backup)
|
|
1355
|
+
except OSError as e:
|
|
1356
|
+
raise HelperError(2, f"could not move {target} aside to {backup.name}: {e}")
|
|
1357
|
+
_ok, output = run_ccc_init(root)
|
|
1358
|
+
fresh = None
|
|
1359
|
+
if target.is_file():
|
|
1360
|
+
try:
|
|
1361
|
+
fresh = load_settings(target)
|
|
1362
|
+
except HelperError:
|
|
1363
|
+
fresh = None
|
|
1364
|
+
if fresh is None or not isinstance(fresh.get("exclude_patterns"), list):
|
|
1365
|
+
try:
|
|
1366
|
+
os.replace(backup, target)
|
|
1367
|
+
except OSError as e:
|
|
1368
|
+
raise HelperError(2, f"could not restore {target} from {backup.name}: {e}")
|
|
1369
|
+
state["ccc_init"] = "failed"
|
|
1370
|
+
state["index_block"] = ("settings.yml lacks the ccc default exclusions and ccc init "
|
|
1371
|
+
f"could not rebuild it ({_output_hint(output)})")
|
|
1372
|
+
warnings.append(
|
|
1373
|
+
"could not rebuild .cocoindex_code/settings.yml on the ccc defaults because "
|
|
1374
|
+
f"ccc init failed ({_output_hint(output)}); the file was left unchanged and "
|
|
1375
|
+
"SKF exclusions were not updated this run; fix ccc init in {project-root} "
|
|
1376
|
+
"and re-run /skf-setup"
|
|
1377
|
+
)
|
|
1378
|
+
return None
|
|
1379
|
+
|
|
1380
|
+
fresh_excludes = [str(p) for p in fresh["exclude_patterns"]]
|
|
1381
|
+
original_excludes = [str(p) for p in (original.get("exclude_patterns") or [])]
|
|
1382
|
+
data = {**fresh, **{k: v for k, v in original.items() if k != "exclude_patterns"}}
|
|
1383
|
+
data["exclude_patterns"] = fresh_excludes + [
|
|
1384
|
+
e for e in original_excludes if e not in fresh_excludes
|
|
1385
|
+
]
|
|
1386
|
+
state.update(ccc_init="rebuilt", written=True)
|
|
1387
|
+
warnings.append(
|
|
1388
|
+
"rebuilt .cocoindex_code/settings.yml on the ccc defaults: it lacked them, so "
|
|
1389
|
+
"ccc was also indexing hidden folders, node_modules and its own database; your "
|
|
1390
|
+
"own entries were kept"
|
|
1391
|
+
)
|
|
1392
|
+
return data
|
|
1393
|
+
|
|
1394
|
+
|
|
1395
|
+
def _inside_always_excluded(value: str) -> bool:
|
|
1396
|
+
"""True when an always-included `**/<name>` pattern already covers the folder."""
|
|
1397
|
+
return any(f"**/{part}" in ALWAYS_INCLUDE for part in PurePosixPath(value).parts)
|
|
1398
|
+
|
|
1399
|
+
|
|
1400
|
+
def _printable(name: str) -> str:
|
|
1401
|
+
"""A name safe to show in a warning: backslashes and control characters become `?`."""
|
|
1402
|
+
return "".join("?" if ch == "\\" or ord(ch) < 0x20 or ord(ch) == 0x7F
|
|
1403
|
+
or 0xD800 <= ord(ch) <= 0xDFFF else ch for ch in name)
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def _sample(names: list[str]) -> str:
|
|
1407
|
+
return ", ".join(_printable(n) for n in names[:SAMPLE_SIZE])
|
|
1408
|
+
|
|
1409
|
+
|
|
1410
|
+
def _collision_warning(key: str, value: str, foreign: list[str], kinds) -> str:
|
|
1411
|
+
installed = (
|
|
1412
|
+
" If those are skills installed from elsewhere, keep the folder: once SKF has "
|
|
1413
|
+
"written its own output there, re-run /skf-setup and SKF excludes only its own "
|
|
1414
|
+
"entries."
|
|
1415
|
+
) if "skills" in kinds else ""
|
|
1416
|
+
return (
|
|
1417
|
+
f"{key} {value} already holds {_plural(len(foreign), 'entry', 'entries')} SKF did "
|
|
1418
|
+
f"not generate and no SKF output (for example {_sample(foreign)}), so SKF left it "
|
|
1419
|
+
f"out of the ccc exclusions and that content stays indexed. If that is source you "
|
|
1420
|
+
f"work on, set {key} in {{project-root}}/_bmad/skf/config.yaml to a folder only SKF "
|
|
1421
|
+
f"uses and re-run /skf-setup.{installed} To exclude it anyway, add {value} to "
|
|
1422
|
+
f"exclude_patterns in .cocoindex_code/settings.yml yourself; SKF never removes "
|
|
1423
|
+
f"entries it did not add"
|
|
1424
|
+
)
|
|
1425
|
+
|
|
1426
|
+
|
|
1427
|
+
def _shared_folder_note(key: str, value: str, foreign: list[str], kinds) -> str:
|
|
1428
|
+
later = (
|
|
1429
|
+
"A skill SKF creates there later stays indexed, twice through its version folder "
|
|
1430
|
+
"and its active link, until you re-run /skf-setup."
|
|
1431
|
+
) if "skills" in kinds else (
|
|
1432
|
+
"Output SKF writes there later stays indexed until you re-run /skf-setup."
|
|
1433
|
+
)
|
|
1434
|
+
return (
|
|
1435
|
+
f"{key} {value} also holds {_plural(len(foreign), 'entry', 'entries')} SKF did not "
|
|
1436
|
+
f"generate (for example {_sample(foreign)}), so SKF excluded only its own entries "
|
|
1437
|
+
f"there and those stay indexed. {later} To exclude the other entries too, add each "
|
|
1438
|
+
f"as {value}/<name> to exclude_patterns in .cocoindex_code/settings.yml yourself; "
|
|
1439
|
+
f"SKF never removes entries it did not add"
|
|
1440
|
+
)
|
|
1441
|
+
|
|
1442
|
+
|
|
1443
|
+
def _unowned_entry_warning(key: str, value: str, foreign: list[str]) -> str:
|
|
1444
|
+
return (
|
|
1445
|
+
f"{key} {value} is in exclude_patterns in .cocoindex_code/settings.yml but SKF has "
|
|
1446
|
+
f"no record of adding it, so SKF left it in place; it also excludes "
|
|
1447
|
+
f"{_plural(len(foreign), 'entry', 'entries')} SKF did not generate (for example "
|
|
1448
|
+
f"{_sample(foreign)}). If those should stay searchable, remove {value} from "
|
|
1449
|
+
f"exclude_patterns and re-run /skf-setup; SKF then excludes only its own output there"
|
|
1450
|
+
)
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
def _unwritable_warning(key: str, value: str, skipped: list[str]) -> str:
|
|
1454
|
+
return (
|
|
1455
|
+
f"{key} {value} holds {_plural(len(skipped), 'SKF entry', 'SKF entries')} whose "
|
|
1456
|
+
f"name SKF cannot write as a ccc pattern (a quote, a backslash, a control character "
|
|
1457
|
+
f"or bytes that are not UTF-8; for example {_sample(skipped)}), so they stay indexed; "
|
|
1458
|
+
f"rename them, or add them to exclude_patterns in .cocoindex_code/settings.yml "
|
|
1459
|
+
f"yourself"
|
|
1460
|
+
)
|
|
1461
|
+
|
|
1462
|
+
|
|
1463
|
+
# ─── a user `!` entry that cancels an SKF pattern ──────────────────────────
|
|
1464
|
+
|
|
1465
|
+
# The child name ccc's matcher probes an excluded directory with when it
|
|
1466
|
+
# checks whether a `!` entry applies below it.
|
|
1467
|
+
CCC_DIR_PROBE = "__probe__"
|
|
1468
|
+
# Brace alternatives of one `!` entry checked at most, and `{` in one entry
|
|
1469
|
+
# expanded at most (guards, not limits any real entry reaches).
|
|
1470
|
+
MAX_BRACE_ALTERNATIVES = 256
|
|
1471
|
+
MAX_BRACE_GROUPS = 64
|
|
1472
|
+
# ccc reads settings.yml on the machine this helper runs on, and its glob
|
|
1473
|
+
# library reads a backslash as `/` on Windows and as an escape elsewhere.
|
|
1474
|
+
CCC_BACKSLASH_IS_SLASH = os.name == "nt"
|
|
1475
|
+
|
|
1476
|
+
|
|
1477
|
+
def _brace_alternatives(pattern: str) -> list[str]:
|
|
1478
|
+
"""Expand the `{a,b}` groups of a glob as ccc does before its prefix check.
|
|
1479
|
+
|
|
1480
|
+
Nested groups recurse, braces and commas inside `[...]` are literal, and
|
|
1481
|
+
an unbalanced group leaves the pattern as it is. A pattern with more
|
|
1482
|
+
than MAX_BRACE_GROUPS `{` is left as it is too, so the recursion stays
|
|
1483
|
+
shallow whatever the entry holds.
|
|
1484
|
+
"""
|
|
1485
|
+
if pattern.count("{") > MAX_BRACE_GROUPS:
|
|
1486
|
+
return [pattern]
|
|
1487
|
+
in_class = False
|
|
1488
|
+
for i, ch in enumerate(pattern):
|
|
1489
|
+
if ch == "[":
|
|
1490
|
+
in_class = True
|
|
1491
|
+
elif ch == "]":
|
|
1492
|
+
in_class = False
|
|
1493
|
+
elif ch == "{" and not in_class:
|
|
1494
|
+
depth, inner_class, start, alternatives, j = 1, False, i + 1, [], i + 1
|
|
1495
|
+
while j < len(pattern) and depth:
|
|
1496
|
+
c = pattern[j]
|
|
1497
|
+
if c == "[":
|
|
1498
|
+
inner_class = True
|
|
1499
|
+
elif c == "]":
|
|
1500
|
+
inner_class = False
|
|
1501
|
+
elif c == "{" and not inner_class:
|
|
1502
|
+
depth += 1
|
|
1503
|
+
elif c == "}" and not inner_class:
|
|
1504
|
+
depth -= 1
|
|
1505
|
+
if depth == 0:
|
|
1506
|
+
alternatives.append(pattern[start:j])
|
|
1507
|
+
elif c == "," and not inner_class and depth == 1:
|
|
1508
|
+
alternatives.append(pattern[start:j])
|
|
1509
|
+
start = j + 1
|
|
1510
|
+
j += 1
|
|
1511
|
+
if depth:
|
|
1512
|
+
return [pattern]
|
|
1513
|
+
out: list[str] = []
|
|
1514
|
+
for alternative in alternatives:
|
|
1515
|
+
out.extend(_brace_alternatives(pattern[:i] + alternative + pattern[j:]))
|
|
1516
|
+
if len(out) >= MAX_BRACE_ALTERNATIVES:
|
|
1517
|
+
return out[:MAX_BRACE_ALTERNATIVES]
|
|
1518
|
+
return out
|
|
1519
|
+
return [pattern]
|
|
1520
|
+
|
|
1521
|
+
|
|
1522
|
+
def _ccc_glob_regex(glob: str, backslash_escape: bool):
|
|
1523
|
+
"""A regex for one brace-free glob as ccc's glob library reads it, or None.
|
|
1524
|
+
|
|
1525
|
+
`*` and `?` also match `/`; `**/` at the start, `/**/` inside and `/**`
|
|
1526
|
+
at the end span whole directories; `[...]` and `[!...]` or `[^...]` are
|
|
1527
|
+
classes. A backslash escapes the next character when `backslash_escape`
|
|
1528
|
+
is set (POSIX); otherwise the caller has already turned it into `/`
|
|
1529
|
+
(Windows).
|
|
264
1530
|
"""
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
if
|
|
1531
|
+
collapsed = None
|
|
1532
|
+
while collapsed != glob: # `**/**` spans what one `**` spans
|
|
1533
|
+
collapsed = glob
|
|
1534
|
+
glob = glob.replace("/**/**", "/**")
|
|
1535
|
+
if glob.startswith("**/**"):
|
|
1536
|
+
glob = glob[3:]
|
|
1537
|
+
if glob in ("**", "**/"):
|
|
1538
|
+
return re.compile(".*", re.S)
|
|
1539
|
+
out, i, n = [], 0, len(glob)
|
|
1540
|
+
if glob.startswith("**/"):
|
|
1541
|
+
out.append("(?:.*/)?")
|
|
1542
|
+
i = 3
|
|
1543
|
+
while i < n:
|
|
1544
|
+
ch = glob[i]
|
|
1545
|
+
if glob.startswith("/**/", i):
|
|
1546
|
+
out.append("(?:/|/.*/)")
|
|
1547
|
+
i += 4
|
|
1548
|
+
elif glob.startswith("/**", i) and i + 3 == n:
|
|
1549
|
+
out.append("/.*")
|
|
1550
|
+
i += 3
|
|
1551
|
+
elif ch == "*":
|
|
1552
|
+
while i < n and glob[i] == "*":
|
|
1553
|
+
i += 1
|
|
1554
|
+
out.append(".*")
|
|
1555
|
+
elif ch == "?":
|
|
1556
|
+
out.append(".")
|
|
1557
|
+
i += 1
|
|
1558
|
+
elif ch == "[":
|
|
1559
|
+
j = i + 1
|
|
1560
|
+
negate = j < n and glob[j] in "!^"
|
|
1561
|
+
if negate:
|
|
1562
|
+
j += 1
|
|
1563
|
+
k = j + 1 if j < n and glob[j] == "]" else j
|
|
1564
|
+
while k < n and glob[k] != "]":
|
|
1565
|
+
k += 1
|
|
1566
|
+
if k >= n:
|
|
1567
|
+
return None
|
|
1568
|
+
body = "".join("\\" + c if c in "\\^[]" else c for c in glob[j:k])
|
|
1569
|
+
out.append(f"[{'^' if negate else ''}{body}]")
|
|
1570
|
+
i = k + 1
|
|
1571
|
+
elif ch == "\\" and backslash_escape:
|
|
1572
|
+
if i + 1 >= n:
|
|
1573
|
+
return None
|
|
1574
|
+
out.append(re.escape(glob[i + 1]))
|
|
1575
|
+
i += 2
|
|
1576
|
+
else:
|
|
1577
|
+
out.append(re.escape(ch))
|
|
1578
|
+
i += 1
|
|
1579
|
+
try:
|
|
1580
|
+
return re.compile("".join(out), re.S)
|
|
1581
|
+
except re.error:
|
|
1582
|
+
return None
|
|
1583
|
+
|
|
1584
|
+
|
|
1585
|
+
def _ccc_glob(glob: str, backslash_is_slash: bool):
|
|
1586
|
+
"""`_ccc_glob_regex` for one brace-free glob, its backslashes read as ccc
|
|
1587
|
+
reads them on Windows (`/`) or elsewhere (an escape)."""
|
|
1588
|
+
if backslash_is_slash:
|
|
1589
|
+
return _ccc_glob_regex(glob.replace("\\", "/"), False)
|
|
1590
|
+
return _ccc_glob_regex(glob, True)
|
|
1591
|
+
|
|
1592
|
+
|
|
1593
|
+
@functools.lru_cache(maxsize=512)
|
|
1594
|
+
def _negation_forms(body: str, backslash_is_slash: bool) -> tuple[tuple[str, ...], tuple]:
|
|
1595
|
+
"""The brace alternatives of a `!` entry's body, and their regexes."""
|
|
1596
|
+
alternatives = tuple(_brace_alternatives(body))
|
|
1597
|
+
globs = (_ccc_glob(alt, backslash_is_slash) for alt in alternatives)
|
|
1598
|
+
return alternatives, tuple(rx for rx in globs if rx is not None)
|
|
1599
|
+
|
|
1600
|
+
|
|
1601
|
+
def _negation_reopens(entry: str, target: str, backslash_is_slash: bool | None = None) -> bool:
|
|
1602
|
+
"""True when the `!` entry makes ccc walk into or index `target` again.
|
|
1603
|
+
|
|
1604
|
+
ccc keeps an excluded path when a `!` entry matches it, or matches the
|
|
1605
|
+
child it probes the path with (`target/__probe__`, so in practice a
|
|
1606
|
+
wildcard child), or when one of the entry's brace alternatives starts
|
|
1607
|
+
with `target/`: a directory on the way to a re-included path is walked.
|
|
1608
|
+
The glob is read as ccc's glob library reads it on this machine
|
|
1609
|
+
(CCC_BACKSLASH_IS_SLASH unless `backslash_is_slash` is given).
|
|
1610
|
+
"""
|
|
1611
|
+
if backslash_is_slash is None:
|
|
1612
|
+
backslash_is_slash = CCC_BACKSLASH_IS_SLASH
|
|
1613
|
+
alternatives, regexes = _negation_forms(entry[1:], backslash_is_slash)
|
|
1614
|
+
if any(alt.startswith(target + "/") for alt in alternatives):
|
|
1615
|
+
return True
|
|
1616
|
+
probe = f"{target}/{CCC_DIR_PROBE}"
|
|
1617
|
+
return any(rx.fullmatch(target) or rx.fullmatch(probe) for rx in regexes)
|
|
1618
|
+
|
|
1619
|
+
|
|
1620
|
+
def _pattern_path(pattern: str) -> str | None:
|
|
1621
|
+
"""The path a pattern SKF wrote names, or None for a real glob.
|
|
1622
|
+
|
|
1623
|
+
Undoes ccc_literal's one-character classes; a `*` left over marks a
|
|
1624
|
+
result or report family, which a `!` entry cannot re-open as a folder.
|
|
1625
|
+
"""
|
|
1626
|
+
out, i = [], 0
|
|
1627
|
+
while i < len(pattern):
|
|
1628
|
+
ch = pattern[i]
|
|
1629
|
+
if (ch == "[" and i + 2 < len(pattern) and pattern[i + 2] == "]"
|
|
1630
|
+
and pattern[i + 1] in CCC_CLASS_ESCAPES):
|
|
1631
|
+
out.append(pattern[i + 1])
|
|
1632
|
+
i += 3
|
|
270
1633
|
continue
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
existing_excludes = [str(p) for p in existing_excludes]
|
|
310
|
-
|
|
311
|
-
merged, newly_added = merge_patterns(existing_excludes, patterns_to_add)
|
|
312
|
-
patterns_already_present = len(patterns_to_add) - len(newly_added)
|
|
313
|
-
|
|
314
|
-
if not newly_added:
|
|
315
|
-
# Idempotent re-run — no write, no mtime change.
|
|
316
|
-
_ok({
|
|
317
|
-
"settings_yml_existed": existed,
|
|
318
|
-
"settings_yml_path": str(target),
|
|
319
|
-
"patterns_added": 0,
|
|
320
|
-
"patterns_added_list": [],
|
|
321
|
-
"patterns_already_present": patterns_already_present,
|
|
322
|
-
"effective_patterns": sorted(set(patterns_to_add)),
|
|
323
|
-
"written": False,
|
|
324
|
-
"warnings": warnings,
|
|
325
|
-
})
|
|
326
|
-
return
|
|
1634
|
+
if ch in CCC_CLASS_ESCAPES:
|
|
1635
|
+
return None
|
|
1636
|
+
out.append(ch)
|
|
1637
|
+
i += 1
|
|
1638
|
+
return "".join(out)
|
|
1639
|
+
|
|
1640
|
+
|
|
1641
|
+
def _negation_warning(key: str, value: str, entries: list[str], whole: bool, kinds,
|
|
1642
|
+
everything: bool = True) -> str:
|
|
1643
|
+
shown = _sample(entries) + (
|
|
1644
|
+
f" and {len(entries) - SAMPLE_SIZE} more" if len(entries) > SAMPLE_SIZE else "")
|
|
1645
|
+
it = "that entry" if len(entries) == 1 else "those entries"
|
|
1646
|
+
if whole:
|
|
1647
|
+
twice = (", twice for each skill through its version folder and its active link"
|
|
1648
|
+
if "skills" in kinds else "")
|
|
1649
|
+
always = next((part for part in PurePosixPath(value).parts
|
|
1650
|
+
if f"**/{part}" in ALWAYS_INCLUDE), None)
|
|
1651
|
+
own = (
|
|
1652
|
+
f"SKF always keeps {always} out of ccc, so keep content you want searchable "
|
|
1653
|
+
f"outside it"
|
|
1654
|
+
) if always else (
|
|
1655
|
+
f"To keep content of your own in {value} searchable, put it in a folder of its "
|
|
1656
|
+
f"own there and re-run /skf-setup; SKF then excludes its entries one by one and "
|
|
1657
|
+
f"leaves yours indexed"
|
|
1658
|
+
)
|
|
1659
|
+
amount = "all" if everything else "some"
|
|
1660
|
+
return (
|
|
1661
|
+
f"{key} {value} is excluded from ccc as one folder, but exclude_patterns in "
|
|
1662
|
+
f".cocoindex_code/settings.yml also lists {shown}: ccc applies a ! entry against "
|
|
1663
|
+
f"every exclusion, so {amount} SKF output in {value} is indexed again{twice}. "
|
|
1664
|
+
f"Remove {it}; SKF output never needs one. {own}"
|
|
1665
|
+
)
|
|
1666
|
+
return (
|
|
1667
|
+
f"{key} {value} is excluded from ccc one SKF entry at a time, but exclude_patterns "
|
|
1668
|
+
f"in .cocoindex_code/settings.yml also lists {shown}: ccc applies a ! entry against "
|
|
1669
|
+
f"every exclusion, so some SKF output there is indexed again. Remove {it}; your own "
|
|
1670
|
+
f"content in {value} stays indexed without it"
|
|
1671
|
+
)
|
|
327
1672
|
|
|
328
|
-
data["exclude_patterns"] = merged
|
|
329
|
-
rendered = render_settings_yml(data, None)
|
|
330
|
-
_atomic_write(target, rendered)
|
|
331
1673
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
1674
|
+
def _negation_warnings(root: Path, merged: list[str], by_value: dict, verdicts: dict,
|
|
1675
|
+
value_patterns: dict, listings: dict) -> list[str]:
|
|
1676
|
+
"""One warning per folder a user `!` entry brings SKF output back into ccc for.
|
|
1677
|
+
|
|
1678
|
+
Only bare and per-entry verdicts are checked, against `merged` (the
|
|
1679
|
+
exclude_patterns this run writes). A pattern counts when ccc walks down
|
|
1680
|
+
to a file of SKF output below it that git lists (`listings` holds the
|
|
1681
|
+
verdict loop's (paths, scan) per folder; a folder inside an
|
|
1682
|
+
always-excluded one is listed here), or, with no such file, into the
|
|
1683
|
+
child it probes the pattern with, as output SKF writes there later
|
|
1684
|
+
would be. The `!` entries that re-open a pattern that counts are named;
|
|
1685
|
+
the bare value's warning says all SKF output is back only when every
|
|
1686
|
+
SKF entry there is reached.
|
|
1687
|
+
"""
|
|
1688
|
+
negations = [e for e in merged if e.startswith("!")]
|
|
1689
|
+
if not negations:
|
|
1690
|
+
return []
|
|
1691
|
+
slash = CCC_BACKSLASH_IS_SLASH
|
|
1692
|
+
# A pattern with no glob character left (SKF's own, mostly) matches one path.
|
|
1693
|
+
literal = {path for path in (_pattern_path(p) for p in merged
|
|
1694
|
+
if not p.startswith("!") and "\\" not in p) if path}
|
|
1695
|
+
globs = [_ccc_glob(alt, slash) for p in merged
|
|
1696
|
+
if not p.startswith("!") and ("\\" in p or _pattern_path(p) is None)
|
|
1697
|
+
for alt in _brace_alternatives(p)]
|
|
1698
|
+
excludes = [rx for rx in globs if rx is not None]
|
|
1699
|
+
kept: dict[str, bool] = {}
|
|
1700
|
+
|
|
1701
|
+
def walked(path: str) -> bool:
|
|
1702
|
+
"""True when ccc walks every directory down to `path`, and `path` itself:
|
|
1703
|
+
each is matched by no exclude pattern, or re-opened by a `!` entry."""
|
|
1704
|
+
parts = PurePosixPath(path).parts
|
|
1705
|
+
for depth in range(1, len(parts) + 1):
|
|
1706
|
+
step = "/".join(parts[:depth])
|
|
1707
|
+
if step not in kept:
|
|
1708
|
+
kept[step] = ((step not in literal
|
|
1709
|
+
and not any(rx.fullmatch(step) for rx in excludes))
|
|
1710
|
+
or any(_negation_reopens(n, step, slash) for n in negations))
|
|
1711
|
+
if not kept[step]:
|
|
1712
|
+
return False
|
|
1713
|
+
return True
|
|
1714
|
+
|
|
1715
|
+
out = []
|
|
1716
|
+
for value, entries in by_value.items():
|
|
1717
|
+
bare = verdicts[value] == "bare"
|
|
1718
|
+
if not bare and verdicts[value] != "entries":
|
|
1719
|
+
continue
|
|
1720
|
+
kinds = {kind for _, kind in entries}
|
|
1721
|
+
if value in listings:
|
|
1722
|
+
paths, scan = listings[value]
|
|
1723
|
+
else:
|
|
1724
|
+
paths = list_folder_paths(root, value)[0]
|
|
1725
|
+
others = [v for v in by_value if v != value]
|
|
1726
|
+
scan = classify_folder(root, value, kinds, paths, others) if paths else None
|
|
1727
|
+
owned = set(scan.owned_groups) | set(scan.owned_files) if scan else set()
|
|
1728
|
+
files: dict[str, list[str]] = {} # the listed files under each first-level entry
|
|
1729
|
+
for rel in paths or []:
|
|
1730
|
+
if not bare or rel[0] in owned:
|
|
1731
|
+
files.setdefault(rel[0], []).append("/".join((value, *rel)))
|
|
1732
|
+
reached, everything = [], True
|
|
1733
|
+
for target in filter(None, map(_pattern_path, value_patterns[value])):
|
|
1734
|
+
if target == value:
|
|
1735
|
+
hits = [any(map(walked, below)) for below in files.values()]
|
|
1736
|
+
else: # `{value}/<entry>`
|
|
1737
|
+
below = files.get(target[len(value) + 1:], [])
|
|
1738
|
+
hits = [any(map(walked, below))] if below else []
|
|
1739
|
+
if not hits:
|
|
1740
|
+
hits = [walked(f"{target}/{CCC_DIR_PROBE}")]
|
|
1741
|
+
everything = everything and all(hits)
|
|
1742
|
+
if any(hits):
|
|
1743
|
+
reached.append(target)
|
|
1744
|
+
cancelling = [e for e in negations
|
|
1745
|
+
if any(_negation_reopens(e, t, slash) for t in reached)]
|
|
1746
|
+
if cancelling:
|
|
1747
|
+
out.append(_negation_warning(" and ".join(k for k, _ in entries), value,
|
|
1748
|
+
cancelling, bare, kinds, everything))
|
|
1749
|
+
return out
|
|
1750
|
+
|
|
1751
|
+
|
|
1752
|
+
def _list_dir(path: Path) -> list[str] | None:
|
|
1753
|
+
"""The names in `path` as the OS reports them (what ccc matches), or None."""
|
|
1754
|
+
try:
|
|
1755
|
+
return os.listdir(path)
|
|
1756
|
+
except OSError:
|
|
1757
|
+
return None
|
|
1758
|
+
|
|
1759
|
+
|
|
1760
|
+
def _contains(outer: str, inner: str) -> bool:
|
|
1761
|
+
"""True when folder `inner` sits below folder `outer`."""
|
|
1762
|
+
o, i = PurePosixPath(outer).parts, PurePosixPath(inner).parts
|
|
1763
|
+
return i[:len(o)] == o and len(i) > len(o)
|
|
1764
|
+
|
|
1765
|
+
|
|
1766
|
+
def _stale_bare_values(existing: list[str], record: list[str], folders: list[str]) -> set[str]:
|
|
1767
|
+
"""Bare folder values SKF wrote after its record was last saved.
|
|
1768
|
+
|
|
1769
|
+
Setup writes settings.yml before it saves the record, so a run stopped in
|
|
1770
|
+
between leaves a bare value the record does not hold. When the record
|
|
1771
|
+
holds `{value}/...` per-entry patterns for a folder and none of them is
|
|
1772
|
+
still in settings.yml, SKF swapped them for the bare value itself: that
|
|
1773
|
+
value is SKF's, not the user's. Patterns of another configured folder
|
|
1774
|
+
inside this one do not count.
|
|
1775
|
+
"""
|
|
1776
|
+
stale: set[str] = set()
|
|
1777
|
+
for value in folders:
|
|
1778
|
+
if value not in existing or value in record:
|
|
1779
|
+
continue
|
|
1780
|
+
inner = [v for v in folders if _contains(value, v)]
|
|
1781
|
+
entries = [r for r in record if r.startswith(value + "/")
|
|
1782
|
+
and not any(r == v or r.startswith(v + "/") for v in inner)]
|
|
1783
|
+
if entries and not set(entries) & set(existing):
|
|
1784
|
+
stale.add(value)
|
|
1785
|
+
return stale
|
|
1786
|
+
|
|
1787
|
+
|
|
1788
|
+
def _reconcile(root: Path, target: Path, backup: Path, data: dict,
|
|
1789
|
+
values: dict[str, tuple[str, str]], record: list[str], prior_exists: bool,
|
|
1790
|
+
prune_allowed: bool, state: dict, warnings: list[str]) -> None:
|
|
1791
|
+
"""Give each folder a verdict, merge and prune; write settings.yml when it changed.
|
|
1792
|
+
|
|
1793
|
+
Verdicts and record trust follow the module docstring. Folders are
|
|
1794
|
+
checked deepest first, so a folder holding another configured folder
|
|
1795
|
+
knows that folder's verdict: when it is `entries` or `left_out`, the
|
|
1796
|
+
outer folder uses per-entry patterns so its bare value never covers
|
|
1797
|
+
content left indexed on purpose.
|
|
1798
|
+
"""
|
|
1799
|
+
existing = [str(p) for p in (data.get("exclude_patterns") or [])]
|
|
1800
|
+
# Legacy: a folder-free record (at most the always-included patterns)
|
|
1801
|
+
# that is empty or sits next to a `**/<value>` form predates the
|
|
1802
|
+
# anchored forms, so those forms count as owned and migrate. A
|
|
1803
|
+
# folder-free record without them comes from a run whose folders were
|
|
1804
|
+
# all user-excluded or left out, and is trusted like any other.
|
|
1805
|
+
folder_free = prior_exists and not (set(record) - set(ALWAYS_INCLUDE))
|
|
1806
|
+
legacy_forms = {f"**/{v}" for v, _k in values.values()} & set(existing)
|
|
1807
|
+
legacy = folder_free and (not record or bool(legacy_forms))
|
|
1808
|
+
owned_prior = set(record) | (legacy_forms if legacy else set())
|
|
1809
|
+
trusted = prior_exists and not legacy
|
|
1810
|
+
owned_prior |= _stale_bare_values(existing, record, [v for v, _k in values.values()])
|
|
1811
|
+
state["legacy"] = legacy
|
|
1812
|
+
hidden_neutral = "**/.*" in existing and not any(e.startswith("!") for e in existing)
|
|
1813
|
+
by_value: dict[str, list[tuple[str, str]]] = {}
|
|
1814
|
+
for key, (value, kind) in values.items():
|
|
1815
|
+
by_value.setdefault(value, []).append((key, kind))
|
|
1816
|
+
verdicts: dict[str, str] = {}
|
|
1817
|
+
value_patterns: dict[str, list[str]] = {}
|
|
1818
|
+
listings: dict[str, tuple] = {}
|
|
1819
|
+
for value in sorted(by_value, key=lambda v: -len(PurePosixPath(v).parts)):
|
|
1820
|
+
entries = by_value[value]
|
|
1821
|
+
keys = " and ".join(key for key, _kind in entries)
|
|
1822
|
+
if trusted and value in existing and value not in owned_prior:
|
|
1823
|
+
verdicts[value], value_patterns[value] = "user", []
|
|
1824
|
+
continue
|
|
1825
|
+
if _inside_always_excluded(value):
|
|
1826
|
+
verdicts[value], value_patterns[value] = "bare", [value]
|
|
1827
|
+
continue
|
|
1828
|
+
paths, problem = list_folder_paths(root, value)
|
|
1829
|
+
if problem:
|
|
1830
|
+
warnings.append(f"{problem} for {keys}; collision check skipped")
|
|
1831
|
+
others = [v for v in by_value if v != value]
|
|
1832
|
+
kinds = {kind for _key, kind in entries}
|
|
1833
|
+
scan = (classify_folder(root, value, kinds, paths, others, hidden_neutral)
|
|
1834
|
+
if paths else None)
|
|
1835
|
+
listings[value] = paths, scan
|
|
1836
|
+
deferred = any(_contains(value, v) and verdicts.get(v) in ("entries", "left_out")
|
|
1837
|
+
for v in others)
|
|
1838
|
+
if (scan is not None and (scan.foreign or deferred)
|
|
1839
|
+
and value in existing and value not in owned_prior):
|
|
1840
|
+
# No trusted record says who added the bare value: keep it, and
|
|
1841
|
+
# tell the user once (the next run has a trusted record).
|
|
1842
|
+
if scan.foreign:
|
|
1843
|
+
warnings.append(_unowned_entry_warning(keys, value, scan.foreign))
|
|
1844
|
+
verdicts[value], value_patterns[value] = "user", []
|
|
1845
|
+
continue
|
|
1846
|
+
if scan is not None and scan.foreign and not scan.has_output:
|
|
1847
|
+
warnings.append(_collision_warning(keys, value, scan.foreign, kinds))
|
|
1848
|
+
verdicts[value], value_patterns[value] = "left_out", []
|
|
1849
|
+
continue
|
|
1850
|
+
if scan is not None and (scan.foreign or deferred):
|
|
1851
|
+
patterns, skipped = entry_patterns(value, kinds, scan.owned_groups,
|
|
1852
|
+
scan.owned_files, _list_dir(root / value),
|
|
1853
|
+
scan.listed)
|
|
1854
|
+
verdicts[value], value_patterns[value] = "entries", patterns
|
|
1855
|
+
if scan.foreign:
|
|
1856
|
+
warnings.append(_shared_folder_note(keys, value, scan.foreign, kinds))
|
|
1857
|
+
if skipped:
|
|
1858
|
+
warnings.append(_unwritable_warning(keys, value, skipped))
|
|
1859
|
+
continue
|
|
1860
|
+
verdicts[value], value_patterns[value] = "bare", [value]
|
|
1861
|
+
produced = list(ALWAYS_INCLUDE)
|
|
1862
|
+
for value in by_value:
|
|
1863
|
+
for p in value_patterns[value]:
|
|
1864
|
+
if p not in produced:
|
|
1865
|
+
produced.append(p)
|
|
1866
|
+
|
|
1867
|
+
merged, added, removed = plan_exclusions(existing, produced, owned_prior, prune_allowed)
|
|
1868
|
+
warnings.extend(_negation_warnings(root, merged, by_value, verdicts, value_patterns,
|
|
1869
|
+
listings))
|
|
1870
|
+
blocked = [] if prune_allowed else sorted((owned_prior - set(produced)) & set(existing))
|
|
1871
|
+
if blocked:
|
|
1872
|
+
shown = ", ".join(blocked[:SAMPLE_SIZE]) + (
|
|
1873
|
+
f" and {len(blocked) - SAMPLE_SIZE} more" if len(blocked) > SAMPLE_SIZE else "")
|
|
1874
|
+
warnings.append(
|
|
1875
|
+
"kept previously recorded SKF exclusions (" + shown + ") because a "
|
|
1876
|
+
"folder value was refused; fix it and re-run /skf-setup to remove patterns SKF "
|
|
1877
|
+
"no longer needs"
|
|
1878
|
+
)
|
|
1879
|
+
|
|
1880
|
+
if merged != existing or state["written"]:
|
|
1881
|
+
data["exclude_patterns"] = merged
|
|
1882
|
+
# ASCII-only output (non-ASCII escaped), as ccc writes it: ccc reads
|
|
1883
|
+
# settings.yml with the platform default encoding.
|
|
1884
|
+
_atomic_write(target, yaml.safe_dump(data, default_flow_style=False, sort_keys=False))
|
|
1885
|
+
state["written"] = True
|
|
1886
|
+
if backup.is_file():
|
|
1887
|
+
try:
|
|
1888
|
+
backup.unlink()
|
|
1889
|
+
except OSError as e:
|
|
1890
|
+
raise HelperError(2, f"could not remove {backup}: {e}")
|
|
1891
|
+
state.update(added=added, removed=removed,
|
|
1892
|
+
present=len(produced) - len(added),
|
|
1893
|
+
effective=sorted(set(produced) | set(blocked)))
|
|
1894
|
+
|
|
1895
|
+
|
|
1896
|
+
def run_merge(project_root, skills_output_folder, forge_data_folder, prior_state_from=None,
|
|
1897
|
+
index_fresh: bool = False, skip_index: bool = False,
|
|
1898
|
+
allow_ccc_init: bool = True, defer_index: bool = False) -> dict:
|
|
1899
|
+
"""Prepare settings.yml, reconcile the SKF patterns, decide the index action.
|
|
1900
|
+
|
|
1901
|
+
Returns the v2 payload documented in the module docstring. Raises
|
|
1902
|
+
HelperError for the exit-1 and exit-2 cases.
|
|
1903
|
+
"""
|
|
1904
|
+
root = Path(project_root)
|
|
1905
|
+
target = root / SETTINGS_DIR / SETTINGS_NAME
|
|
1906
|
+
backup = target.with_name(BACKUP_NAME)
|
|
1907
|
+
gitignore = root / ".gitignore"
|
|
1908
|
+
warnings: list[str] = []
|
|
1909
|
+
gitignore_before = _read_bytes_or_none(gitignore)
|
|
1910
|
+
|
|
1911
|
+
# Crash recovery: a leftover backup is the pre-repair original.
|
|
1912
|
+
if backup.is_file():
|
|
1913
|
+
try:
|
|
1914
|
+
os.replace(backup, target)
|
|
1915
|
+
except OSError as e:
|
|
1916
|
+
raise HelperError(2, f"could not restore {target} from {backup.name}: {e}")
|
|
1917
|
+
warnings.append("restored .cocoindex_code/settings.yml from an interrupted SKF repair")
|
|
1918
|
+
|
|
1919
|
+
values: dict[str, tuple[str, str]] = {}
|
|
1920
|
+
prune_allowed = True
|
|
1921
|
+
for (key, kind), raw in zip(FOLDER_KEYS, (skills_output_folder, forge_data_folder)):
|
|
1922
|
+
value, warn = validate_config_value(key, resolve_repo_relative(raw, root))
|
|
1923
|
+
if value is None:
|
|
1924
|
+
prune_allowed = False
|
|
1925
|
+
warnings.append(warn)
|
|
1926
|
+
else:
|
|
1927
|
+
values[key] = (value, kind)
|
|
1928
|
+
|
|
1929
|
+
record = read_owned_record(prior_state_from, warnings)
|
|
1930
|
+
# With no forge-tier.yaml at all, SKF never finished a setup here and
|
|
1931
|
+
# owns nothing yet; _reconcile decides whether the record is legacy.
|
|
1932
|
+
prior_exists = prior_state_from is not None and Path(prior_state_from).is_file()
|
|
1933
|
+
|
|
1934
|
+
state = {
|
|
1935
|
+
"existed": target.is_file(),
|
|
1936
|
+
"ccc_init": "not_needed",
|
|
1937
|
+
"ready": True,
|
|
1938
|
+
"reason": None,
|
|
1939
|
+
"written": False,
|
|
1940
|
+
"added": [],
|
|
1941
|
+
"removed": [],
|
|
1942
|
+
"present": 0,
|
|
1943
|
+
"effective": None,
|
|
1944
|
+
"index_block": None,
|
|
1945
|
+
"legacy": False,
|
|
1946
|
+
}
|
|
1947
|
+
|
|
1948
|
+
data = _prepare_settings(root, target, backup, allow_ccc_init, state, warnings)
|
|
1949
|
+
if data is not None:
|
|
1950
|
+
_reconcile(root, target, backup, data, values, record, prior_exists, prune_allowed,
|
|
1951
|
+
state, warnings)
|
|
1952
|
+
if state["legacy"] and not prune_allowed:
|
|
1953
|
+
# Keep the folder-free record so the next valid run can still
|
|
1954
|
+
# migrate the legacy `**/<value>` forms of the refused value.
|
|
1955
|
+
state["effective"] = None
|
|
1956
|
+
|
|
1957
|
+
if state["ready"] and target.is_file():
|
|
1958
|
+
res = _git(root, "check-ignore", "-q", "--no-index", "--",
|
|
1959
|
+
".cocoindex_code/target_sqlite.db")
|
|
1960
|
+
if res is not None and res[0] == 1:
|
|
1961
|
+
warnings.append(
|
|
1962
|
+
"/.cocoindex_code/ is not gitignored here, so git lists the ccc index "
|
|
1963
|
+
"database (tens of MB) as untracked; add /.cocoindex_code/ to "
|
|
1964
|
+
"{project-root}/.gitignore"
|
|
1965
|
+
)
|
|
1966
|
+
|
|
1967
|
+
action = decide_index_action(state["ready"], skip_index, state["written"], index_fresh,
|
|
1968
|
+
usable=state["index_block"] is None, defer=defer_index)
|
|
1969
|
+
reason = state["reason"] or (state["index_block"] if action == "fail" else None)
|
|
1970
|
+
if action == "skip" and state["written"]:
|
|
1971
|
+
warnings.append(
|
|
1972
|
+
f"--ccc-skip-index: settings.yml exclusions changed (+{len(state['added'])}, "
|
|
1973
|
+
f"-{len(state['removed'])}) but the index was not rebuilt; run ccc index in "
|
|
1974
|
+
"{project-root} or re-run /skf-setup without --ccc-skip-index"
|
|
1975
|
+
)
|
|
1976
|
+
|
|
1977
|
+
return {
|
|
1978
|
+
"status": "ok",
|
|
1979
|
+
"version": "v2",
|
|
1980
|
+
"settings_yml_path": str(target),
|
|
1981
|
+
"settings_yml_existed": state["existed"],
|
|
1982
|
+
"ccc_init": state["ccc_init"],
|
|
1983
|
+
"settings_ready": state["ready"],
|
|
1984
|
+
"not_ready_reason": _payload_safe(reason) if reason else None,
|
|
1985
|
+
"patterns_added": len(state["added"]),
|
|
1986
|
+
"patterns_added_list": state["added"],
|
|
1987
|
+
"patterns_removed": len(state["removed"]),
|
|
1988
|
+
"patterns_removed_list": state["removed"],
|
|
1989
|
+
"patterns_already_present": state["present"],
|
|
1990
|
+
"effective_patterns": state["effective"],
|
|
1991
|
+
"written": state["written"],
|
|
1992
|
+
"gitignore_updated": _read_bytes_or_none(gitignore) != gitignore_before,
|
|
1993
|
+
"index_action": action,
|
|
1994
|
+
"warnings": [_payload_safe(w) for w in warnings],
|
|
1995
|
+
}
|
|
1996
|
+
|
|
1997
|
+
|
|
1998
|
+
# ─── Clone mode ─────────────────────────────────────────────────────────────
|
|
1999
|
+
|
|
2000
|
+
|
|
2001
|
+
def quote_bare_glob_items(text: str) -> tuple[str, int]:
|
|
2002
|
+
"""Single-quote each bare `- *...` list item; return (text, items quoted).
|
|
2003
|
+
|
|
2004
|
+
YAML reads a plain scalar that starts with `*` as an alias, so a
|
|
2005
|
+
`- **/build` item an earlier hand edit wrote stops ccc from loading the
|
|
2006
|
+
file. ccc writes these items quoted, as yaml.safe_dump does. A trailing
|
|
2007
|
+
comment stays a comment, after the closing quote.
|
|
2008
|
+
"""
|
|
2009
|
+
return BARE_GLOB_ITEM_RE.subn(lambda m: f"{m.group(1)}'{m.group(2)}'{m.group(3)}", text)
|
|
2010
|
+
|
|
2011
|
+
|
|
2012
|
+
def _load_clone_settings(path: Path, warnings: list[str]) -> tuple[dict, bool]:
|
|
2013
|
+
"""Parse a clone's settings.yml: (data, repaired).
|
|
2014
|
+
|
|
2015
|
+
A file YAML cannot read is read again with its bare glob items quoted,
|
|
2016
|
+
and `repaired` is then True so the caller writes it back. Raises
|
|
2017
|
+
HelperError(1) when it still cannot be read, is not a mapping, or holds
|
|
2018
|
+
an exclude_patterns or include_patterns that is neither a list nor null.
|
|
2019
|
+
"""
|
|
2020
|
+
try:
|
|
2021
|
+
text = path.read_text(encoding="utf-8")
|
|
2022
|
+
except OSError as e:
|
|
2023
|
+
raise HelperError(1, f"failed to read {path}: {e}")
|
|
2024
|
+
repaired = False
|
|
2025
|
+
try:
|
|
2026
|
+
data = yaml.safe_load(text)
|
|
2027
|
+
except (yaml.YAMLError, ValueError) as e:
|
|
2028
|
+
fixed, count = quote_bare_glob_items(text)
|
|
2029
|
+
try:
|
|
2030
|
+
data = yaml.safe_load(fixed) if count else None
|
|
2031
|
+
except (yaml.YAMLError, ValueError):
|
|
2032
|
+
count = 0
|
|
2033
|
+
if not count:
|
|
2034
|
+
raise HelperError(1, f"failed to parse {path}: {e}")
|
|
2035
|
+
repaired = True
|
|
2036
|
+
warnings.append(f"quoted {_plural(count, 'list item', 'list items')} of settings.yml that YAML "
|
|
2037
|
+
"read as an alias, so ccc can load the file again")
|
|
2038
|
+
if data is None:
|
|
2039
|
+
return {}, repaired
|
|
2040
|
+
if not isinstance(data, dict):
|
|
2041
|
+
raise HelperError(1, f"expected mapping at top of {path}, got {type(data).__name__}")
|
|
2042
|
+
for key in ("exclude_patterns", "include_patterns"):
|
|
2043
|
+
value = data.get(key)
|
|
2044
|
+
if value is not None and not isinstance(value, list):
|
|
2045
|
+
raise HelperError(1, f"{key} in {path} is not a list (got {type(value).__name__})")
|
|
2046
|
+
return data, repaired
|
|
2047
|
+
|
|
2048
|
+
|
|
2049
|
+
def include_covers(pattern, ext: str) -> bool:
|
|
2050
|
+
"""True when an include_patterns entry matches a nested `*.<ext>` file as ccc reads it."""
|
|
2051
|
+
if not isinstance(pattern, str):
|
|
2052
|
+
return False
|
|
2053
|
+
probe = INCLUDE_PROBE.format(ext=ext)
|
|
2054
|
+
for alternative in _brace_alternatives(pattern):
|
|
2055
|
+
rx = _ccc_glob(alternative, CCC_BACKSLASH_IS_SLASH)
|
|
2056
|
+
if rx is not None and rx.fullmatch(probe):
|
|
2057
|
+
return True
|
|
2058
|
+
return False
|
|
2059
|
+
|
|
2060
|
+
|
|
2061
|
+
# ─── Index build (--build-index) ────────────────────────────────────────────
|
|
2062
|
+
|
|
2063
|
+
|
|
2064
|
+
def _run_ccc(root: Path, args: tuple[str, ...], timeout: int) -> tuple[bool, str]:
|
|
2065
|
+
"""Run `ccc <args>` in root. Return (exit code was 0, stdout plus stderr).
|
|
2066
|
+
|
|
2067
|
+
A ccc that cannot be found, started or finished in time returns False
|
|
2068
|
+
with the reason as its output.
|
|
2069
|
+
"""
|
|
2070
|
+
exe = _resolve_outside_cwd("ccc")
|
|
2071
|
+
if exe is None:
|
|
2072
|
+
return False, "ccc was not found on PATH"
|
|
2073
|
+
try:
|
|
2074
|
+
result = subprocess.run(
|
|
2075
|
+
[exe, *args],
|
|
2076
|
+
cwd=str(root),
|
|
2077
|
+
stdin=subprocess.DEVNULL,
|
|
2078
|
+
capture_output=True,
|
|
2079
|
+
env=_child_env(),
|
|
2080
|
+
timeout=timeout,
|
|
2081
|
+
)
|
|
2082
|
+
except subprocess.TimeoutExpired:
|
|
2083
|
+
return False, f"ccc {args[0]} timed out after {timeout}s"
|
|
2084
|
+
except (OSError, ValueError) as e:
|
|
2085
|
+
return False, f"ccc {args[0]} could not start: {e}"
|
|
2086
|
+
output = (result.stdout + result.stderr).decode("utf-8", errors="replace")
|
|
2087
|
+
if result.returncode != 0:
|
|
2088
|
+
return False, f"ccc {args[0]} exited {result.returncode}: {' '.join(output.split())[:300]}"
|
|
2089
|
+
return True, output
|
|
2090
|
+
|
|
2091
|
+
|
|
2092
|
+
def parse_status_file_count(output: str) -> int | None:
|
|
2093
|
+
"""The `Files:` count of `ccc status`, or None when it shows none (no index yet)."""
|
|
2094
|
+
match = CCC_FILES_RE.search(output)
|
|
2095
|
+
return int(match.group(1)) if match else None
|
|
2096
|
+
|
|
2097
|
+
|
|
2098
|
+
def _prior_index(prior_state_from) -> dict:
|
|
2099
|
+
"""`ccc_index` of the prior forge-tier.yaml, {} when there is none to read."""
|
|
2100
|
+
if prior_state_from is None or not Path(prior_state_from).is_file():
|
|
2101
|
+
return {}
|
|
2102
|
+
try:
|
|
2103
|
+
data = yaml.safe_load(Path(prior_state_from).read_text(encoding="utf-8"))
|
|
2104
|
+
except (OSError, ValueError, yaml.YAMLError):
|
|
2105
|
+
return {}
|
|
2106
|
+
ccc = data.get("ccc_index") if isinstance(data, dict) else None
|
|
2107
|
+
return ccc if isinstance(ccc, dict) else {}
|
|
2108
|
+
|
|
2109
|
+
|
|
2110
|
+
def _utc_now() -> str:
|
|
2111
|
+
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
|
2112
|
+
|
|
2113
|
+
|
|
2114
|
+
def build_index(root: Path, indexed_path: str, action: str, not_ready_reason,
|
|
2115
|
+
prior_state_from, warnings: list[str]) -> dict:
|
|
2116
|
+
"""The index state a setup run ends with, after acting on `action`.
|
|
2117
|
+
|
|
2118
|
+
See "Index build" in the module docstring. `warnings` gains a note when
|
|
2119
|
+
a pass is still running after the reruns.
|
|
2120
|
+
"""
|
|
2121
|
+
unset = {"indexed_path": None, "last_indexed": None, "file_count": None, "failed_reason": None}
|
|
2122
|
+
if action in ("skip", "defer"):
|
|
2123
|
+
return {"status": "skipped", **unset}
|
|
2124
|
+
if action == "fail":
|
|
2125
|
+
return {"status": "failed", **unset,
|
|
2126
|
+
"failed_reason": not_ready_reason or "no usable .cocoindex_code/settings.yml"}
|
|
2127
|
+
if action == "keep":
|
|
2128
|
+
prior = _prior_index(prior_state_from)
|
|
2129
|
+
last = prior.get("last_indexed")
|
|
2130
|
+
count = prior.get("file_count")
|
|
2131
|
+
return {"status": "fresh", **unset, "indexed_path": indexed_path,
|
|
2132
|
+
"last_indexed": last.isoformat() if isinstance(last, datetime) else last,
|
|
2133
|
+
"file_count": count if isinstance(count, int) and not isinstance(count, bool) else None}
|
|
2134
|
+
|
|
2135
|
+
def failed(reason: str) -> dict:
|
|
2136
|
+
return {"status": "failed", **unset, "failed_reason": _payload_safe(reason)}
|
|
2137
|
+
|
|
2138
|
+
ok, output = _run_ccc(root, ("index",), CCC_INDEX_TIMEOUT_SEC)
|
|
2139
|
+
if not ok:
|
|
2140
|
+
return failed(output)
|
|
2141
|
+
reruns = 0
|
|
2142
|
+
while True:
|
|
2143
|
+
ok, status = _run_ccc(root, ("status",), CCC_STATUS_TIMEOUT_SEC)
|
|
2144
|
+
if not ok:
|
|
2145
|
+
return failed(status)
|
|
2146
|
+
if CCC_IN_PROGRESS not in status or reruns == CCC_INDEX_RERUNS:
|
|
2147
|
+
break
|
|
2148
|
+
reruns += 1
|
|
2149
|
+
ok, output = _run_ccc(root, ("index",), CCC_INDEX_TIMEOUT_SEC)
|
|
2150
|
+
if not ok:
|
|
2151
|
+
return failed(output)
|
|
2152
|
+
if CCC_IN_PROGRESS in status:
|
|
2153
|
+
warnings.append(f"ccc still reports an indexing pass in progress after {CCC_INDEX_RERUNS} reruns "
|
|
2154
|
+
"of ccc index; the file count may grow")
|
|
2155
|
+
count = parse_status_file_count(status)
|
|
2156
|
+
if count is None:
|
|
2157
|
+
return failed("ccc index finished, but ccc status shows no index")
|
|
2158
|
+
if count == 0:
|
|
2159
|
+
return failed("ccc index finished, but ccc status counts no indexed file")
|
|
2160
|
+
return {"status": "created", **unset, "indexed_path": indexed_path, "last_indexed": _utc_now(),
|
|
2161
|
+
"file_count": count}
|
|
2162
|
+
|
|
2163
|
+
|
|
2164
|
+
def read_config_folders(path: Path) -> tuple[str, str]:
|
|
2165
|
+
"""The raw skills_output_folder and forge_data_folder values of config.yaml."""
|
|
2166
|
+
try:
|
|
2167
|
+
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
2168
|
+
except FileNotFoundError:
|
|
2169
|
+
raise HelperError(1, f"--config does not exist: {path}")
|
|
2170
|
+
except (OSError, ValueError, yaml.YAMLError) as e:
|
|
2171
|
+
raise HelperError(1, f"--config cannot be read as YAML: {path}: {e}")
|
|
2172
|
+
if data is None:
|
|
2173
|
+
data = {}
|
|
2174
|
+
if not isinstance(data, dict):
|
|
2175
|
+
raise HelperError(1, f"--config is not a YAML mapping: {path}")
|
|
2176
|
+
values = (data.get("skills_output_folder"), data.get("forge_data_folder"))
|
|
2177
|
+
return tuple("" if v is None else str(v) for v in values)
|
|
2178
|
+
|
|
2179
|
+
|
|
2180
|
+
def run_clone_merge(clone_root, include_exts=(), allow_ccc_init: bool = True) -> dict:
|
|
2181
|
+
"""Clone mode: prepare a workspace clone's settings.yml and merge into it.
|
|
2182
|
+
|
|
2183
|
+
Returns the clone-mode payload documented in the module docstring.
|
|
2184
|
+
Raises HelperError for the exit-1 and exit-2 cases.
|
|
2185
|
+
"""
|
|
2186
|
+
root = Path(clone_root)
|
|
2187
|
+
target = root / SETTINGS_DIR / SETTINGS_NAME
|
|
2188
|
+
warnings: list[str] = []
|
|
2189
|
+
existed = target.is_file()
|
|
2190
|
+
ccc_init, ready, reason, written = "not_needed", True, None, False
|
|
2191
|
+
added: list[str] = []
|
|
2192
|
+
includes_added: list[str] = []
|
|
2193
|
+
covered: list[str] = []
|
|
2194
|
+
present = 0
|
|
2195
|
+
|
|
2196
|
+
if not existed:
|
|
2197
|
+
if not allow_ccc_init:
|
|
2198
|
+
ready, reason = False, "settings.yml is missing and --no-ccc-init was given"
|
|
2199
|
+
else:
|
|
2200
|
+
_ok, output = run_ccc_init(root)
|
|
2201
|
+
if target.is_file():
|
|
2202
|
+
ccc_init, written = "created", True
|
|
2203
|
+
else:
|
|
2204
|
+
ccc_init, ready = "failed", False
|
|
2205
|
+
reason = ("ccc init did not create .cocoindex_code/settings.yml "
|
|
2206
|
+
f"({_output_hint(output)})")
|
|
2207
|
+
|
|
2208
|
+
if ready:
|
|
2209
|
+
data, repaired = _load_clone_settings(target, warnings)
|
|
2210
|
+
excludes = data.get("exclude_patterns")
|
|
2211
|
+
if isinstance(excludes, list):
|
|
2212
|
+
existing = [str(p) for p in excludes]
|
|
2213
|
+
added = [p for p in CLONE_EXCLUDES if p not in existing]
|
|
2214
|
+
present = len(CLONE_EXCLUDES) - len(added)
|
|
2215
|
+
if added:
|
|
2216
|
+
data["exclude_patterns"] = [*excludes, *added]
|
|
2217
|
+
else:
|
|
2218
|
+
warnings.append("settings.yml has no exclude_patterns list, so SKF added no exclusion: "
|
|
2219
|
+
"a list written from scratch replaces the ccc default exclusions")
|
|
2220
|
+
includes = data.get("include_patterns")
|
|
2221
|
+
if include_exts and isinstance(includes, list):
|
|
2222
|
+
current = list(includes)
|
|
2223
|
+
for ext in dict.fromkeys(include_exts):
|
|
2224
|
+
if any(include_covers(p, ext) for p in current):
|
|
2225
|
+
covered.append(ext)
|
|
2226
|
+
else:
|
|
2227
|
+
current.append(f"**/*.{ext}")
|
|
2228
|
+
includes_added.append(f"**/*.{ext}")
|
|
2229
|
+
if includes_added:
|
|
2230
|
+
data["include_patterns"] = current
|
|
2231
|
+
elif include_exts:
|
|
2232
|
+
warnings.append("settings.yml has no include_patterns list, so SKF added no file type: "
|
|
2233
|
+
"a list written from scratch replaces the ccc default file types")
|
|
2234
|
+
if added or includes_added or repaired:
|
|
2235
|
+
# ASCII-only output (non-ASCII escaped), as ccc writes it.
|
|
2236
|
+
_atomic_write(target, yaml.safe_dump(data, default_flow_style=False, sort_keys=False),
|
|
2237
|
+
unique=True)
|
|
2238
|
+
written = True
|
|
2239
|
+
|
|
2240
|
+
return {
|
|
2241
|
+
"status": "ok",
|
|
2242
|
+
"version": "v2",
|
|
2243
|
+
"settings_yml_path": str(target),
|
|
2244
|
+
"settings_yml_existed": existed,
|
|
2245
|
+
"ccc_init": ccc_init,
|
|
2246
|
+
"settings_ready": ready,
|
|
2247
|
+
"not_ready_reason": _payload_safe(reason) if reason else None,
|
|
2248
|
+
"patterns_added": len(added),
|
|
2249
|
+
"patterns_added_list": added,
|
|
2250
|
+
"patterns_already_present": present,
|
|
2251
|
+
"includes_added_list": includes_added,
|
|
2252
|
+
"includes_covered_list": covered,
|
|
2253
|
+
"written": written,
|
|
2254
|
+
"warnings": [_payload_safe(w) for w in warnings],
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
|
|
2258
|
+
def _ext_arg(value: str) -> str:
|
|
2259
|
+
"""argparse type for --include-ext: a plain file extension, a leading `.` dropped."""
|
|
2260
|
+
ext = str(value).strip()
|
|
2261
|
+
ext = ext[1:] if ext.startswith(".") else ext
|
|
2262
|
+
if not EXTENSION_RE.match(ext):
|
|
2263
|
+
raise argparse.ArgumentTypeError(f"expected a file extension such as ex or .ex, got {value}")
|
|
2264
|
+
return ext
|
|
2265
|
+
|
|
2266
|
+
|
|
2267
|
+
def _bool_arg(value: str) -> bool:
|
|
2268
|
+
"""argparse type for true|false (case-insensitive); anything else is a usage error."""
|
|
2269
|
+
normalized = str(value).strip().lower()
|
|
2270
|
+
if normalized in ("true", "false"):
|
|
2271
|
+
return normalized == "true"
|
|
2272
|
+
raise argparse.ArgumentTypeError(f"expected true or false, got {value}")
|
|
2273
|
+
|
|
2274
|
+
|
|
2275
|
+
class _JsonErrorParser(argparse.ArgumentParser):
|
|
2276
|
+
"""Report usage errors as the same sanitized stderr JSON as every other exit.
|
|
2277
|
+
|
|
2278
|
+
A caller that reads stderr JSON may forward the message into a
|
|
2279
|
+
single-quoted payload; argparse's own text quotes the bad value.
|
|
2280
|
+
"""
|
|
2281
|
+
|
|
2282
|
+
def error(self, message: str) -> None:
|
|
2283
|
+
_die(2, f"usage error: {message}")
|
|
342
2284
|
|
|
343
2285
|
|
|
344
2286
|
def main() -> None:
|
|
345
|
-
parser =
|
|
346
|
-
description="
|
|
2287
|
+
parser = _JsonErrorParser(
|
|
2288
|
+
description="Prepare .cocoindex_code/settings.yml (running ccc init when needed), "
|
|
2289
|
+
"reconcile the SKF exclusion patterns in it, and decide whether the "
|
|
2290
|
+
"ccc index must be built.",
|
|
347
2291
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
348
2292
|
)
|
|
349
|
-
parser.
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
2293
|
+
roots = parser.add_mutually_exclusive_group(required=True)
|
|
2294
|
+
roots.add_argument(
|
|
2295
|
+
"--project-root",
|
|
2296
|
+
help="Absolute path to the project root. The script reads and writes "
|
|
2297
|
+
"{project-root}/.cocoindex_code/settings.yml and runs ccc init there. "
|
|
2298
|
+
"--build-index records the value as given as the index's indexed_path.",
|
|
2299
|
+
)
|
|
2300
|
+
roots.add_argument(
|
|
2301
|
+
"--clone-root", type=Path,
|
|
2302
|
+
help="Clone mode: absolute path to an SKF workspace clone. The script runs ccc init -f "
|
|
2303
|
+
"there when .cocoindex_code/settings.yml is missing, and merges the standard build "
|
|
2304
|
+
"and dependency exclusions (and the --include-ext file types) into it.",
|
|
353
2305
|
)
|
|
354
2306
|
parser.add_argument(
|
|
355
|
-
"--skills-output-folder", default=
|
|
356
|
-
help="Raw value of skills_output_folder from {project-root}/_bmad/skf/config.yaml
|
|
357
|
-
"
|
|
358
|
-
"
|
|
2307
|
+
"--skills-output-folder", default=None,
|
|
2308
|
+
help="Raw value of skills_output_folder from {project-root}/_bmad/skf/config.yaml, "
|
|
2309
|
+
"forwarded verbatim. Normalized, validated and checked, then excluded as a "
|
|
2310
|
+
"root-anchored pattern, or entry by entry when the folder also holds content "
|
|
2311
|
+
"SKF did not generate. Refused values produce a warning.",
|
|
359
2312
|
)
|
|
360
2313
|
parser.add_argument(
|
|
361
|
-
"--forge-data-folder", default=
|
|
2314
|
+
"--forge-data-folder", default=None,
|
|
362
2315
|
help="Raw value of forge_data_folder from {project-root}/_bmad/skf/config.yaml. "
|
|
363
|
-
"Same
|
|
2316
|
+
"Same handling as --skills-output-folder.",
|
|
2317
|
+
)
|
|
2318
|
+
parser.add_argument(
|
|
2319
|
+
"--config", type=Path, default=None,
|
|
2320
|
+
help="Setup mode, instead of the two folder flags: the _bmad/skf/config.yaml "
|
|
2321
|
+
"to read the raw skills_output_folder and forge_data_folder values from.",
|
|
2322
|
+
)
|
|
2323
|
+
parser.add_argument(
|
|
2324
|
+
"--prior-state-from", type=Path, default=None,
|
|
2325
|
+
help="Path to forge-tier.yaml. Its ccc_index.exclude_patterns is the record of "
|
|
2326
|
+
"the patterns SKF owns, per-entry patterns included; recorded patterns the "
|
|
2327
|
+
"current config no longer produces are removed, and with this record a "
|
|
2328
|
+
"folder already in exclude_patterns that it does not hold is left to the "
|
|
2329
|
+
"user. Missing file: empty record.",
|
|
2330
|
+
)
|
|
2331
|
+
parser.add_argument(
|
|
2332
|
+
"--index-fresh", type=_bool_arg, default=None, metavar="true|false",
|
|
2333
|
+
help="Whether the prior ccc index is still fresh. With true and an unchanged "
|
|
2334
|
+
"settings.yml, index_action is keep. Default false.",
|
|
2335
|
+
)
|
|
2336
|
+
parser.add_argument(
|
|
2337
|
+
"--skip-index", type=_bool_arg, default=None, metavar="true|false",
|
|
2338
|
+
help="Whether the setup run opted out of indexing (--ccc-skip-index). With true, "
|
|
2339
|
+
"a ready settings.yml gives index_action skip. Default false.",
|
|
2340
|
+
)
|
|
2341
|
+
parser.add_argument(
|
|
2342
|
+
"--defer-index", action="store_true",
|
|
2343
|
+
help="Leave a due index build (index_action index) to a later run: index_action is defer "
|
|
2344
|
+
"and the index skipped. A fresh index is still kept. Setup passes it on a "
|
|
2345
|
+
"--require-tier miss.",
|
|
2346
|
+
)
|
|
2347
|
+
parser.add_argument(
|
|
2348
|
+
"--no-ccc-init", action="store_true",
|
|
2349
|
+
help="Never run ccc init (tests and diagnostics). A missing settings.yml is "
|
|
2350
|
+
"reported as not ready; a rebuild is skipped with a warning.",
|
|
2351
|
+
)
|
|
2352
|
+
parser.add_argument(
|
|
2353
|
+
"--build-index", action="store_true",
|
|
2354
|
+
help="Setup mode: act on index_action (run ccc index when it is index) and add "
|
|
2355
|
+
"the index result, stamped by this script, to the output.",
|
|
2356
|
+
)
|
|
2357
|
+
parser.add_argument(
|
|
2358
|
+
"--result-to", type=Path, default=None,
|
|
2359
|
+
help="Also write the JSON result, or the JSON error of a run that fails, to this file.",
|
|
2360
|
+
)
|
|
2361
|
+
parser.add_argument(
|
|
2362
|
+
"--include-ext", type=_ext_arg, action="append", default=[], metavar="EXT",
|
|
2363
|
+
help="Clone mode only, repeatable: a file extension (ex, or .ex) whose files ccc must "
|
|
2364
|
+
"index. Unless an include_patterns entry already matches it, **/*.EXT is appended "
|
|
2365
|
+
"to include_patterns.",
|
|
364
2366
|
)
|
|
365
2367
|
args = parser.parse_args()
|
|
366
2368
|
|
|
367
|
-
|
|
2369
|
+
def fail(code: int, message: str) -> None:
|
|
2370
|
+
if args.result_to is not None:
|
|
2371
|
+
_write_result(args.result_to, {"status": "error", "message": _payload_safe(message)})
|
|
2372
|
+
_die(code, message)
|
|
2373
|
+
|
|
2374
|
+
def done(payload: dict) -> None:
|
|
2375
|
+
if args.result_to is not None:
|
|
2376
|
+
_write_result(args.result_to, payload)
|
|
2377
|
+
print(json.dumps(payload))
|
|
2378
|
+
|
|
2379
|
+
if args.clone_root is not None:
|
|
2380
|
+
# A setup flag is refused whatever its value: `--skip-index false` or an empty folder too.
|
|
2381
|
+
setup_flags = [flag for flag, value in (
|
|
2382
|
+
("--skills-output-folder", args.skills_output_folder),
|
|
2383
|
+
("--forge-data-folder", args.forge_data_folder),
|
|
2384
|
+
("--config", args.config),
|
|
2385
|
+
("--prior-state-from", args.prior_state_from),
|
|
2386
|
+
("--index-fresh", args.index_fresh),
|
|
2387
|
+
("--skip-index", args.skip_index),
|
|
2388
|
+
("--build-index", args.build_index or None),
|
|
2389
|
+
("--defer-index", args.defer_index or None),
|
|
2390
|
+
) if value is not None]
|
|
2391
|
+
if setup_flags:
|
|
2392
|
+
_die(2, f"usage error: --clone-root takes no setup flag ({', '.join(setup_flags)})")
|
|
2393
|
+
if not args.clone_root.is_dir():
|
|
2394
|
+
fail(1, f"--clone-root is not a directory: {args.clone_root}")
|
|
2395
|
+
try:
|
|
2396
|
+
payload = run_clone_merge(args.clone_root, args.include_ext,
|
|
2397
|
+
allow_ccc_init=not args.no_ccc_init)
|
|
2398
|
+
except HelperError as e:
|
|
2399
|
+
fail(e.code, str(e))
|
|
2400
|
+
except Exception as e: # noqa: BLE001 (every failure must stay stderr JSON)
|
|
2401
|
+
fail(2, f"unexpected error: {type(e).__name__}: {e}")
|
|
2402
|
+
done(payload)
|
|
2403
|
+
return
|
|
2404
|
+
if args.include_ext:
|
|
2405
|
+
_die(2, "usage error: --include-ext needs --clone-root")
|
|
2406
|
+
if args.config is not None and (args.skills_output_folder is not None or args.forge_data_folder is not None):
|
|
2407
|
+
_die(2, "usage error: --config replaces --skills-output-folder and --forge-data-folder")
|
|
2408
|
+
|
|
2409
|
+
root = Path(args.project_root)
|
|
2410
|
+
if not root.is_dir():
|
|
2411
|
+
fail(1, f"--project-root is not a directory: {args.project_root}")
|
|
2412
|
+
try:
|
|
2413
|
+
skills_output_folder, forge_data_folder = (
|
|
2414
|
+
read_config_folders(args.config) if args.config is not None
|
|
2415
|
+
else (args.skills_output_folder or "", args.forge_data_folder or ""))
|
|
2416
|
+
payload = run_merge(
|
|
2417
|
+
root,
|
|
2418
|
+
skills_output_folder,
|
|
2419
|
+
forge_data_folder,
|
|
2420
|
+
prior_state_from=args.prior_state_from,
|
|
2421
|
+
index_fresh=bool(args.index_fresh),
|
|
2422
|
+
skip_index=bool(args.skip_index),
|
|
2423
|
+
allow_ccc_init=not args.no_ccc_init,
|
|
2424
|
+
defer_index=args.defer_index,
|
|
2425
|
+
)
|
|
2426
|
+
if args.build_index:
|
|
2427
|
+
if args.result_to is not None and payload["index_action"] == "index":
|
|
2428
|
+
# settings.yml is already rewritten, and `ccc index` can run for
|
|
2429
|
+
# an hour: a host that stops the call must still find this run's
|
|
2430
|
+
# exclusion record, with an index that did not finish.
|
|
2431
|
+
_write_result(args.result_to, {**payload, "index": {
|
|
2432
|
+
"status": "failed", "indexed_path": None, "last_indexed": None,
|
|
2433
|
+
"file_count": None, "failed_reason": INDEX_UNFINISHED}})
|
|
2434
|
+
warnings: list[str] = []
|
|
2435
|
+
payload["index"] = build_index(root, args.project_root, payload["index_action"],
|
|
2436
|
+
payload["not_ready_reason"], args.prior_state_from, warnings)
|
|
2437
|
+
payload["warnings"] += [_payload_safe(w) for w in warnings]
|
|
2438
|
+
except HelperError as e:
|
|
2439
|
+
fail(e.code, str(e))
|
|
2440
|
+
except Exception as e: # noqa: BLE001 — every failure must stay stderr JSON
|
|
2441
|
+
fail(2, f"unexpected error: {type(e).__name__}: {e}")
|
|
2442
|
+
done(payload)
|
|
2443
|
+
|
|
2444
|
+
|
|
2445
|
+
def _write_result(path: Path, value: dict) -> None:
|
|
2446
|
+
"""Write --result-to; a file that cannot be written is one more stderr error."""
|
|
2447
|
+
try:
|
|
2448
|
+
_atomic_write(path, json.dumps(value) + "\n")
|
|
2449
|
+
except HelperError as e:
|
|
2450
|
+
print(json.dumps({"status": "error", "message": _payload_safe(str(e))}), file=sys.stderr)
|
|
2451
|
+
sys.exit(e.code)
|
|
368
2452
|
|
|
369
2453
|
|
|
370
2454
|
if __name__ == "__main__":
|