@cbartens/bkit 6.0.0 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bkit/README.md +154 -151
- package/.bkit/core/.agents/context.md +95 -29
- package/.bkit/core/.agents/docs.md +17 -15
- package/.bkit/core/.agents/plugins/bkit/hooks/bkit-gateguard.cjs +1 -0
- package/.bkit/core/.agents/plugins/bkit/hooks/bkit-guardrails.cjs +6 -5
- package/.bkit/core/.agents/plugins/bkit/hooks/bkit-readguard.cjs +1 -0
- package/.bkit/core/.agents/plugins/bkit/hooks/bkit-rules.cjs +65 -0
- package/.bkit/core/.agents/rules.md +108 -107
- package/.bkit/core/.agents/skills.md +63 -15
- package/.bkit/core/.bkit/README.md +154 -151
- package/.bkit/core/.bkit/authoring.md +714 -84
- package/.bkit/core/.bkit/cli.md +407 -0
- package/.bkit/core/.bkit/cloud.md +281 -193
- package/.bkit/core/.bkit/comparison.md +43 -41
- package/.bkit/core/.bkit/install.md +154 -84
- package/.bkit/core/.bkit/jsonl.md +193 -130
- package/.bkit/core/.bkit/pipeline.md +114 -114
- package/.bkit/core/.bkit/prompts.md +272 -165
- package/.bkit/core/.bkit/providers.md +457 -271
- package/.bkit/core/.bkit/router.md +69 -0
- package/.bkit/core/.bkit/templates.md +529 -256
- package/.bkit/core/.bkit/trackers.md +80 -52
- package/.bkit/core/.bkit/workflows.md +589 -504
- package/.bkit/core/.claude/hooks/bkit-gateguard.cjs +2 -2
- package/.bkit/core/.claude/hooks/bkit-guardrails.cjs +3 -3
- package/.bkit/core/.claude/hooks/bkit-preflight.sh +45 -14
- package/.bkit/core/.codex/hooks/bkit-guardrails.cjs +3 -3
- package/.bkit/core/.codex/hooks/bkit-rules.cjs +57 -0
- package/.bkit/core/.cursor/hooks/bkit-guardrails.cjs +7 -7
- package/.bkit/core/.cursor/hooks/bkit-preflight.sh +47 -12
- package/.bkit/core/.opencode/plugin/bkit-rules.js +89 -0
- package/.bkit/core/AGENTS.md +5 -73
- package/.bkit/core/docs/templates/bkit-audit-accessibility.md +127 -105
- package/.bkit/core/docs/templates/bkit-audit-adoption.md +104 -70
- package/.bkit/core/docs/templates/bkit-audit-agents.md +94 -72
- package/.bkit/core/docs/templates/bkit-audit-aio-seo.md +99 -78
- package/.bkit/core/docs/templates/bkit-audit-analytics.md +140 -82
- package/.bkit/core/docs/templates/bkit-audit-annotations.md +98 -72
- package/.bkit/core/docs/templates/bkit-audit-app-performance.md +121 -96
- package/.bkit/core/docs/templates/bkit-audit-app-store.md +157 -76
- package/.bkit/core/docs/templates/bkit-audit-architecture.md +113 -71
- package/.bkit/core/docs/templates/bkit-audit-ci-performance.md +75 -72
- package/.bkit/core/docs/templates/bkit-audit-config.md +122 -76
- package/.bkit/core/docs/templates/bkit-audit-cost.md +85 -74
- package/.bkit/core/docs/templates/bkit-audit-coverage.md +75 -68
- package/.bkit/core/docs/templates/bkit-audit-database.md +101 -70
- package/.bkit/core/docs/templates/bkit-audit-dead-code.md +164 -83
- package/.bkit/core/docs/templates/bkit-audit-deferrals.md +115 -86
- package/.bkit/core/docs/templates/bkit-audit-dependencies.md +264 -88
- package/.bkit/core/docs/templates/bkit-audit-documentation.md +118 -93
- package/.bkit/core/docs/templates/bkit-audit-duplication.md +139 -86
- package/.bkit/core/docs/templates/bkit-audit-failures.md +110 -81
- package/.bkit/core/docs/templates/bkit-audit-flakes.md +99 -74
- package/.bkit/core/docs/templates/bkit-audit-gates.md +105 -77
- package/.bkit/core/docs/templates/bkit-audit-issues.md +104 -80
- package/.bkit/core/docs/templates/bkit-audit-llm.md +131 -58
- package/.bkit/core/docs/templates/bkit-audit-logs.md +123 -100
- package/.bkit/core/docs/templates/bkit-audit-models.md +98 -53
- package/.bkit/core/docs/templates/bkit-audit-privacy.md +56 -52
- package/.bkit/core/docs/templates/bkit-audit-prompts.md +63 -54
- package/.bkit/core/docs/templates/bkit-audit-red-team.md +124 -68
- package/.bkit/core/docs/templates/bkit-audit-reliability.md +95 -70
- package/.bkit/core/docs/templates/bkit-audit-repo.md +169 -73
- package/.bkit/core/docs/templates/bkit-audit-security.md +206 -96
- package/.bkit/core/docs/templates/bkit-audit-slop.md +77 -70
- package/.bkit/core/docs/templates/bkit-audit-testing.md +162 -114
- package/.bkit/core/docs/templates/bkit-audit-usability.md +113 -71
- package/.bkit/core/docs/templates/bkit-plan.md +138 -132
- package/.bkit/core/engine/checks/after/auth.ts +40 -37
- package/.bkit/core/engine/checks/after/ci-yml.ts +65 -2
- package/.bkit/core/engine/checks/after/index.ts +12 -2
- package/.bkit/core/engine/checks/after/tools.ts +80 -33
- package/.bkit/core/engine/checks/before/context-fetch.ts +227 -33
- package/.bkit/core/engine/checks/before/env.ts +1 -1
- package/.bkit/core/engine/checks/before/index.ts +7 -0
- package/.bkit/core/engine/checks/before/repo.ts +40 -0
- package/.bkit/core/engine/checks/status-holes.ts +916 -123
- package/.bkit/core/engine/checks/tips/catalogue.ts +68 -17
- package/.bkit/core/engine/checks/tips/detect.ts +32 -0
- package/.bkit/core/engine/checks/tips/index.ts +47 -6
- package/.bkit/core/engine/cli.ts +238 -84
- package/.bkit/core/engine/commands/analysis.ts +128 -76
- package/.bkit/core/engine/commands/ci/ci.ts +363 -29
- package/.bkit/core/engine/commands/ci/pins.ts +172 -0
- package/.bkit/core/engine/commands/ci/push-gate.ts +223 -0
- package/.bkit/core/engine/commands/install/uninstall.ts +21 -27
- package/.bkit/core/engine/commands/install/upgrade.ts +32 -15
- package/.bkit/core/engine/commands/land.ts +126 -11
- package/.bkit/core/engine/commands/lib.ts +303 -6
- package/.bkit/core/engine/commands/registry.ts +6 -2
- package/.bkit/core/engine/commands/setup/setup.ts +1 -0
- package/.bkit/core/engine/commands/stats.ts +91 -35
- package/.bkit/core/engine/commands/workflow/branches.ts +1477 -117
- package/.bkit/core/engine/commands/workflow/challenge.ts +114 -39
- package/.bkit/core/engine/commands/workflow/completeness.ts +109 -0
- package/.bkit/core/engine/commands/workflow/context.ts +433 -55
- package/.bkit/core/engine/commands/workflow/counters.ts +244 -39
- package/.bkit/core/engine/commands/workflow/decisions.ts +61 -17
- package/.bkit/core/engine/commands/workflow/disk.ts +237 -55
- package/.bkit/core/engine/commands/workflow/fids.ts +19 -1
- package/.bkit/core/engine/commands/workflow/issues.ts +6 -0
- package/.bkit/core/engine/commands/workflow/ledgers.ts +458 -18
- package/.bkit/core/engine/commands/workflow/locks.ts +2 -0
- package/.bkit/core/engine/commands/workflow/plan-blocks.ts +124 -15
- package/.bkit/core/engine/commands/workflow/plans.ts +257 -0
- package/.bkit/core/engine/commands/workflow/prs.ts +12 -3
- package/.bkit/core/engine/commands/workflow/repo.ts +8 -0
- package/.bkit/core/engine/commands/workflow/roadmap.ts +56 -15
- package/.bkit/core/engine/commands/workflow/run-facts.ts +22 -31
- package/.bkit/core/engine/commands/workflow/runs.ts +652 -34
- package/.bkit/core/engine/commands/workflow/runtime.ts +3 -0
- package/.bkit/core/engine/commands/workflow/skills.ts +1 -0
- package/.bkit/core/engine/commands/workflow/status.ts +200 -17
- package/.bkit/core/engine/commands/workflow/step-table.ts +75 -33
- package/.bkit/core/engine/commands/workflow/steps.ts +635 -24
- package/.bkit/core/engine/commands/workflow/summary-findings.ts +177 -0
- package/.bkit/core/engine/commands/workflow/summary.ts +192 -46
- package/.bkit/core/engine/commands/workflow/tasks.ts +409 -0
- package/.bkit/core/engine/commands/workflow/test-blocks.ts +254 -0
- package/.bkit/core/engine/commands/workflow/tests-changed.ts +201 -0
- package/.bkit/core/engine/commands/workflow/tests.ts +1285 -356
- package/.bkit/core/engine/commands/workflow/workflows.ts +228 -43
- package/.bkit/core/engine/commands/workflow/worktrees.ts +389 -47
- package/.bkit/core/engine/install/cloud-env.ts +251 -121
- package/.bkit/core/engine/install/context-ci-workflow.ts +102 -0
- package/.bkit/core/engine/install/manifest.ts +60 -12
- package/.bkit/core/engine/install/modules/antigravity.ts +3 -2
- package/.bkit/core/engine/install/modules/cloud.ts +3 -3
- package/.bkit/core/engine/install/modules/codex.ts +15 -2
- package/.bkit/core/engine/install/modules/core.ts +956 -52
- package/.bkit/core/engine/install/modules/cursor.ts +6 -6
- package/.bkit/core/engine/install/modules/opencode.ts +16 -4
- package/.bkit/core/engine/install/template-notes.ts +130 -0
- package/.bkit/core/engine/install/version-file.ts +3 -2
- package/.bkit/core/engine/install/write.ts +5 -0
- package/.bkit/core/engine/ledgers/compact.ts +124 -0
- package/.bkit/core/engine/ledgers/journal.ts +291 -0
- package/.bkit/core/engine/ledgers/status.ts +160 -9
- package/.bkit/core/engine/ledgers/telemetry.ts +158 -21
- package/.bkit/core/engine/render/hosts.ts +114 -14
- package/.bkit/core/engine/shared/bkit-latest.ts +26 -4
- package/.bkit/core/engine/shared/dep-updater.ts +60 -0
- package/.bkit/core/engine/shared/egress.ts +1 -1
- package/.bkit/core/engine/shared/package-manager.ts +9 -0
- package/.bkit/core/engine/shared/settings.ts +92 -17
- package/.bkit/core/engine/shared/timeouts.ts +2 -0
- package/.bkit/core/engine/tools/cli-doc.ts +111 -0
- package/.bkit/core/engine/tools/grammar-density.ts +1 -1
- package/.bkit/core/engine/tools/instructions-sources.ts +15 -17
- package/.bkit/core/engine/tools/instructions.ts +94 -43
- package/.bkit/core/hooks/antigravity-hook.js +25 -10
- package/.bkit/core/hooks/gateguard-engine.js +53 -29
- package/.bkit/core/hooks/guardrails-shell.js +4 -4
- package/.bkit/core/hooks/guardrails-tiers.js +3 -3
- package/.bkit/core/hooks/preflight-linear.sh +9 -9
- package/.bkit/core/hooks/preflight-rules.sh +8 -0
- package/.bkit/core/hooks/readguard-engine.js +29 -10
- package/.bkit/core/hooks/session-rules-advisory.js +61 -0
- package/.bkit/core/prompts/bkit-audit-generate.md +187 -116
- package/.bkit/core/prompts/bkit-build-challenge-1.md +56 -37
- package/.bkit/core/prompts/bkit-build-challenge-2.md +65 -45
- package/.bkit/core/prompts/bkit-build-install-skills.md +38 -29
- package/.bkit/core/prompts/bkit-build-plan-convert.md +43 -32
- package/.bkit/core/prompts/bkit-build-plan-current.md +49 -39
- package/.bkit/core/prompts/bkit-build-plan-gaps-remediate.md +79 -55
- package/.bkit/core/prompts/bkit-build-plan-tasks.md +149 -0
- package/.bkit/core/prompts/bkit-build-plan-tests.md +120 -0
- package/.bkit/core/prompts/bkit-build-plan-verify.md +87 -50
- package/.bkit/core/prompts/bkit-debug-abort-handover.md +36 -35
- package/.bkit/core/prompts/bkit-debug-failure-diagnose.md +96 -64
- package/.bkit/core/prompts/bkit-debug-failure-remediate.md +54 -45
- package/.bkit/core/prompts/bkit-debug-failure-reproduce.md +42 -33
- package/.bkit/core/prompts/bkit-debug-failure-soak.md +53 -37
- package/.bkit/core/prompts/bkit-design-brainstorm-ideate-1.md +35 -30
- package/.bkit/core/prompts/bkit-design-brainstorm-ideate-2.md +34 -29
- package/.bkit/core/prompts/bkit-design-brainstorm-ideate-3.md +32 -28
- package/.bkit/core/prompts/bkit-design-generate.md +68 -52
- package/.bkit/core/prompts/bkit-design-organise.md +73 -35
- package/.bkit/core/prompts/bkit-design-review-1.md +58 -54
- package/.bkit/core/prompts/bkit-design-review-2.md +54 -50
- package/.bkit/core/prompts/bkit-design-spike.md +54 -47
- package/.bkit/core/prompts/bkit-design-update.md +42 -31
- package/.bkit/core/prompts/bkit-plan-generate.md +76 -58
- package/.bkit/core/prompts/bkit-plan-review-1.md +63 -49
- package/.bkit/core/prompts/bkit-plan-review-2.md +60 -48
- package/.bkit/core/prompts/bkit-plan-scope-check.md +53 -46
- package/.bkit/core/prompts/bkit-plan-update.md +70 -52
- package/.bkit/core/prompts/bkit-ship-conflict-resolve.md +82 -50
- package/.bkit/core/prompts/bkit-ship-deferred-sweep.md +80 -45
- package/.bkit/core/prompts/bkit-ship-update-docs.md +35 -29
- package/.bkit/core/prompts/bkit-utility-compress-markdown.md +55 -49
- package/.bkit/core/prompts/bkit-utility-update-decisions.md +76 -49
- package/.bkit/core/prompts/bkit-utility-update-deferrals.md +70 -41
- package/.bkit/core/prompts/bkit-utility-update-documentation.md +33 -30
- package/.bkit/core/prompts/bkit-utility-update-failures.md +73 -43
- package/.bkit/core/prompts/bkit-utility-update-knowledge.md +77 -50
- package/.bkit/core/prompts/bkit-utility-update-roadmap.md +77 -45
- package/.bkit/core/prompts/bkit-utility-update-tracker.md +144 -68
- package/.bkit/core/workflows/bkit-auto.md +53 -30
- package/.bkit/core/workflows/bkit-build.md +635 -304
- package/.bkit/core/workflows/bkit-debug.md +301 -148
- package/.bkit/core/workflows/bkit-design.md +167 -95
- package/.bkit/core/workflows/bkit-plan.md +172 -84
- package/.bkit/core/workflows/bkit-ship.md +299 -152
- package/.bkit/shared/.agents/plugins/bkit/hooks.json +10 -0
- package/.bkit/shared/.claude/bkit-cloud.md +12 -10
- package/.bkit/shared/.claude/settings.json +3 -0
- package/.bkit/shared/.codex/bkit-cloud.md +98 -0
- package/.bkit/shared/.codex/hooks.json +20 -0
- package/.bkit/shared/.cursor/bkit-cloud.md +3 -3
- package/.bkit/shared/.opencode/opencode.json +1 -1
- package/.bkit/shared/CLAUDE.md +5 -1
- package/.bkit/shared/GEMINI.md +4 -1
- package/.bkit/shared/docs/architecture.md +4 -7
- package/.bkit/shared/docs/data-schema.md +4 -7
- package/.bkit/shared/docs/deployments.md +239 -166
- package/.bkit/shared/docs/design.md +11 -12
- package/.bkit/shared/docs/noise.md +8 -7
- package/.bkit/shared/docs/observability.md +5 -8
- package/.bkit/shared/docs/security.md +35 -17
- package/.bkit/shared/docs/setup.md +29 -18
- package/.bkit/shared/docs/technology.md +3 -6
- package/.bkit/shared/docs/testing.md +282 -176
- package/.bkit/shared/gitignore +6 -2
- package/.bkit/templates/.claude/hooks/bkit-cloud-bootstrap.sh +15 -16
- package/.bkit/templates/.cursor/Dockerfile +12 -8
- package/.bkit/templates/.cursor/environment.json +1 -1
- package/.bkit/templates/.github/ISSUE_TEMPLATE/bkit-bug.yml +10 -8
- package/.bkit/templates/.github/ISSUE_TEMPLATE/bkit-change.yml +24 -12
- package/.bkit/templates/.github/ISSUE_TEMPLATE/bkit-idea.yml +3 -3
- package/.bkit/templates/.github/renovate.json5 +34 -6
- package/.bkit/templates/.github/workflows/bkit-ci.yml +53 -50
- package/.bkit/templates/.github/workflows/bkit-cloud.yml +13 -14
- package/.bkit/templates/.github/workflows/bkit-health.yml +97 -0
- package/.bkit/templates/.github/workflows/bkit-renovate.yml +5 -4
- package/.bkit/templates/.github/workflows/bkit-trivy.yml +5 -4
- package/.bkit/templates/.husky/pre-commit +2 -2
- package/.bkit/templates/.husky/pre-push +95 -35
- package/.bkit/templates/.prettierignore +6 -7
- package/.bkit/templates/.vale.ini +27 -27
- package/.bkit/templates/context/bkit-cloudrun.md +23 -15
- package/.bkit/templates/context/bkit-expo.md +70 -52
- package/.bkit/templates/context/bkit-github.md +93 -56
- package/.bkit/templates/context/bkit-inngest.md +21 -15
- package/.bkit/templates/context/bkit-maestro.md +22 -16
- package/.bkit/templates/context/bkit-playwright.md +23 -17
- package/.bkit/templates/context/bkit-resend.md +12 -10
- package/.bkit/templates/context/bkit-sentry.md +30 -20
- package/.bkit/templates/context/bkit-serena.md +8 -7
- package/.bkit/templates/context/bkit-stripe.md +14 -13
- package/.bkit/templates/context/bkit-supabase.md +16 -13
- package/.bkit/templates/context/bkit-tavily.md +14 -11
- package/.bkit/templates/context/bkit-vercel.md +27 -18
- package/.bkit/templates/docs/capabilities.md +15 -14
- package/.bkit/templates/scripts/ci/{check-accepted-divergences.mjs → bkit-check-accepted-divergences.mjs} +6 -3
- package/.bkit/templates/scripts/ci/{check-accepted-divergences.test.mjs → bkit-check-accepted-divergences.test.mjs} +3 -3
- package/.bkit/templates/scripts/ci/{check-capstone.mjs → bkit-check-capstone.mjs} +2 -2
- package/.bkit/templates/scripts/ci/{check-capstone.test.mjs → bkit-check-capstone.test.mjs} +4 -4
- package/.bkit/templates/scripts/ci/{check-dependency-audit.mjs → bkit-check-dependency-audit.mjs} +103 -18
- package/.bkit/templates/scripts/ci/{check-dependency-audit.test.mjs → bkit-check-dependency-audit.test.mjs} +31 -3
- package/.bkit/templates/scripts/ci/bkit-machine-lock.sh +387 -0
- package/.bkit/templates/scripts/ci/bkit-test-awake.mjs +75 -0
- package/.bkit/templates/scripts/ci/bkit-test-awake.test.mjs +160 -0
- package/.bkit/templates/scripts/ci/lanes/{resolve-push-base.mjs → bkit-resolve-push-base.mjs} +3 -3
- package/.bkit/templates/scripts/ci/lanes/{resolve-push-base.test.mjs → bkit-resolve-push-base.test.mjs} +4 -4
- package/.bkit/templates/scripts/cleanup/bkit-disk-prune.mjs +9 -1
- package/.bkit/templates/scripts/graphify/bkit-graph-drift.py +1 -1
- package/.bkit/templates/scripts/graphify/bkit-graph-eager.py +3 -3
- package/.bkit/templates/scripts/graphify/bkit-graph-embed.py +63 -0
- package/.bkit/templates/scripts/graphify/bkit-graph-imports.py +2 -2
- package/.bkit/templates/scripts/graphify/bkit-graph-pre-push.sh +69 -18
- package/.bkit/templates/scripts/graphify/bkit-graph-repair.py +3 -3
- package/.bkit/templates/scripts/graphify/bkit-graph-surface.py +1 -1
- package/.bkit/templates/scripts/graphify/bkit-graphify-graphml.py +1 -1
- package/.bkit/templates/scripts/graphify/bkit-graphify-rollup.py +197 -126
- package/.bkit/templates/scripts/graphify/bkit-graphify.sh +64 -41
- package/.bkit/templates/scripts/slop/bkit-slop-type-suppression.mjs +23 -7
- package/.bkit/templates/slop-scan.config.json +9 -2
- package/CHANGELOG.md +119 -0
- package/README.md +250 -76
- package/dist/checks/after/auth.js +36 -33
- package/dist/checks/after/ci-yml.js +52 -2
- package/dist/checks/after/index.js +12 -4
- package/dist/checks/after/tools.js +73 -33
- package/dist/checks/before/context-fetch.js +218 -33
- package/dist/checks/before/env.js +1 -1
- package/dist/checks/before/index.js +6 -2
- package/dist/checks/before/repo.js +35 -1
- package/dist/checks/status-holes.js +739 -114
- package/dist/checks/tips/catalogue.js +59 -17
- package/dist/checks/tips/detect.js +30 -0
- package/dist/checks/tips/index.js +48 -6
- package/dist/cli.js +222 -84
- package/dist/commands/analysis.js +77 -53
- package/dist/commands/ci/ci.js +302 -25
- package/dist/commands/ci/pins.js +142 -0
- package/dist/commands/ci/push-gate.js +194 -0
- package/dist/commands/install/uninstall.js +23 -27
- package/dist/commands/install/upgrade.js +21 -11
- package/dist/commands/land.js +126 -10
- package/dist/commands/lib.js +244 -5
- package/dist/commands/registry.js +6 -2
- package/dist/commands/setup/setup.js +1 -0
- package/dist/commands/stats.js +69 -30
- package/dist/commands/workflow/branches.js +1232 -111
- package/dist/commands/workflow/challenge.js +90 -37
- package/dist/commands/workflow/completeness.js +97 -0
- package/dist/commands/workflow/context.js +371 -51
- package/dist/commands/workflow/counters.js +206 -40
- package/dist/commands/workflow/decisions.js +60 -17
- package/dist/commands/workflow/disk.js +213 -55
- package/dist/commands/workflow/fids.js +18 -1
- package/dist/commands/workflow/issues.js +6 -0
- package/dist/commands/workflow/ledgers.js +372 -21
- package/dist/commands/workflow/locks.js +2 -0
- package/dist/commands/workflow/plan-blocks.js +78 -14
- package/dist/commands/workflow/plans.js +205 -0
- package/dist/commands/workflow/prs.js +12 -3
- package/dist/commands/workflow/repo.js +6 -0
- package/dist/commands/workflow/roadmap.js +50 -17
- package/dist/commands/workflow/run-facts.js +23 -28
- package/dist/commands/workflow/runs.js +530 -33
- package/dist/commands/workflow/runtime.js +3 -0
- package/dist/commands/workflow/skills.js +1 -0
- package/dist/commands/workflow/status.js +166 -19
- package/dist/commands/workflow/step-table.js +61 -29
- package/dist/commands/workflow/steps.js +542 -26
- package/dist/commands/workflow/summary-findings.js +117 -0
- package/dist/commands/workflow/summary.js +154 -41
- package/dist/commands/workflow/tasks.js +378 -0
- package/dist/commands/workflow/test-blocks.js +231 -0
- package/dist/commands/workflow/tests-changed.js +172 -0
- package/dist/commands/workflow/tests.js +1013 -321
- package/dist/commands/workflow/workflows.js +191 -38
- package/dist/commands/workflow/worktrees.js +310 -42
- package/dist/install/cloud-env.js +233 -99
- package/dist/install/context-ci-workflow.js +85 -0
- package/dist/install/manifest.js +57 -12
- package/dist/install/modules/antigravity.js +3 -2
- package/dist/install/modules/cloud.js +3 -3
- package/dist/install/modules/codex.js +12 -2
- package/dist/install/modules/core.js +851 -53
- package/dist/install/modules/cursor.js +6 -6
- package/dist/install/modules/opencode.js +11 -5
- package/dist/install/template-notes.js +116 -0
- package/dist/install/version-file.js +3 -2
- package/dist/ledgers/compact.js +87 -0
- package/dist/ledgers/journal.js +233 -0
- package/dist/ledgers/status.js +115 -5
- package/dist/ledgers/telemetry.js +121 -17
- package/dist/render/hosts.js +100 -14
- package/dist/shared/bkit-latest.js +26 -4
- package/dist/shared/dep-updater.js +49 -0
- package/dist/shared/egress.js +1 -1
- package/dist/shared/package-manager.js +6 -0
- package/dist/shared/settings.js +59 -10
- package/dist/shared/timeouts.js +2 -0
- package/dist/tools/cli-doc.js +97 -0
- package/dist/tools/grammar-density.js +1 -1
- package/dist/tools/instructions-sources.js +15 -16
- package/dist/tools/instructions.js +83 -39
- package/package.json +58 -42
- package/.bkit/core/.bkit/lite.md +0 -73
- package/.bkit/core/engine/commands/workflow/workstreams.ts +0 -172
- package/.bkit/core/prompts/bkit-build-plan-execute.md +0 -124
- package/.bkit/templates/scripts/ci/bkit-local-stack-lock.sh +0 -194
- package/dist/commands/workflow/workstreams.js +0 -149
- /package/.bkit/templates/.vale/styles/{House → Bkit}/Filler.yml +0 -0
- /package/.bkit/templates/.vale/styles/{House → Bkit}/Hedging.yml +0 -0
- /package/.bkit/templates/.vale/styles/{House → Bkit}/LLMSlop.yml +0 -0
- /package/.bkit/templates/.vale/styles/{House → Bkit}/Verbose.yml +0 -0
- /package/.bkit/templates/scripts/ci/{accepted-advisories.json → bkit-accepted-advisories.json} +0 -0
- /package/.bkit/templates/scripts/ci/{accepted-divergences.json → bkit-accepted-divergences.json} +0 -0
- /package/.bkit/templates/scripts/launchd/{launchd.env.example → bkit-launchd.env.example} +0 -0
- /package/.bkit/templates/scripts/launchd/{ai.bkit.launchd.plist → bkit-launchd.plist} +0 -0
|
@@ -8,24 +8,45 @@
|
|
|
8
8
|
// ran (not executable / not found), so it is not recorded as a failing test.
|
|
9
9
|
// The failure count comes from the JUnit reports the gate's runners write into
|
|
10
10
|
// `{log}.reports/` (`BKIT_TEST_REPORT_DIR`), never from reading the log.
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
// file through the consumer's `bkit:test:file` script
|
|
14
|
-
// never
|
|
15
|
-
// unwritten code all exit non-zero exactly like a real failure. A red
|
|
16
|
-
//
|
|
11
|
+
// bkit tests red / green — test-first evidence for build tasks. `red` runs every TEST FIRST
|
|
12
|
+
// test of one group's open tasks after the tests step wrote them and before any task code
|
|
13
|
+
// exists: one run per test file through the consumer's `bkit:test:file` script, each test
|
|
14
|
+
// judged by the JUnit report, never the exit code alone (a misspelled path, a syntax error
|
|
15
|
+
// and an import of unwritten code all exit non-zero exactly like a real failure). A red is
|
|
16
|
+
// the NAMED test failing, or its file failing to load; `green` reads the group's red row and
|
|
17
|
+
// needs the named test to run and pass.
|
|
17
18
|
|
|
18
19
|
import { spawnSync } from "node:child_process";
|
|
20
|
+
import { createHash } from "node:crypto";
|
|
19
21
|
import * as fs from "node:fs";
|
|
20
22
|
import * as os from "node:os";
|
|
21
23
|
import * as path from "node:path";
|
|
22
24
|
|
|
23
|
-
import { BKIT_TEST_FILE_SCRIPT } from "../../install/manifest.js";
|
|
25
|
+
import { BKIT_LINT_FILE_SCRIPT, BKIT_TEST_FILE_SCRIPT } from "../../install/manifest.js";
|
|
24
26
|
import { clampLogPath } from "../../ledgers/clamp-log.js";
|
|
25
|
-
import {
|
|
27
|
+
import {
|
|
28
|
+
appendLine,
|
|
29
|
+
statusTimestamp,
|
|
30
|
+
validateCiSelection,
|
|
31
|
+
testRunProblem,
|
|
32
|
+
} from "../../ledgers/status.js";
|
|
33
|
+
import {
|
|
34
|
+
classifyLockExit,
|
|
35
|
+
CommandError,
|
|
36
|
+
EXIT,
|
|
37
|
+
git,
|
|
38
|
+
runGit,
|
|
39
|
+
testsLockHolder,
|
|
40
|
+
wholeNumber,
|
|
41
|
+
type CommandSpec,
|
|
42
|
+
} from "../lib.js";
|
|
26
43
|
import { parseJson } from "../../shared/json.js";
|
|
27
44
|
import { readPackageJson } from "../../shared/package-json.js";
|
|
28
|
-
import {
|
|
45
|
+
import {
|
|
46
|
+
COULD_NOT_RUN_EXIT_CODES,
|
|
47
|
+
scriptCommand,
|
|
48
|
+
testCommand,
|
|
49
|
+
} from "../../shared/package-manager.js";
|
|
29
50
|
import { CONFIG_DEFAULTS, readBkitConfig, type TaskCaps } from "../../shared/settings.js";
|
|
30
51
|
import {
|
|
31
52
|
FILE_FLAG,
|
|
@@ -33,17 +54,22 @@ import {
|
|
|
33
54
|
WORKFLOW_FLAG,
|
|
34
55
|
positiveInt,
|
|
35
56
|
readStatusRows,
|
|
57
|
+
readRuns,
|
|
36
58
|
statusFile,
|
|
37
59
|
str,
|
|
38
60
|
} from "./status.js";
|
|
39
61
|
import {
|
|
62
|
+
NO_MARKERS_REASON,
|
|
63
|
+
derivedTestMatrix,
|
|
40
64
|
fenceNeverClosed,
|
|
41
65
|
parsePlanBlocks,
|
|
42
66
|
reuseInventory,
|
|
67
|
+
supersessionInventory,
|
|
43
68
|
type PlanTaskBlock,
|
|
44
69
|
} from "./plan-blocks.js";
|
|
45
70
|
import { writeStepRow } from "./steps.js";
|
|
46
71
|
import { appendRecord, clampFile, type ClampRecord } from "./context.js";
|
|
72
|
+
import { testsChangedVerb } from "./tests-changed.js";
|
|
47
73
|
|
|
48
74
|
/** Exit code: no candidate gate is declared in package.json. */
|
|
49
75
|
export const EXIT_NO_GATE = 3;
|
|
@@ -52,9 +78,14 @@ export const EXIT_COULD_NOT_RUN = 4;
|
|
|
52
78
|
/** Exit code: --artifact could not be written, trimmed or recorded. */
|
|
53
79
|
export const EXIT_ARTIFACT_STEP = 5;
|
|
54
80
|
|
|
81
|
+
/** The status row type `tests run --file` appends, one per run. */
|
|
82
|
+
export const TEST_RUN_ROW = "test-run";
|
|
83
|
+
|
|
55
84
|
/** The env var naming the folder every runner a gate calls writes its JUnit XML into. */
|
|
56
85
|
export const REPORT_DIR_ENV = "BKIT_TEST_REPORT_DIR";
|
|
57
86
|
|
|
87
|
+
export { classifyLockExit } from "../lib.js";
|
|
88
|
+
|
|
58
89
|
const NO_REPORT_HINT =
|
|
59
90
|
`no JUnit report in $${REPORT_DIR_ENV} — failures unknown; turn on each test runner's JUnit ` +
|
|
60
91
|
`reporter in the gate and point its output into $${REPORT_DIR_ENV} ` +
|
|
@@ -98,7 +129,8 @@ export function runToLog(
|
|
|
98
129
|
): number {
|
|
99
130
|
fs.mkdirSync(path.dirname(log), { recursive: true });
|
|
100
131
|
const fd = fs.openSync(log, "w");
|
|
101
|
-
const
|
|
132
|
+
const protectedArgv = testCommand(argv);
|
|
133
|
+
const res = spawnSync(protectedArgv[0]!, protectedArgv.slice(1), {
|
|
102
134
|
cwd,
|
|
103
135
|
stdio: ["ignore", fd, fd],
|
|
104
136
|
env: env === undefined ? process.env : { ...process.env, ...env },
|
|
@@ -138,14 +170,369 @@ const tailLines = (file: string, n: number) =>
|
|
|
138
170
|
.split(/\r?\n/)
|
|
139
171
|
.slice(-n);
|
|
140
172
|
|
|
173
|
+
/**
|
|
174
|
+
* Append one `test-run` row to `file`. `run` numbers this step's test-run rows in the
|
|
175
|
+
* workflow's active run (since its last workflow-started row), this one included, so a
|
|
176
|
+
* gate re-run under the same attempt still shows.
|
|
177
|
+
*/
|
|
178
|
+
function appendTestRun(
|
|
179
|
+
file: string,
|
|
180
|
+
r: {
|
|
181
|
+
workflow: string;
|
|
182
|
+
step: string;
|
|
183
|
+
attempt: number | undefined;
|
|
184
|
+
verdict: string | null;
|
|
185
|
+
exitCode: number | null;
|
|
186
|
+
failures: number | null;
|
|
187
|
+
scope: string;
|
|
188
|
+
log: string;
|
|
189
|
+
artifact: string | undefined;
|
|
190
|
+
source?: "ci" | "local";
|
|
191
|
+
evidence?: Record<string, unknown>;
|
|
192
|
+
},
|
|
193
|
+
): Record<string, unknown> {
|
|
194
|
+
const rows = readStatusRows(file);
|
|
195
|
+
let start = -1;
|
|
196
|
+
rows.forEach((x, i) => {
|
|
197
|
+
if (x.type === "workflow-started" && x.workflow === r.workflow) start = i;
|
|
198
|
+
});
|
|
199
|
+
const run =
|
|
200
|
+
rows
|
|
201
|
+
.slice(start + 1)
|
|
202
|
+
.filter((x) => x.type === TEST_RUN_ROW && x.workflow === r.workflow && x.step === r.step)
|
|
203
|
+
.length + 1;
|
|
204
|
+
const row = {
|
|
205
|
+
type: TEST_RUN_ROW,
|
|
206
|
+
workflow: r.workflow,
|
|
207
|
+
at: statusTimestamp(),
|
|
208
|
+
step: r.step,
|
|
209
|
+
...(r.attempt === undefined ? {} : { attempt: r.attempt }),
|
|
210
|
+
run,
|
|
211
|
+
verdict: r.verdict,
|
|
212
|
+
exitCode: r.exitCode,
|
|
213
|
+
failures: r.failures,
|
|
214
|
+
scope: r.scope,
|
|
215
|
+
log: r.log,
|
|
216
|
+
...(r.artifact === undefined ? {} : { artifact: r.artifact }),
|
|
217
|
+
...(r.source === undefined ? {} : { source: r.source }),
|
|
218
|
+
...r.evidence,
|
|
219
|
+
};
|
|
220
|
+
const problem = testRunProblem(row);
|
|
221
|
+
if (problem !== null) throw new CommandError(EXIT.USAGE, problem);
|
|
222
|
+
fs.appendFileSync(file, appendLine(row));
|
|
223
|
+
return row;
|
|
224
|
+
}
|
|
225
|
+
|
|
141
226
|
// ---------------------------------------------------------------------------
|
|
142
227
|
// Test-first: JUnit reading, plan `Test first:` lines, red / green judgement
|
|
143
228
|
// ---------------------------------------------------------------------------
|
|
144
229
|
|
|
145
230
|
/** The env var naming the JUnit report path `bkit:test:file` must write. */
|
|
146
231
|
export const REPORT_ENV = "BKIT_TEST_REPORT";
|
|
147
|
-
/** The status row step a
|
|
148
|
-
export const RED_STEP = "build-
|
|
232
|
+
/** The status row step a group's red writes: one row per `tests red` run, a state and blob per planned test. */
|
|
233
|
+
export const RED_STEP = "build-tests-red";
|
|
234
|
+
|
|
235
|
+
/** How a planned test stood at the group's red. */
|
|
236
|
+
export type RedState =
|
|
237
|
+
| "red"
|
|
238
|
+
| "red-not-loaded"
|
|
239
|
+
| "passed-accepted"
|
|
240
|
+
| "updated"
|
|
241
|
+
| "vacuous"
|
|
242
|
+
| "missing"
|
|
243
|
+
| "no-report"
|
|
244
|
+
| "lint"
|
|
245
|
+
| "unknown";
|
|
246
|
+
|
|
247
|
+
/** One planned test on a red row: its state, and the blob of its file as it stood then. */
|
|
248
|
+
export interface RedTest {
|
|
249
|
+
task: number;
|
|
250
|
+
test: string;
|
|
251
|
+
testName: string;
|
|
252
|
+
state: RedState;
|
|
253
|
+
/** `git hash-object` of the test file, written to the object store; null when the file is missing. */
|
|
254
|
+
blob: string | null;
|
|
255
|
+
/**
|
|
256
|
+
* Hash of the test's own lines (`testBlock`) as they stood; null when the file holds no
|
|
257
|
+
* block for the name. Null, and absent on rows written before per-test locking, lock by `blob`.
|
|
258
|
+
*/
|
|
259
|
+
block?: string | null;
|
|
260
|
+
/** The tests step's reason, on `passed-accepted`. */
|
|
261
|
+
reason?: string;
|
|
262
|
+
/** On `updated`: the commit the group started at, where the test already existed. */
|
|
263
|
+
base?: string;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* The states that count as a red: the test failed, its file did not load, the tests step
|
|
268
|
+
* vouched for its pass, or it is an existing test updated and passing, whose diff the
|
|
269
|
+
* verifier judges (it may add expectations, never remove or loosen one).
|
|
270
|
+
*/
|
|
271
|
+
export const RED_PROVEN: ReadonlySet<RedState> = new Set([
|
|
272
|
+
"red",
|
|
273
|
+
"red-not-loaded",
|
|
274
|
+
"passed-accepted",
|
|
275
|
+
"updated",
|
|
276
|
+
]);
|
|
277
|
+
|
|
278
|
+
/** The `build-tests-red` rows of `workflow` in status.jsonl, in file order; `group` narrows. */
|
|
279
|
+
function redRows(file: string, workflow: string, group?: number): Record<string, unknown>[] {
|
|
280
|
+
if (!fs.existsSync(file)) return [];
|
|
281
|
+
return (readStatusRows(file) as Array<Record<string, unknown>>).filter(
|
|
282
|
+
(r) =>
|
|
283
|
+
r.type === "step-done" &&
|
|
284
|
+
r.workflow === workflow &&
|
|
285
|
+
r.step === RED_STEP &&
|
|
286
|
+
typeof r.group === "number" &&
|
|
287
|
+
(group === undefined || r.group === group),
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** A red row's `tests`, as written. */
|
|
292
|
+
export const redTestsOf = (row: Record<string, unknown>): RedTest[] =>
|
|
293
|
+
Array.isArray(row.tests) ? (row.tests as RedTest[]) : [];
|
|
294
|
+
|
|
295
|
+
/** The key a planned test is looked up by: task numbers are unique across the plan. */
|
|
296
|
+
export const redKey = (task: number, test: string, name: string) =>
|
|
297
|
+
JSON.stringify([task, test, name]);
|
|
298
|
+
|
|
299
|
+
/** The number this group's next red row carries: its rows so far, plus one. */
|
|
300
|
+
function redAttempt(file: string, workflow: string, group: number): number {
|
|
301
|
+
if (!fs.existsSync(file)) return 1;
|
|
302
|
+
return (
|
|
303
|
+
readStatusRows(file).filter((r) => {
|
|
304
|
+
const row = r as Record<string, unknown>;
|
|
305
|
+
return (
|
|
306
|
+
row.type === "step-done" &&
|
|
307
|
+
row.workflow === workflow &&
|
|
308
|
+
row.step === RED_STEP &&
|
|
309
|
+
row.group === group
|
|
310
|
+
);
|
|
311
|
+
}).length + 1
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Every planned test the red rows recorded, keyed by task, file and name. A later row's entry
|
|
317
|
+
* wins, so a retry or a later task of the group overrides only the tests it ran, except that a
|
|
318
|
+
* proven red is never replaced by an unproven result for the same test lines. `group` narrows
|
|
319
|
+
* to one group.
|
|
320
|
+
*/
|
|
321
|
+
export function redEntries(file: string, workflow: string, group?: number): Map<string, RedTest> {
|
|
322
|
+
const out = new Map<string, RedTest>();
|
|
323
|
+
for (const row of redRows(file, workflow, group))
|
|
324
|
+
for (const t of redTestsOf(row)) {
|
|
325
|
+
const key = redKey(t.task, t.test, t.testName);
|
|
326
|
+
const earlier = out.get(key);
|
|
327
|
+
const keepsProof =
|
|
328
|
+
earlier !== undefined &&
|
|
329
|
+
RED_PROVEN.has(earlier.state) &&
|
|
330
|
+
!RED_PROVEN.has(t.state) &&
|
|
331
|
+
typeof earlier.block === "string" &&
|
|
332
|
+
earlier.block === t.block;
|
|
333
|
+
if (!keepsProof) out.set(key, t);
|
|
334
|
+
}
|
|
335
|
+
return out;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** A planned test that differs from how the group's tests step left it. */
|
|
339
|
+
export interface TouchedTest {
|
|
340
|
+
test: string;
|
|
341
|
+
name: string;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* The planned tests of `group` a code step changed since their red: a test whose own lines
|
|
346
|
+
* (`testBlock`) no longer hash to the recorded block. A row with no block (written before
|
|
347
|
+
* per-test locking, or when no line carried the name) derives the block from the recorded
|
|
348
|
+
* blob; when that holds no block either, the whole file's blob is compared. Other edits to a
|
|
349
|
+
* test file are free.
|
|
350
|
+
*/
|
|
351
|
+
export function touchedTests(
|
|
352
|
+
top: string,
|
|
353
|
+
file: string,
|
|
354
|
+
workflow: string,
|
|
355
|
+
group: number,
|
|
356
|
+
): TouchedTest[] {
|
|
357
|
+
const touched: TouchedTest[] = [];
|
|
358
|
+
const texts = new Map<string, string | null>();
|
|
359
|
+
const textOf = (test: string) => {
|
|
360
|
+
if (!texts.has(test)) {
|
|
361
|
+
const abs = path.join(top, test);
|
|
362
|
+
texts.set(test, fs.existsSync(abs) ? fs.readFileSync(abs, "utf-8") : null);
|
|
363
|
+
}
|
|
364
|
+
return texts.get(test)!;
|
|
365
|
+
};
|
|
366
|
+
for (const t of redEntries(file, workflow, group).values()) {
|
|
367
|
+
let block = t.block ?? null;
|
|
368
|
+
if (block === null) block = blockHash(blobText(top, t.blob), t.testName);
|
|
369
|
+
if (block === null) {
|
|
370
|
+
if (git(top, ["hash-object", "--", t.test]) !== t.blob)
|
|
371
|
+
touched.push({ test: t.test, name: t.testName });
|
|
372
|
+
continue;
|
|
373
|
+
}
|
|
374
|
+
if (blockHash(textOf(t.test), t.testName) !== block)
|
|
375
|
+
touched.push({ test: t.test, name: t.testName });
|
|
376
|
+
}
|
|
377
|
+
return touched;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/** The text a recorded blob holds, untrimmed; null when the object store lacks it. */
|
|
381
|
+
function blobText(top: string, blob: string | null): string | null {
|
|
382
|
+
if (blob === null) return null;
|
|
383
|
+
const res = spawnSync("git", ["cat-file", "-p", blob], { cwd: top, encoding: "utf-8" });
|
|
384
|
+
return res.status === 0 ? res.stdout : null;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** Leading whitespace width, a tab counting as one column. */
|
|
388
|
+
const indentOf = (line: string) => line.length - line.trimStart().length;
|
|
389
|
+
|
|
390
|
+
/** A line that only closes the block opened at its own indent. */
|
|
391
|
+
const CLOSER = /^(?:[)\]}]|end\b)/;
|
|
392
|
+
|
|
393
|
+
const escapeRegExp = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
394
|
+
|
|
395
|
+
/** A comment line: a name in it is prose, not the test. */
|
|
396
|
+
const COMMENT = /^(?:\/\/|\/\*|\*|#|--|<!--)/;
|
|
397
|
+
|
|
398
|
+
/** A line opening a suite (`describe`, `context`, `suite`, a class), not a test. */
|
|
399
|
+
const SUITE = /^(?:[\w$]+\.)?(?:describe|context|suite|class)\b/;
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* The first line `matches` that is no comment and opens no suite; else the first matching
|
|
403
|
+
* suite line, so a suite that alone carries the name still yields a block; else -1.
|
|
404
|
+
*/
|
|
405
|
+
function testStart(lines: string[], matches: (line: string) => boolean): number {
|
|
406
|
+
let suite = -1;
|
|
407
|
+
for (let i = 0; i < lines.length; i++) {
|
|
408
|
+
const line = lines[i]!.trimStart();
|
|
409
|
+
if (!matches(lines[i]!) || COMMENT.test(line)) continue;
|
|
410
|
+
if (!SUITE.test(line)) return i;
|
|
411
|
+
if (suite < 0) suite = i;
|
|
412
|
+
}
|
|
413
|
+
return suite;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* The lines of the test named `name` in `text`, or null when no line names it. The test
|
|
418
|
+
* starts on the first line naming it as a quoted string (else as a whole word) that is no
|
|
419
|
+
* comment and opens no suite, or on the line before when the name opens its line and that
|
|
420
|
+
* line ends with `(`; it runs through every deeper-indented line after the start, and one
|
|
421
|
+
* closing line (`}`, `)`, `]`, `end`) at the start's indent. Framework-agnostic: it needs
|
|
422
|
+
* only a name and indentation.
|
|
423
|
+
*/
|
|
424
|
+
export function testBlock(text: string | null, name: string): string | null {
|
|
425
|
+
if (text === null) return null;
|
|
426
|
+
const span = testSpan(text, name);
|
|
427
|
+
return span === null
|
|
428
|
+
? null
|
|
429
|
+
: text
|
|
430
|
+
.split("\n")
|
|
431
|
+
.slice(span.start - 1, span.end)
|
|
432
|
+
.join("\n");
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/**
|
|
436
|
+
* The 1-based first and last line of the test named `name` in `text` (the lines
|
|
437
|
+
* `testBlock` returns), or null when no line names it.
|
|
438
|
+
*/
|
|
439
|
+
export function testSpan(text: string, name: string): { start: number; end: number } | null {
|
|
440
|
+
const lines = text.split("\n");
|
|
441
|
+
const quoted = [`"${name}"`, `'${name}'`, `\`${name}\``];
|
|
442
|
+
let at = testStart(lines, (l) => quoted.some((q) => l.includes(q)));
|
|
443
|
+
if (at < 0) {
|
|
444
|
+
const word = new RegExp(`(^|[^\\w])${escapeRegExp(name)}([^\\w]|$)`);
|
|
445
|
+
at = testStart(lines, (l) => word.test(l));
|
|
446
|
+
}
|
|
447
|
+
if (at < 0) {
|
|
448
|
+
// A suite path (`suite > test`, as a JUnit report names it): the last segment is the test.
|
|
449
|
+
const last = name.split(" > ").pop()!;
|
|
450
|
+
return last === name ? null : testSpan(text, last);
|
|
451
|
+
}
|
|
452
|
+
let start = at;
|
|
453
|
+
while (
|
|
454
|
+
start > 0 &&
|
|
455
|
+
/^["'`]/.test(lines[start]!.trimStart()) &&
|
|
456
|
+
/\($/.test(lines[start - 1]!.trimEnd())
|
|
457
|
+
)
|
|
458
|
+
start -= 1;
|
|
459
|
+
const base = indentOf(lines[start]!);
|
|
460
|
+
let end = start;
|
|
461
|
+
for (let i = start + 1; i < lines.length; i++) {
|
|
462
|
+
const line = lines[i]!;
|
|
463
|
+
if (line.trim() === "") continue;
|
|
464
|
+
if (indentOf(line) > base) {
|
|
465
|
+
end = i;
|
|
466
|
+
continue;
|
|
467
|
+
}
|
|
468
|
+
if (indentOf(line) === base && CLOSER.test(line.trimStart())) end = i;
|
|
469
|
+
break;
|
|
470
|
+
}
|
|
471
|
+
return { start: start + 1, end: end + 1 };
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/**
|
|
475
|
+
* The line diff of a test's own lines, `before` to `after`: unified lines (` `, `-`, `+`)
|
|
476
|
+
* over a longest common subsequence, and the counts of lines removed and added. A change
|
|
477
|
+
* that removes no line cannot have dropped an expectation; it can still have disabled one,
|
|
478
|
+
* which only a read of the added lines tells.
|
|
479
|
+
*/
|
|
480
|
+
export function lineDiff(
|
|
481
|
+
before: string,
|
|
482
|
+
after: string,
|
|
483
|
+
): { text: string; removed: number; added: number } {
|
|
484
|
+
const a = before.split("\n");
|
|
485
|
+
const b = after.split("\n");
|
|
486
|
+
const lcs: number[][] = Array.from({ length: a.length + 1 }, () =>
|
|
487
|
+
new Array<number>(b.length + 1).fill(0),
|
|
488
|
+
);
|
|
489
|
+
for (let i = a.length - 1; i >= 0; i--)
|
|
490
|
+
for (let j = b.length - 1; j >= 0; j--)
|
|
491
|
+
lcs[i]![j] =
|
|
492
|
+
a[i] === b[j] ? lcs[i + 1]![j + 1]! + 1 : Math.max(lcs[i + 1]![j]!, lcs[i]![j + 1]!);
|
|
493
|
+
const out: string[] = [];
|
|
494
|
+
let removed = 0;
|
|
495
|
+
let added = 0;
|
|
496
|
+
let i = 0;
|
|
497
|
+
let j = 0;
|
|
498
|
+
while (i < a.length || j < b.length) {
|
|
499
|
+
if (i < a.length && j < b.length && a[i] === b[j]) {
|
|
500
|
+
out.push(` ${a[i]}`);
|
|
501
|
+
i++;
|
|
502
|
+
j++;
|
|
503
|
+
} else if (i < a.length && (j >= b.length || lcs[i + 1]![j]! >= lcs[i]![j + 1]!)) {
|
|
504
|
+
out.push(`-${a[i]}`);
|
|
505
|
+
removed++;
|
|
506
|
+
i++;
|
|
507
|
+
} else {
|
|
508
|
+
out.push(`+${b[j]}`);
|
|
509
|
+
added++;
|
|
510
|
+
j++;
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
return { text: out.join("\n"), removed, added };
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/** The sha-1 of a test's own lines (`testBlock`), or null when `text` holds no such test. */
|
|
517
|
+
export function blockHash(text: string | null, name: string): string | null {
|
|
518
|
+
const block = testBlock(text, name);
|
|
519
|
+
return block === null ? null : createHash("sha1").update(block).digest("hex");
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/** The commit the group started at: its `build-group-start` row's `parentSha`. */
|
|
523
|
+
function groupStartSha(file: string, workflow: string, group: number): string | undefined {
|
|
524
|
+
if (!fs.existsSync(file)) return undefined;
|
|
525
|
+
const row = (readStatusRows(file) as Array<Record<string, unknown>>)
|
|
526
|
+
.filter(
|
|
527
|
+
(r) =>
|
|
528
|
+
r.type === "step-done" &&
|
|
529
|
+
r.workflow === workflow &&
|
|
530
|
+
r.step === "build-group-start" &&
|
|
531
|
+
r.group === group,
|
|
532
|
+
)
|
|
533
|
+
.at(-1);
|
|
534
|
+
return typeof row?.parentSha === "string" ? row.parentSha : undefined;
|
|
535
|
+
}
|
|
149
536
|
|
|
150
537
|
const NO_TEST_FILE_HINT =
|
|
151
538
|
`declare "${BKIT_TEST_FILE_SCRIPT}" in package.json: it runs the one test file passed as its ` +
|
|
@@ -156,7 +543,10 @@ export type CaseState = "passed" | "failure" | "error" | "skipped";
|
|
|
156
543
|
export interface JunitCase {
|
|
157
544
|
name: string;
|
|
158
545
|
state: CaseState;
|
|
546
|
+
/** The failure element's `message` attribute: the runner's one-line summary. */
|
|
159
547
|
message: string;
|
|
548
|
+
/** The failure element's text: the assertion output, where some runners put the detail. */
|
|
549
|
+
detail: string;
|
|
160
550
|
}
|
|
161
551
|
|
|
162
552
|
const XML_ENTITIES: Record<string, string> = {
|
|
@@ -182,18 +572,27 @@ function attr(attrs: string, name: string): string | undefined {
|
|
|
182
572
|
return m ? decodeXml(m[1] ?? m[2] ?? "") : undefined;
|
|
183
573
|
}
|
|
184
574
|
|
|
185
|
-
/**
|
|
575
|
+
/** CDATA text as escaped character data, so a tag inside it never ends an element early. */
|
|
576
|
+
const escapeCdata = (xml: string) =>
|
|
577
|
+
xml.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, (_, text: string) =>
|
|
578
|
+
text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">"),
|
|
579
|
+
);
|
|
580
|
+
|
|
581
|
+
/** Every `<testcase>` in a JUnit report: its name, outcome, first failure message and detail. */
|
|
186
582
|
export function parseJunit(xml: string): JunitCase[] {
|
|
187
|
-
const body = xml
|
|
583
|
+
const body = escapeCdata(xml).replace(/<!--[\s\S]*?-->/g, "");
|
|
188
584
|
const cases: JunitCase[] = [];
|
|
189
585
|
const testcase = new RegExp(String.raw`<testcase${ATTRS}\s*(?:\/>|>([\s\S]*?)<\/testcase>)`, "g");
|
|
190
|
-
const outcome = new RegExp(
|
|
586
|
+
const outcome = new RegExp(
|
|
587
|
+
String.raw`<(failure|error|skipped)${ATTRS}\s*(?:\/>|>([\s\S]*?)<\/\1>)`,
|
|
588
|
+
);
|
|
191
589
|
for (const m of body.matchAll(testcase)) {
|
|
192
590
|
const child = outcome.exec(m[2] ?? "");
|
|
193
591
|
cases.push({
|
|
194
592
|
name: attr(m[1] ?? "", "name") ?? "",
|
|
195
593
|
state: child ? (child[1] as CaseState) : "passed",
|
|
196
594
|
message: child ? (attr(child[2] ?? "", "message") ?? "") : "",
|
|
595
|
+
detail: child ? decodeXml((child[3] ?? "").replace(/<[^>]*>/g, "")).trim() : "",
|
|
197
596
|
});
|
|
198
597
|
}
|
|
199
598
|
return cases;
|
|
@@ -211,36 +610,51 @@ const caseNames = (cases: JunitCase[]) =>
|
|
|
211
610
|
.map((c) => `"${c.name}"`)
|
|
212
611
|
.join(", ")}${cases.length > 5 ? ", …" : ""}`;
|
|
213
612
|
|
|
214
|
-
/**
|
|
215
|
-
|
|
613
|
+
/** The stem a load error names: the test file's base name without its extension. */
|
|
614
|
+
const fileStem = (file: string) => path.posix.basename(file).replace(/\.[^.]*$/, "");
|
|
216
615
|
|
|
217
|
-
/**
|
|
616
|
+
/** The red states `judgeRed` tells apart. */
|
|
617
|
+
export type JudgedRed = Extract<
|
|
618
|
+
RedState,
|
|
619
|
+
"red" | "red-not-loaded" | "vacuous" | "missing" | "no-report"
|
|
620
|
+
>;
|
|
621
|
+
|
|
622
|
+
/**
|
|
623
|
+
* How `want` stood in a red run of `file`, from its JUnit report: `red` (the named test
|
|
624
|
+
* failed or errored), `red-not-loaded` (the file ran no test, or the only cases are failures
|
|
625
|
+
* named after the file itself: a load error, which an import of unwritten code is),
|
|
626
|
+
* `vacuous` (the named test passed, so it proves nothing before the task's code), `missing`
|
|
627
|
+
* (it did not run, or was skipped) or `no-report`.
|
|
628
|
+
*/
|
|
218
629
|
export function judgeRed(
|
|
219
|
-
exitCode: number,
|
|
220
630
|
cases: JunitCase[] | null,
|
|
221
631
|
want: string,
|
|
222
|
-
|
|
632
|
+
file: string,
|
|
633
|
+
): { state: JudgedRed; detail: string } {
|
|
223
634
|
if (cases === null)
|
|
224
635
|
return {
|
|
225
|
-
|
|
226
|
-
|
|
636
|
+
state: "no-report",
|
|
637
|
+
detail: `no JUnit report at $${REPORT_ENV} — ${BKIT_TEST_FILE_SCRIPT} must write one`,
|
|
227
638
|
};
|
|
228
639
|
const named = cases.filter((c) => isNamedCase(c.name, want));
|
|
229
|
-
if (named.length === 0)
|
|
230
|
-
|
|
640
|
+
if (named.length === 0) {
|
|
641
|
+
const stem = fileStem(file);
|
|
642
|
+
if (cases.length === 0 || cases.every((c) => c.state !== "passed" && c.name.includes(stem)))
|
|
643
|
+
return {
|
|
644
|
+
state: "red-not-loaded",
|
|
645
|
+
detail: `\`${file}\` did not load${cases[0]?.message ? `: ${cases[0].message}` : ""}`,
|
|
646
|
+
};
|
|
647
|
+
return { state: "missing", detail: `"${want}" did not run; ${caseNames(cases)}` };
|
|
648
|
+
}
|
|
231
649
|
if (named.some((c) => c.state === "passed"))
|
|
232
650
|
return {
|
|
233
|
-
|
|
234
|
-
|
|
651
|
+
state: "vacuous",
|
|
652
|
+
detail: `"${want}" passes at its red, so it proves nothing`,
|
|
235
653
|
};
|
|
236
654
|
if (named.some((c) => c.state === "skipped"))
|
|
237
|
-
return {
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
kind: "exit-mismatch",
|
|
241
|
-
reason: `"${want}" failed in the report but the script exited 0`,
|
|
242
|
-
};
|
|
243
|
-
return null;
|
|
655
|
+
return { state: "missing", detail: `"${want}" was skipped, not run` };
|
|
656
|
+
const failed = named.find((c) => c.state !== "passed")!;
|
|
657
|
+
return { state: "red", detail: `${failed.state}${failed.message ? `: ${failed.message}` : ""}` };
|
|
244
658
|
}
|
|
245
659
|
|
|
246
660
|
/** Why a run is not green for `want`, or null when every case of the named test passed. */
|
|
@@ -260,7 +674,7 @@ export interface PlannedTest {
|
|
|
260
674
|
name: string;
|
|
261
675
|
}
|
|
262
676
|
|
|
263
|
-
/** A human-recorded acceptance of a task's
|
|
677
|
+
/** A human-recorded acceptance of a task's missing reds: the tests it covers, its 1-based plan lines. */
|
|
264
678
|
export interface TestOrderException {
|
|
265
679
|
tests: PlannedTest[];
|
|
266
680
|
lines: number[];
|
|
@@ -271,12 +685,9 @@ export type TestFirst =
|
|
|
271
685
|
| { task: number; kind: "na"; reason: string }
|
|
272
686
|
| { task: number; kind: "missing" | "malformed" };
|
|
273
687
|
|
|
274
|
-
/** The task field naming a task's tests, and the one recording an accepted
|
|
688
|
+
/** The task field naming a task's tests, and the one recording an accepted missing red. */
|
|
275
689
|
export const TEST_FIRST = "**TEST FIRST:**";
|
|
276
690
|
export const TEST_EXCEPTION = "**TEST EXCEPTION:**";
|
|
277
|
-
/** Why `tests green` refuses a plan with no group and task blocks. */
|
|
278
|
-
export const NO_MARKERS_REASON =
|
|
279
|
-
"this plan has no task markers; run /bkit-build again to convert it";
|
|
280
691
|
|
|
281
692
|
/** The plan caps `package.json` `bkit.workflows.tasks` sets for the checkout `cwd` sits in. */
|
|
282
693
|
export const planCaps = (cwd: string): TaskCaps => readBkitConfig(toplevel(cwd)).tasks;
|
|
@@ -286,6 +697,30 @@ export const capsText = (c: TaskCaps) =>
|
|
|
286
697
|
`${c.maxPerGroup} tasks and ${c.maxTestsPerGroup} TEST FIRST tests per group; ${c.maxTestsPerTask} tests across ${c.maxTestFilesPerTask} test files per task`;
|
|
287
698
|
|
|
288
699
|
/** A task block's TEST FIRST read as one `TestFirst`; a TEST EXCEPTION with no TEST FIRST is its `n/a`. */
|
|
700
|
+
/**
|
|
701
|
+
* Planned tests of open tasks whose name a test in their file already carries: the tests
|
|
702
|
+
* step cannot write a new test under that name, so it either updates the existing one or
|
|
703
|
+
* finds it passing. Reported only — the plan names the update, or a new name.
|
|
704
|
+
*/
|
|
705
|
+
function existingTestNames(
|
|
706
|
+
text: string,
|
|
707
|
+
top: string,
|
|
708
|
+
): Array<{ line: number; task: number; test: string; name: string }> {
|
|
709
|
+
const out: Array<{ line: number; task: number; test: string; name: string }> = [];
|
|
710
|
+
for (const t of parsePlanBlocks(text).tasks) {
|
|
711
|
+
if (t.done) continue;
|
|
712
|
+
const s = testFirstOf(t);
|
|
713
|
+
if (s.kind !== "test") continue;
|
|
714
|
+
for (const x of s.tests) {
|
|
715
|
+
const abs = path.join(top, x.test);
|
|
716
|
+
if (!fs.existsSync(abs)) continue;
|
|
717
|
+
if (testBlock(fs.readFileSync(abs, "utf-8"), x.name) !== null)
|
|
718
|
+
out.push({ line: t.testFirst?.line ?? t.line, task: t.task, test: x.test, name: x.name });
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
return out;
|
|
722
|
+
}
|
|
723
|
+
|
|
289
724
|
function testFirstOf(t: PlanTaskBlock): TestFirst {
|
|
290
725
|
const { task, testFirst: tf, testException: exc } = t;
|
|
291
726
|
const plain = (tests: PlannedTest[]) => tests.map(({ test, name }) => ({ test, name }));
|
|
@@ -304,7 +739,7 @@ function testFirstOf(t: PlanTaskBlock): TestFirst {
|
|
|
304
739
|
* Each task block's TEST FIRST field (`plan-blocks.ts`): one test per sub-bullet (first
|
|
305
740
|
* backticked span = file, second = test name), `n/a — {reason}`, missing, or malformed
|
|
306
741
|
* (an entry with fewer than two spans, or no entry). A TEST EXCEPTION field names, in
|
|
307
|
-
* the same entry form, the tests whose
|
|
742
|
+
* the same entry form, the tests whose missing reds it accepts. `group` narrows to
|
|
308
743
|
* one group's tasks.
|
|
309
744
|
*/
|
|
310
745
|
export function readTestFirst(plan: string, group?: number): TestFirst[] {
|
|
@@ -341,7 +776,16 @@ export interface GroupSize {
|
|
|
341
776
|
export interface PlanProblem {
|
|
342
777
|
/** 1-based plan line. */
|
|
343
778
|
line: number;
|
|
344
|
-
kind:
|
|
779
|
+
kind:
|
|
780
|
+
| "format"
|
|
781
|
+
| "stray"
|
|
782
|
+
| "field"
|
|
783
|
+
| "depends"
|
|
784
|
+
| "cap"
|
|
785
|
+
| "inventory"
|
|
786
|
+
| "matrix"
|
|
787
|
+
| "supersession"
|
|
788
|
+
| "markers";
|
|
345
789
|
message: string;
|
|
346
790
|
}
|
|
347
791
|
|
|
@@ -370,7 +814,8 @@ const kb1 = (bytes: number) => Math.round((bytes / 1024) * 10) / 10;
|
|
|
370
814
|
* task-like text outside every block, a task without DONE WHEN or without TEST FIRST and
|
|
371
815
|
* TEST EXCEPTION, a malformed TEST FIRST, a DEPENDS ON naming a missing task or one in a
|
|
372
816
|
* later group, a group or task over a cap in `caps`, a CREATE file with no NEW Reuse
|
|
373
|
-
* Inventory row
|
|
817
|
+
* Inventory row, a `Derived Test Matrix` row naming no task / an undefined task / two
|
|
818
|
+
* tasks or a test-bearing task no row names, and a DELETE file with no Supersession row.
|
|
374
819
|
*/
|
|
375
820
|
export function checkPlanText(text: string, caps: TaskCaps = CONFIG_DEFAULTS.tasks): PlanCheck {
|
|
376
821
|
const blocks = parsePlanBlocks(text);
|
|
@@ -446,6 +891,62 @@ export function checkPlanText(text: string, caps: TaskCaps = CONFIG_DEFAULTS.tas
|
|
|
446
891
|
message: `TASK ${t.task}: CREATE in \`${c.file}\` has no NEW row naming it in the Reuse Inventory's File column`,
|
|
447
892
|
});
|
|
448
893
|
}
|
|
894
|
+
// The matrix maps each row to one task and each test-bearing task to a row. A plan with no
|
|
895
|
+
// matrix section yields no problem: the refusal is on a row, and issue-form plans carry none.
|
|
896
|
+
const matrix = derivedTestMatrix(text);
|
|
897
|
+
if (matrix !== null) {
|
|
898
|
+
if (matrix.taskColumn === null)
|
|
899
|
+
problems.push({
|
|
900
|
+
line: matrix.line,
|
|
901
|
+
kind: "matrix",
|
|
902
|
+
message:
|
|
903
|
+
"the Derived Test Matrix header has no `Task` column; add one naming the task each row's test belongs to",
|
|
904
|
+
});
|
|
905
|
+
else {
|
|
906
|
+
const mapped = new Set<number>();
|
|
907
|
+
for (const row of matrix.rows) {
|
|
908
|
+
const named = row.tasks.map((n) => `TASK ${n}`);
|
|
909
|
+
if (row.tasks.length === 0)
|
|
910
|
+
problems.push({
|
|
911
|
+
line: row.number,
|
|
912
|
+
kind: "matrix",
|
|
913
|
+
message: `Derived Test Matrix row maps to no task: name one in its \`Task\` cell`,
|
|
914
|
+
});
|
|
915
|
+
else if (row.tasks.length > 1)
|
|
916
|
+
problems.push({
|
|
917
|
+
line: row.number,
|
|
918
|
+
kind: "matrix",
|
|
919
|
+
message: `Derived Test Matrix row maps to ${named.join(" and ")}; name exactly one task`,
|
|
920
|
+
});
|
|
921
|
+
else {
|
|
922
|
+
const n = row.tasks[0]!;
|
|
923
|
+
if (!byTask.has(n))
|
|
924
|
+
problems.push({
|
|
925
|
+
line: row.number,
|
|
926
|
+
kind: "matrix",
|
|
927
|
+
message: `Derived Test Matrix row maps to TASK ${n}, which the plan does not define`,
|
|
928
|
+
});
|
|
929
|
+
mapped.add(n);
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
for (const t of blocks.tasks)
|
|
933
|
+
if (specs.get(t.task)!.kind === "test" && !mapped.has(t.task))
|
|
934
|
+
problems.push({
|
|
935
|
+
line: t.line,
|
|
936
|
+
kind: "matrix",
|
|
937
|
+
message: `TASK ${t.task} names TEST FIRST tests but no Derived Test Matrix row maps to it`,
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
const superseded = new Set((supersessionInventory(text) ?? []).flatMap((r) => r.files));
|
|
942
|
+
for (const t of blocks.tasks)
|
|
943
|
+
for (const d of t.delete)
|
|
944
|
+
if (!superseded.has(path.posix.normalize(d.file)))
|
|
945
|
+
problems.push({
|
|
946
|
+
line: d.line,
|
|
947
|
+
kind: "supersession",
|
|
948
|
+
message: `TASK ${t.task}: DELETE in \`${d.file}\` has no row naming it in the Supersession Inventory`,
|
|
949
|
+
});
|
|
449
950
|
const groups: GroupSize[] = blocks.groups.map((g) => {
|
|
450
951
|
const tests = sizes.filter((s) => s.group === g.group).reduce((n, s) => n + s.tests, 0);
|
|
451
952
|
const bytes = Buffer.byteLength(lines.slice(g.line - 1, g.end).join("\n"));
|
|
@@ -617,29 +1118,6 @@ export function planExistence(text: string, root: string): ExistenceFinding[] {
|
|
|
617
1118
|
return out.sort((a, b) => a.line - b.line);
|
|
618
1119
|
}
|
|
619
1120
|
|
|
620
|
-
/** True when `after` keeps every non-blank line of `before`, in order, ignoring whitespace: lines were only added. */
|
|
621
|
-
export function onlyAddsLines(before: string, after: string): boolean {
|
|
622
|
-
const norm = (text: string) =>
|
|
623
|
-
text
|
|
624
|
-
.split("\n")
|
|
625
|
-
.map((l) => l.replace(/\s+/g, ""))
|
|
626
|
-
.filter(Boolean);
|
|
627
|
-
const want = norm(before);
|
|
628
|
-
let kept = 0;
|
|
629
|
-
for (const line of norm(after)) if (kept < want.length && line === want[kept]) kept++;
|
|
630
|
-
return kept === want.length;
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
/** `test` as a repo-relative path inside `top`; usage error when absolute, outside, or absent. */
|
|
634
|
-
function repoTestPath(top: string, test: string): string {
|
|
635
|
-
const rel = path.relative(top, path.resolve(top, test));
|
|
636
|
-
if (path.isAbsolute(test) || rel.startsWith("..") || rel === "")
|
|
637
|
-
throw new CommandError(EXIT.USAGE, `--test must be a path inside the repo, got "${test}"`);
|
|
638
|
-
if (!fs.statSync(path.join(top, rel), { throwIfNoEntry: false })?.isFile())
|
|
639
|
-
throw new CommandError(EXIT.USAGE, `--test ${rel} is not a file`);
|
|
640
|
-
return rel.split(path.sep).join("/");
|
|
641
|
-
}
|
|
642
|
-
|
|
643
1121
|
/** Run one test file through `bkit:test:file`; the report is read and removed. */
|
|
644
1122
|
function runTestFile(
|
|
645
1123
|
top: string,
|
|
@@ -659,72 +1137,86 @@ function runTestFile(
|
|
|
659
1137
|
const couldNotRunMessage = (exitCode: number) =>
|
|
660
1138
|
`${gateScope(BKIT_TEST_FILE_SCRIPT)} could not run (exit ${exitCode}: ${exitCode === 126 ? "not executable" : "command not found"}) — no test executed`;
|
|
661
1139
|
|
|
662
|
-
/** The copy of the test file a red keeps beside its log. */
|
|
663
|
-
export const snapshotPath = (log: string) => `${log.replace(/\.md$/, "")}.snapshot`;
|
|
664
|
-
|
|
665
1140
|
const toplevel = (cwd: string) => git(cwd, ["rev-parse", "--show-toplevel"]) ?? cwd;
|
|
666
1141
|
|
|
667
1142
|
/** A fence longer than any backtick run in `text`. */
|
|
668
1143
|
const fenceFor = (text: string) =>
|
|
669
1144
|
"`".repeat(Math.max(3, ...[...text.matchAll(/`+/g)].map((m) => m[0].length + 1)));
|
|
670
1145
|
|
|
1146
|
+
/**
|
|
1147
|
+
* Every gap the audit emits, and the verifier's `Fixable` for it: `null` = the
|
|
1148
|
+
* answer needs a diff read, which only the verifier can make. A test with no proven red
|
|
1149
|
+
* is no gap: the audit lists it as unproven, and the owner answers every unproven test
|
|
1150
|
+
* once, at the end of the build.
|
|
1151
|
+
*/
|
|
1152
|
+
const GAP_FIXABLE = {
|
|
1153
|
+
"not-green": true,
|
|
1154
|
+
"delete-still-referenced": null,
|
|
1155
|
+
"no-test-first": false,
|
|
1156
|
+
"test-first-malformed": false,
|
|
1157
|
+
"test-first-unread": false,
|
|
1158
|
+
"exception-refused": false,
|
|
1159
|
+
} as const satisfies Record<string, boolean | null>;
|
|
1160
|
+
|
|
1161
|
+
type GapKind = keyof typeof GAP_FIXABLE;
|
|
1162
|
+
|
|
671
1163
|
interface Gap {
|
|
672
1164
|
/** null for a plan-level gap (`test-first-unread`). */
|
|
673
1165
|
task: number | null;
|
|
674
|
-
kind:
|
|
1166
|
+
kind: GapKind;
|
|
675
1167
|
detail: string;
|
|
1168
|
+
fixable: boolean | null;
|
|
1169
|
+
/** The line the verifier appends, carrying `Fixable: null` for a decision it makes itself. */
|
|
1170
|
+
logLine: string;
|
|
676
1171
|
}
|
|
677
1172
|
|
|
678
|
-
/**
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
]
|
|
1173
|
+
/** One gap, with the `fixable` its kind settles and the ready-to-append log line. */
|
|
1174
|
+
function gapOf(
|
|
1175
|
+
task: number | null,
|
|
1176
|
+
kind: GapKind,
|
|
1177
|
+
detail: string,
|
|
1178
|
+
test?: { file: string; name: string },
|
|
1179
|
+
): Gap {
|
|
1180
|
+
const fixable = GAP_FIXABLE[kind];
|
|
1181
|
+
const subject = test === undefined ? "" : `\`${test.file}\` \`${test.name}\` — `;
|
|
1182
|
+
return {
|
|
1183
|
+
task,
|
|
1184
|
+
kind,
|
|
1185
|
+
detail,
|
|
1186
|
+
fixable,
|
|
1187
|
+
logLine: `${task === null ? "Plan" : `TASK ${task}`}: ${subject}${kind} — Fixable: ${fixable}`,
|
|
1188
|
+
};
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
/** The report's verdict from its gaps: a decision one gap still needs (`null`) outranks one it refuses. */
|
|
1192
|
+
const verdictOf = (gaps: Gap[]): "PASS" | "FAIL" | "CONCERNS" | null =>
|
|
1193
|
+
gaps.length === 0
|
|
1194
|
+
? "PASS"
|
|
1195
|
+
: gaps.some((g) => g.fixable === null)
|
|
1196
|
+
? null
|
|
1197
|
+
: gaps.some((g) => g.fixable === false)
|
|
1198
|
+
? "FAIL"
|
|
1199
|
+
: "CONCERNS";
|
|
686
1200
|
|
|
687
1201
|
/** The gap for a task whose only test field is a TEST EXCEPTION that is not honoured. */
|
|
688
1202
|
export const EXCEPTION_REFUSED = "exception-refused";
|
|
689
1203
|
|
|
690
1204
|
/** The status row step plan-execute's finish writes; its commit carries the code of each task its log records DONE. */
|
|
691
|
-
export const
|
|
1205
|
+
export const PLAN_TASKS_STEP = "build-plan-tasks";
|
|
692
1206
|
|
|
693
1207
|
const repoRel = (top: string, cwd: string, p: string) =>
|
|
694
1208
|
path.relative(top, path.resolve(cwd, p)).split(path.sep).join("/");
|
|
695
1209
|
|
|
696
1210
|
/**
|
|
697
|
-
*
|
|
698
|
-
*
|
|
699
|
-
*
|
|
1211
|
+
* The task numbers a plan-execute log records `DONE — TASK N` for, in log order. `verbs`
|
|
1212
|
+
* widens the reader to the log's other closing lines (`BLOCKED`), so one parser serves
|
|
1213
|
+
* every caller; the default reads `DONE` only, which is what "the task is done" means.
|
|
700
1214
|
*/
|
|
701
|
-
function
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
): Map<number, string> {
|
|
707
|
-
const statusRel = repoRel(top, cwd, file);
|
|
708
|
-
const commits = new Map<number, string>();
|
|
709
|
-
for (const r of readStatusRows(file) as Array<Record<string, unknown>>) {
|
|
710
|
-
if (r.type !== "step-done" || r.workflow !== workflow || r.step !== PLAN_EXECUTE_STEP) continue;
|
|
711
|
-
const log = Array.isArray(r.artifacts) ? r.artifacts.map(String)[0] : undefined;
|
|
712
|
-
if (typeof r.parentSha !== "string" || log === undefined) continue;
|
|
713
|
-
const range = `${r.parentSha}..HEAD`;
|
|
714
|
-
const commit = git(top, ["rev-list", "--reverse", "--ancestry-path", range, "--", statusRel])
|
|
715
|
-
?.split("\n")
|
|
716
|
-
.find(Boolean);
|
|
717
|
-
const text =
|
|
718
|
-
commit === undefined ? null : git(top, ["show", `${commit}:${repoRel(top, cwd, log)}`]);
|
|
719
|
-
if (commit === undefined || text === null) continue;
|
|
720
|
-
for (const n of doneTasks(text)) if (!commits.has(n)) commits.set(n, commit);
|
|
721
|
-
}
|
|
722
|
-
return commits;
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
/** The task numbers a plan-execute log records `DONE — TASK N` for, in log order. */
|
|
726
|
-
export function doneTasks(log: string): number[] {
|
|
727
|
-
return [...log.matchAll(/^DONE\s*[—–-]+\s*TASK\s+(\d+)\b/gim)].map((m) => Number(m[1]));
|
|
1215
|
+
export function doneTasks(log: string, verbs: readonly string[] = ["DONE"]): number[] {
|
|
1216
|
+
const closing = verbs.map((v) => v.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|");
|
|
1217
|
+
return [...log.matchAll(new RegExp(`^(?:${closing})\\s*[—–-]+\\s*TASK\\s+(\\d+)\\b`, "gim"))].map(
|
|
1218
|
+
(m) => Number(m[1]),
|
|
1219
|
+
);
|
|
728
1220
|
}
|
|
729
1221
|
|
|
730
1222
|
/**
|
|
@@ -855,8 +1347,7 @@ function deleteGaps(
|
|
|
855
1347
|
)
|
|
856
1348
|
.map((c) => path.posix.normalize(c.file))
|
|
857
1349
|
: [];
|
|
858
|
-
const gap = (detail: string) =>
|
|
859
|
-
gaps.push({ task: t.task, kind: DELETE_STILL_REFERENCED, detail });
|
|
1350
|
+
const gap = (detail: string) => gaps.push(gapOf(t.task, DELETE_STILL_REFERENCED, detail));
|
|
860
1351
|
const abs = path.join(top, file);
|
|
861
1352
|
const exists = fs.statSync(abs, { throwIfNoEntry: false })?.isFile() === true;
|
|
862
1353
|
let what: string;
|
|
@@ -901,19 +1392,150 @@ interface AuditRow {
|
|
|
901
1392
|
task: number;
|
|
902
1393
|
test: string;
|
|
903
1394
|
red: string;
|
|
904
|
-
changed: string;
|
|
905
1395
|
green: string;
|
|
906
1396
|
}
|
|
907
1397
|
|
|
1398
|
+
/** A planned test whose pass the tests step accepted: no red is due for it. */
|
|
1399
|
+
interface Landed {
|
|
1400
|
+
task: number;
|
|
1401
|
+
test: string;
|
|
1402
|
+
name: string;
|
|
1403
|
+
reason: string;
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
/** A planned test with no proven red: the owner answers it at the end of the build. */
|
|
1407
|
+
interface Unproven {
|
|
1408
|
+
task: number;
|
|
1409
|
+
test: string;
|
|
1410
|
+
name: string;
|
|
1411
|
+
detail: string;
|
|
1412
|
+
}
|
|
1413
|
+
|
|
908
1414
|
/**
|
|
909
|
-
*
|
|
910
|
-
*
|
|
1415
|
+
* An existing test the tests step changed: the verifier judges its diff since `base`.
|
|
1416
|
+
* `additive` = the test's own lines since `base` gained lines and lost none (null when
|
|
1417
|
+
* either side cannot be read); `diff` = those lines' unified diff.
|
|
911
1418
|
*/
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
1419
|
+
interface Updated {
|
|
1420
|
+
task: number;
|
|
1421
|
+
test: string;
|
|
1422
|
+
name: string;
|
|
1423
|
+
base: string | null;
|
|
1424
|
+
additive: boolean | null;
|
|
1425
|
+
diff: string | null;
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
/** The `Updated` entry for a planned test: its own lines at `base` against the file now. */
|
|
1429
|
+
function updatedEntry(
|
|
1430
|
+
top: string,
|
|
1431
|
+
task: number,
|
|
1432
|
+
t: { test: string; name: string },
|
|
1433
|
+
base: string | null,
|
|
1434
|
+
): Updated {
|
|
1435
|
+
const entry: Updated = { task, test: t.test, name: t.name, base, additive: null, diff: null };
|
|
1436
|
+
if (base === null) return entry;
|
|
1437
|
+
const shown = runGit(top, ["show", `${base}:${t.test}`]);
|
|
1438
|
+
const abs = path.join(top, t.test);
|
|
1439
|
+
if (!shown.ok || !fs.existsSync(abs)) return entry;
|
|
1440
|
+
const was = testBlock(shown.stdout, t.name);
|
|
1441
|
+
const now = testBlock(fs.readFileSync(abs, "utf-8"), t.name);
|
|
1442
|
+
if (was === null || now === null) return entry;
|
|
1443
|
+
const d = lineDiff(was, now);
|
|
1444
|
+
return { ...entry, additive: d.removed === 0, diff: d.text };
|
|
1445
|
+
}
|
|
1446
|
+
|
|
1447
|
+
/** A planned test the group's red did not prove, and why. */
|
|
1448
|
+
interface RedGap {
|
|
1449
|
+
task: number;
|
|
1450
|
+
test: string;
|
|
1451
|
+
name: string;
|
|
1452
|
+
kind: Extract<RedState, "vacuous" | "missing" | "no-report" | "lint">;
|
|
1453
|
+
detail: string;
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
/** A `PASSES — TASK N: \`file\` \`name\`: reason` line of the tests step's log. */
|
|
1457
|
+
interface PassesLine extends PlannedTest {
|
|
1458
|
+
task: number;
|
|
1459
|
+
reason: string;
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
/** Every PASSES line of a tests step's log: the tests the writer saw passing and vouches for. */
|
|
1463
|
+
function passesLines(log: string): PassesLine[] {
|
|
1464
|
+
return [
|
|
1465
|
+
...log.matchAll(/^PASSES\s*[—–-]+\s*TASK\s+(\d+):\s*`([^`]+)`\s+`([^`]+)`\s*:\s*(.+)$/gm),
|
|
1466
|
+
].map((m) => ({
|
|
1467
|
+
task: Number(m[1]),
|
|
1468
|
+
test: m[2]!.trim(),
|
|
1469
|
+
name: m[3]!.trim(),
|
|
1470
|
+
reason: m[4]!.trim(),
|
|
1471
|
+
}));
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
/** Why a plan cannot be read for its tests (no file, no task markers, a fence never closed), or undefined. */
|
|
1475
|
+
const planRefusal = (
|
|
1476
|
+
plan: string | undefined,
|
|
1477
|
+
parsed: ReturnType<typeof parsePlanBlocks>,
|
|
1478
|
+
): string | undefined =>
|
|
1479
|
+
plan === undefined
|
|
1480
|
+
? `no plan file; ${NO_MARKERS_REASON}`
|
|
1481
|
+
: parsed.tasks.length === 0
|
|
1482
|
+
? parsed.fenceOpen === undefined
|
|
1483
|
+
? NO_MARKERS_REASON
|
|
1484
|
+
: fenceNeverClosed(parsed.fenceOpen)
|
|
1485
|
+
: undefined;
|
|
1486
|
+
|
|
1487
|
+
/** A table cell: pipes escaped, line breaks folded. */
|
|
1488
|
+
const cell = (s: string) => s.replace(/\s+/g, " ").replace(/\|/g, "\\|").trim();
|
|
1489
|
+
|
|
1490
|
+
function writeRedReport(
|
|
1491
|
+
log: string,
|
|
1492
|
+
o: {
|
|
1493
|
+
enforced: boolean;
|
|
1494
|
+
reason?: string;
|
|
1495
|
+
plan?: string;
|
|
1496
|
+
group: number;
|
|
1497
|
+
attempt: number;
|
|
1498
|
+
tests: RedTest[];
|
|
1499
|
+
details: Map<RedTest, string>;
|
|
1500
|
+
gaps: RedGap[];
|
|
1501
|
+
outputs: Array<{ title: string; text: string }>;
|
|
1502
|
+
},
|
|
1503
|
+
): void {
|
|
1504
|
+
const red = o.tests.filter((t) => t.state === "red" || t.state === "red-not-loaded").length;
|
|
1505
|
+
const accepted = o.tests.filter((t) => t.state === "passed-accepted").length;
|
|
1506
|
+
const updated = o.tests.filter((t) => t.state === "updated").length;
|
|
1507
|
+
const out = [
|
|
1508
|
+
`Enforced: ${o.enforced}`,
|
|
1509
|
+
`Red: ${red}`,
|
|
1510
|
+
`Accepted: ${accepted}`,
|
|
1511
|
+
`Updated: ${updated}`,
|
|
1512
|
+
`Gaps: ${o.gaps.length}`,
|
|
1513
|
+
"",
|
|
1514
|
+
"<!-- verdict:end -->",
|
|
1515
|
+
"",
|
|
1516
|
+
`# Tests Red — group ${o.group}, attempt ${o.attempt}`,
|
|
1517
|
+
"",
|
|
1518
|
+
];
|
|
1519
|
+
if (o.plan) out.push(`Plan: \`${o.plan}\``, "");
|
|
1520
|
+
if (!o.enforced) out.push(`Not enforced: ${o.reason}.`, "");
|
|
1521
|
+
if (o.tests.length > 0) {
|
|
1522
|
+
out.push("| Task | Test | State | Detail |", "| --- | --- | --- | --- |");
|
|
1523
|
+
for (const t of o.tests)
|
|
1524
|
+
out.push(
|
|
1525
|
+
`| ${t.task} | \`${t.test}\` "${cell(t.testName)}" | ${t.state} | ${cell(o.details.get(t) ?? "")} |`,
|
|
1526
|
+
);
|
|
1527
|
+
out.push("");
|
|
1528
|
+
}
|
|
1529
|
+
out.push("## Gaps", "");
|
|
1530
|
+
if (o.gaps.length === 0) out.push("None.");
|
|
1531
|
+
for (const g of o.gaps)
|
|
1532
|
+
out.push(`- TASK ${g.task} \`${g.test}\` "${g.name}" \`${g.kind}\` — ${g.detail}`);
|
|
1533
|
+
for (const t of o.outputs) {
|
|
1534
|
+
const fence = fenceFor(t.text);
|
|
1535
|
+
out.push("", `## ${t.title}`, "", fence, t.text.replace(/\n$/, ""), fence);
|
|
1536
|
+
}
|
|
1537
|
+
fs.mkdirSync(path.dirname(log), { recursive: true });
|
|
1538
|
+
fs.writeFileSync(log, `${out.join("\n")}\n`);
|
|
917
1539
|
}
|
|
918
1540
|
|
|
919
1541
|
function writeAuditReport(
|
|
@@ -922,19 +1544,21 @@ function writeAuditReport(
|
|
|
922
1544
|
enforced: boolean;
|
|
923
1545
|
reason?: string;
|
|
924
1546
|
plan?: string;
|
|
1547
|
+
verdict: "PASS" | "FAIL" | "CONCERNS" | null;
|
|
925
1548
|
rows: AuditRow[];
|
|
926
1549
|
gaps: Gap[];
|
|
927
|
-
testOrderOnly: boolean;
|
|
928
1550
|
exceptions: ExceptionResult[];
|
|
929
|
-
|
|
930
|
-
|
|
1551
|
+
landed: Landed[];
|
|
1552
|
+
unproven: Unproven[];
|
|
1553
|
+
updated: Updated[];
|
|
931
1554
|
outputs: Array<{ test: string; tail: string }>;
|
|
932
1555
|
},
|
|
933
1556
|
): void {
|
|
934
1557
|
const out = [
|
|
935
1558
|
`Enforced: ${o.enforced}`,
|
|
1559
|
+
`Verdict: ${o.verdict}`,
|
|
936
1560
|
`Gaps: ${o.gaps.length}`,
|
|
937
|
-
`
|
|
1561
|
+
`Unproven: ${o.unproven.length}`,
|
|
938
1562
|
"",
|
|
939
1563
|
"<!-- verdict:end -->",
|
|
940
1564
|
"",
|
|
@@ -943,18 +1567,16 @@ function writeAuditReport(
|
|
|
943
1567
|
if (o.plan) out.push(`Plan: \`${o.plan}\``, "");
|
|
944
1568
|
if (!o.enforced) out.push(`Not enforced: ${o.reason}.`, "");
|
|
945
1569
|
if (o.rows.length > 0) {
|
|
946
|
-
out.push(
|
|
947
|
-
|
|
948
|
-
"| --- | --- | --- | --- | --- |",
|
|
949
|
-
);
|
|
950
|
-
for (const r of o.rows)
|
|
951
|
-
out.push(`| ${r.task} | ${r.test} | ${r.red} | ${r.changed} | ${r.green} |`);
|
|
1570
|
+
out.push("| Task | Test | Red | Green |", "| --- | --- | --- | --- |");
|
|
1571
|
+
for (const r of o.rows) out.push(`| ${r.task} | ${r.test} | ${r.red} | ${r.green} |`);
|
|
952
1572
|
out.push("");
|
|
953
1573
|
}
|
|
954
1574
|
out.push("## Gaps", "");
|
|
955
1575
|
if (o.gaps.length === 0) out.push("None.");
|
|
956
1576
|
for (const g of o.gaps)
|
|
957
|
-
out.push(
|
|
1577
|
+
out.push(
|
|
1578
|
+
`- ${g.task === null ? "Plan" : `Task ${g.task}`} \`${g.kind}\` — ${g.detail} — Fixable: ${g.fixable} — log: ${g.logLine}`,
|
|
1579
|
+
);
|
|
958
1580
|
if (o.exceptions.length > 0) out.push("", "## Test exceptions", "");
|
|
959
1581
|
for (const e of o.exceptions)
|
|
960
1582
|
out.push(
|
|
@@ -962,22 +1584,33 @@ function writeAuditReport(
|
|
|
962
1584
|
? `- Task ${e.task} — honoured; cleared ${e.cleared.length > 0 ? e.cleared.join(", ") : "nothing"}`
|
|
963
1585
|
: `- Task ${e.task} — not honoured: ${e.refusal}`,
|
|
964
1586
|
);
|
|
965
|
-
if (o.
|
|
966
|
-
for (const
|
|
1587
|
+
if (o.landed.length > 0) out.push("", "## Already landed", "");
|
|
1588
|
+
for (const l of o.landed)
|
|
967
1589
|
out.push(
|
|
968
|
-
`- Task ${task}
|
|
1590
|
+
`- Task ${l.task} \`${l.test}\` "${l.name}" — passed at the group's red, and the tests step accepted it: ${l.reason}; no red is due`,
|
|
969
1591
|
);
|
|
970
|
-
|
|
971
|
-
|
|
1592
|
+
if (o.updated.length > 0) out.push("", "## Updated tests", "");
|
|
1593
|
+
for (const u of o.updated) {
|
|
1594
|
+
const since = u.base === null ? "" : ` since \`${u.base.slice(0, 9)}\``;
|
|
1595
|
+
const judged =
|
|
1596
|
+
u.additive === null
|
|
1597
|
+
? "its lines at the base commit could not be read, so read the group diff"
|
|
1598
|
+
: u.additive
|
|
1599
|
+
? "additive: no line of the test was removed, so no expectation was dropped — read the added lines for one that disables an earlier expectation"
|
|
1600
|
+
: "not additive: a line of the test was removed or changed — read the diff for an expectation removed or loosened";
|
|
972
1601
|
out.push(
|
|
973
|
-
"",
|
|
974
|
-
`## Diff since red — Task ${d.task} \`${d.test}\``,
|
|
975
|
-
"",
|
|
976
|
-
`${fence}diff`,
|
|
977
|
-
d.diff,
|
|
978
|
-
fence,
|
|
1602
|
+
`- Task ${u.task} \`${u.test}\` "${u.name}" — an existing test the tests step changed${since}; the update may add expectations, never remove or loosen one. ${judged}`,
|
|
979
1603
|
);
|
|
1604
|
+
if (u.diff !== null) {
|
|
1605
|
+
const fence = fenceFor(u.diff);
|
|
1606
|
+
out.push("", ` ${fence}diff`, ...u.diff.split("\n").map((l) => ` ${l}`), ` ${fence}`, "");
|
|
1607
|
+
}
|
|
980
1608
|
}
|
|
1609
|
+
if (o.unproven.length > 0) out.push("", "## Unproven", "");
|
|
1610
|
+
for (const u of o.unproven)
|
|
1611
|
+
out.push(
|
|
1612
|
+
`- Task ${u.task} \`${u.test}\` "${u.name}" — ${u.detail}; the owner answers it at the end of the build`,
|
|
1613
|
+
);
|
|
981
1614
|
for (const t of o.outputs) {
|
|
982
1615
|
const fence = fenceFor(t.tail);
|
|
983
1616
|
out.push("", `## Runner output — \`${t.test}\``, "", fence, t.tail, fence);
|
|
@@ -991,8 +1624,11 @@ export const testsCommand: CommandSpec = {
|
|
|
991
1624
|
summary:
|
|
992
1625
|
"Run the deepest declared local gate, or one test file to record a red and audit a green.",
|
|
993
1626
|
verbs: {
|
|
1627
|
+
changed: testsChangedVerb,
|
|
994
1628
|
run: {
|
|
995
|
-
summary:
|
|
1629
|
+
summary:
|
|
1630
|
+
"Run the first declared --gate, and with --file append a test-run status row for the run; exit 0 PASS, 1 FAIL, 4 could not run, 81 tests lock held.",
|
|
1631
|
+
use: "by-hand",
|
|
996
1632
|
flags: [
|
|
997
1633
|
{
|
|
998
1634
|
name: "gate",
|
|
@@ -1012,16 +1648,46 @@ export const testsCommand: CommandSpec = {
|
|
|
1012
1648
|
type: "string",
|
|
1013
1649
|
help: `lines of the log echoed to stderr afterwards (default ${DEFAULT_TAIL}; 0 for none)`,
|
|
1014
1650
|
},
|
|
1651
|
+
{
|
|
1652
|
+
name: "lock-wait-s",
|
|
1653
|
+
type: "string",
|
|
1654
|
+
help: "whole-number lock wait >= 0, passed as LOCAL_STACK_LOCK_WAIT_S (default BKIT_LOCK_WAIT_S, else 60)",
|
|
1655
|
+
},
|
|
1015
1656
|
{
|
|
1016
1657
|
name: "artifact",
|
|
1017
1658
|
type: "string",
|
|
1018
1659
|
help: "copy the log here, trim the copy (debug-evidence) and record the clamp; needs --fid and --step",
|
|
1019
1660
|
},
|
|
1020
1661
|
{ name: "fid", type: "string", help: "--artifact: feature id, for the clamp log" },
|
|
1021
|
-
{
|
|
1022
|
-
|
|
1662
|
+
{
|
|
1663
|
+
name: "step",
|
|
1664
|
+
type: "string",
|
|
1665
|
+
help: "step name: the test-run row's (--file) and the clamp log's (--artifact)",
|
|
1666
|
+
},
|
|
1667
|
+
{
|
|
1668
|
+
name: "attempt",
|
|
1669
|
+
type: "string",
|
|
1670
|
+
help: "attempt: on the test-run row (--file) and in the clamp log's name (--artifact)",
|
|
1671
|
+
},
|
|
1672
|
+
{
|
|
1673
|
+
name: "file",
|
|
1674
|
+
type: "string",
|
|
1675
|
+
help: "status.jsonl: append one test-run row for this run, pass or fail; needs --workflow and --step",
|
|
1676
|
+
},
|
|
1677
|
+
{ name: "workflow", type: "string", help: "--file: workflow name, for the test-run row" },
|
|
1678
|
+
{ name: "ci-run", type: "string", help: "selected GitHub evidence run ID" },
|
|
1679
|
+
{ name: "ci-url", type: "string", help: "selected GitHub evidence attempt URL" },
|
|
1023
1680
|
],
|
|
1024
1681
|
output: {
|
|
1682
|
+
source: "ci for pinned red evidence; local for every executed gate",
|
|
1683
|
+
runAttempt: "selected CI attempt for a CI evidence round",
|
|
1684
|
+
ciGeneration: "active selection generation for a CI evidence round",
|
|
1685
|
+
ciRun: "selected CI run ID for a CI evidence round",
|
|
1686
|
+
ciUrl: "selected CI attempt URL for a CI evidence round",
|
|
1687
|
+
repo: "selected repository for a CI evidence round",
|
|
1688
|
+
workflowId: "selected workflow ID for a CI evidence round",
|
|
1689
|
+
workflowPath: "selected workflow path for a CI evidence round",
|
|
1690
|
+
headSha: "selected commit for a CI evidence round",
|
|
1025
1691
|
verdict: "PASS | FAIL (null when the gate could not run)",
|
|
1026
1692
|
exitCode: "the gate's exit code",
|
|
1027
1693
|
gate: "the script that ran",
|
|
@@ -1032,14 +1698,24 @@ export const testsCommand: CommandSpec = {
|
|
|
1032
1698
|
logBytes: "size of the saved output",
|
|
1033
1699
|
artifact: "--artifact as given, else null",
|
|
1034
1700
|
clamp: "the artifact's clamp record (raw = the log, source = scope), else null",
|
|
1701
|
+
row: `the ${TEST_RUN_ROW} row appended to --file ({type, workflow, at, step, attempt?, run, verdict, exitCode, failures, scope, log, artifact?}; run = this step's test-run rows in the workflow's active run, this one included), else null; a tests-lock wait writes no row`,
|
|
1702
|
+
lockHolder: "tests-lock holder when the wait gave up, else null",
|
|
1035
1703
|
},
|
|
1036
|
-
textOmit: ["clamp"],
|
|
1704
|
+
textOmit: ["clamp", "row"],
|
|
1037
1705
|
exitCodes: {
|
|
1038
1706
|
[EXIT_NO_GATE]: "no candidate gate is declared in package.json",
|
|
1039
1707
|
[EXIT_COULD_NOT_RUN]: "the gate could not run (exit 126 / 127): not a test failure",
|
|
1040
1708
|
[EXIT_ARTIFACT_STEP]: "--artifact could not be written, trimmed or recorded",
|
|
1041
1709
|
},
|
|
1042
1710
|
run: (ctx) => {
|
|
1711
|
+
const waitSource = ctx.flags.lockWaitS ?? process.env.BKIT_LOCK_WAIT_S;
|
|
1712
|
+
const waitS = wholeNumber(
|
|
1713
|
+
waitSource,
|
|
1714
|
+
ctx.flags.lockWaitS === undefined && process.env.BKIT_LOCK_WAIT_S !== undefined
|
|
1715
|
+
? "lock-wait-s (BKIT_LOCK_WAIT_S)"
|
|
1716
|
+
: "lock-wait-s",
|
|
1717
|
+
60,
|
|
1718
|
+
);
|
|
1043
1719
|
const candidates = ctx.flags.gate as string[];
|
|
1044
1720
|
const logArg = String(ctx.flags.log);
|
|
1045
1721
|
if (logArg === "") throw new CommandError(EXIT.USAGE, "--log must be non-empty");
|
|
@@ -1049,6 +1725,109 @@ export const testsCommand: CommandSpec = {
|
|
|
1049
1725
|
const attempt = positiveInt(ctx.flags.attempt, "attempt");
|
|
1050
1726
|
if (artifact !== undefined && (!fid || !step))
|
|
1051
1727
|
throw new CommandError(EXIT.USAGE, "--artifact needs --fid and --step");
|
|
1728
|
+
const statusArg = str(ctx.flags.file);
|
|
1729
|
+
const workflow = str(ctx.flags.workflow);
|
|
1730
|
+
if (statusArg !== undefined && (!workflow || !step))
|
|
1731
|
+
throw new CommandError(EXIT.USAGE, "--file needs --workflow and --step");
|
|
1732
|
+
const status = statusArg === undefined ? undefined : path.resolve(ctx.cwd, statusArg);
|
|
1733
|
+
if (status !== undefined && !fs.existsSync(status))
|
|
1734
|
+
throw new CommandError(
|
|
1735
|
+
EXIT.USAGE,
|
|
1736
|
+
`--file ${statusArg} does not exist (a test-run row never starts a log)`,
|
|
1737
|
+
);
|
|
1738
|
+
if (ctx.flags.ciRun !== undefined || ctx.flags.ciUrl !== undefined) {
|
|
1739
|
+
const active = status === undefined ? undefined : readRuns(status).at(-1);
|
|
1740
|
+
const selection = active?.ciSelection;
|
|
1741
|
+
if (
|
|
1742
|
+
!status ||
|
|
1743
|
+
workflow !== "debug" ||
|
|
1744
|
+
active?.workflow !== "debug" ||
|
|
1745
|
+
active.status !== "IN_PROGRESS" ||
|
|
1746
|
+
!validateCiSelection(selection) ||
|
|
1747
|
+
selection.mode !== "ci" ||
|
|
1748
|
+
ctx.flags.ciRun !== selection.runId ||
|
|
1749
|
+
ctx.flags.ciUrl !== selection.runUrl ||
|
|
1750
|
+
!Number.isSafeInteger(active.ciGeneration) ||
|
|
1751
|
+
Number(active.ciGeneration) < 1 ||
|
|
1752
|
+
!artifact
|
|
1753
|
+
)
|
|
1754
|
+
throw new CommandError(
|
|
1755
|
+
EXIT.USAGE,
|
|
1756
|
+
"CI round needs an active matching debug selection and artifact",
|
|
1757
|
+
);
|
|
1758
|
+
const fixed = active.steps.some(
|
|
1759
|
+
(s) =>
|
|
1760
|
+
s.status === "DONE" &&
|
|
1761
|
+
s.ciGeneration === active.ciGeneration &&
|
|
1762
|
+
["debug-failure-remediate", "debug-failure-soak"].includes(s.step),
|
|
1763
|
+
);
|
|
1764
|
+
if (!fixed) {
|
|
1765
|
+
const scope = `ci:${selection.runId}`;
|
|
1766
|
+
const text = [
|
|
1767
|
+
"Verdict: FAIL",
|
|
1768
|
+
"Source: ci",
|
|
1769
|
+
`Scope: ${scope}`,
|
|
1770
|
+
`Repository: ${selection.repo}`,
|
|
1771
|
+
`Workflow: ${selection.workflowPath} (${selection.workflowId})`,
|
|
1772
|
+
`Run: ${selection.runId}`,
|
|
1773
|
+
`Run attempt: ${selection.runAttempt}`,
|
|
1774
|
+
`CI generation: ${active.ciGeneration}`,
|
|
1775
|
+
`Commit: ${selection.headSha}`,
|
|
1776
|
+
`URL: ${selection.runUrl}`,
|
|
1777
|
+
"",
|
|
1778
|
+
"<!-- verdict:end -->",
|
|
1779
|
+
"",
|
|
1780
|
+
"# CI evidence",
|
|
1781
|
+
"",
|
|
1782
|
+
"The local gate was not run.",
|
|
1783
|
+
"",
|
|
1784
|
+
].join("\n");
|
|
1785
|
+
for (const dest of new Set([artifact, logArg])) {
|
|
1786
|
+
const out = path.resolve(ctx.cwd, dest);
|
|
1787
|
+
fs.mkdirSync(path.dirname(out), { recursive: true });
|
|
1788
|
+
fs.writeFileSync(out, text);
|
|
1789
|
+
}
|
|
1790
|
+
const evidence = {
|
|
1791
|
+
gate: null,
|
|
1792
|
+
ciRun: selection.runId,
|
|
1793
|
+
ciUrl: selection.runUrl,
|
|
1794
|
+
runAttempt: selection.runAttempt,
|
|
1795
|
+
ciGeneration: active.ciGeneration,
|
|
1796
|
+
repo: selection.repo,
|
|
1797
|
+
workflowId: selection.workflowId,
|
|
1798
|
+
workflowPath: selection.workflowPath,
|
|
1799
|
+
headSha: selection.headSha,
|
|
1800
|
+
};
|
|
1801
|
+
const row = appendTestRun(status, {
|
|
1802
|
+
workflow,
|
|
1803
|
+
step: step!,
|
|
1804
|
+
attempt,
|
|
1805
|
+
source: "ci",
|
|
1806
|
+
verdict: "FAIL",
|
|
1807
|
+
exitCode: null,
|
|
1808
|
+
failures: null,
|
|
1809
|
+
scope,
|
|
1810
|
+
log: logArg,
|
|
1811
|
+
artifact,
|
|
1812
|
+
evidence,
|
|
1813
|
+
});
|
|
1814
|
+
ctx.emit({
|
|
1815
|
+
source: "ci",
|
|
1816
|
+
verdict: "FAIL",
|
|
1817
|
+
scope,
|
|
1818
|
+
exitCode: null,
|
|
1819
|
+
failures: null,
|
|
1820
|
+
...evidence,
|
|
1821
|
+
artifact,
|
|
1822
|
+
log: logArg,
|
|
1823
|
+
logBytes: Buffer.byteLength(text),
|
|
1824
|
+
reports: null,
|
|
1825
|
+
clamp: null,
|
|
1826
|
+
row,
|
|
1827
|
+
});
|
|
1828
|
+
return EXIT.NO;
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1052
1831
|
const tailRaw = str(ctx.flags.tail) ?? String(DEFAULT_TAIL);
|
|
1053
1832
|
if (!/^[0-9]+$/.test(tailRaw)) {
|
|
1054
1833
|
throw new CommandError(
|
|
@@ -1073,13 +1852,27 @@ export const testsCommand: CommandSpec = {
|
|
|
1073
1852
|
const reports = freshReportDir(log);
|
|
1074
1853
|
ctx.log(`running ${scope} — full output in ${logArg}`);
|
|
1075
1854
|
const fetchedAt = new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
1076
|
-
const
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1855
|
+
const who = ["bkit tests run", gate, fid, step].filter(Boolean).join(" ");
|
|
1856
|
+
const exitCode = runToLog(scriptCommand(gate), top, log, {
|
|
1857
|
+
[REPORT_DIR_ENV]: reports,
|
|
1858
|
+
LOCAL_STACK_LOCK_WAIT_S: String(waitS),
|
|
1859
|
+
LOCAL_STACK_LOCK_WHO: who,
|
|
1860
|
+
});
|
|
1861
|
+
const logText = fs.readFileSync(log, "utf-8");
|
|
1862
|
+
const lockExit = classifyLockExit({ exitCode, log: logText, who });
|
|
1863
|
+
const lockWait = lockExit === "lock-wait";
|
|
1864
|
+
const lockHolder = lockWait ? testsLockHolder(logText, who) : null;
|
|
1865
|
+
const lines =
|
|
1866
|
+
lockExit === "gate"
|
|
1867
|
+
? Number(tailRaw) > 0
|
|
1868
|
+
? tailLines(log, Number(tailRaw))
|
|
1869
|
+
: []
|
|
1870
|
+
: logText.split(/\r?\n/).filter((line) => line.startsWith("[machine-lock]"));
|
|
1871
|
+
for (const line of lines) ctx.log(line);
|
|
1872
|
+
const couldNotRun = lockExit === "could-not-run" || COULD_NOT_RUN_EXIT_CODES.has(exitCode);
|
|
1873
|
+
const verdict = couldNotRun || lockWait ? null : exitCode === 0 ? "PASS" : "FAIL";
|
|
1874
|
+
const counted = lockExit === "gate" ? countReportFailures(reports) : null;
|
|
1875
|
+
if (counted === null && lockExit === "gate") ctx.log(NO_REPORT_HINT);
|
|
1083
1876
|
else if (verdict === "FAIL" && counted === 0)
|
|
1084
1877
|
ctx.log(
|
|
1085
1878
|
`the JUnit reports in $${REPORT_DIR_ENV} hold no failing case — the gate failed outside its reported tests; failures unknown`,
|
|
@@ -1087,7 +1880,7 @@ export const testsCommand: CommandSpec = {
|
|
|
1087
1880
|
const failures = verdict === "PASS" ? 0 : counted !== null && counted > 0 ? counted : null;
|
|
1088
1881
|
let clamp: ClampRecord | null = null;
|
|
1089
1882
|
let artifactFailed: string | null = null;
|
|
1090
|
-
if (artifact !== undefined) {
|
|
1883
|
+
if (artifact !== undefined && !lockWait) {
|
|
1091
1884
|
try {
|
|
1092
1885
|
const out = path.resolve(ctx.cwd, artifact);
|
|
1093
1886
|
fs.mkdirSync(path.dirname(out), { recursive: true });
|
|
@@ -1108,7 +1901,23 @@ export const testsCommand: CommandSpec = {
|
|
|
1108
1901
|
artifactFailed = (err as Error).message;
|
|
1109
1902
|
}
|
|
1110
1903
|
}
|
|
1904
|
+
const row =
|
|
1905
|
+
status === undefined || lockWait
|
|
1906
|
+
? null
|
|
1907
|
+
: appendTestRun(status, {
|
|
1908
|
+
workflow: workflow!,
|
|
1909
|
+
step: step!,
|
|
1910
|
+
attempt,
|
|
1911
|
+
verdict,
|
|
1912
|
+
exitCode,
|
|
1913
|
+
failures,
|
|
1914
|
+
scope,
|
|
1915
|
+
log: logArg,
|
|
1916
|
+
artifact,
|
|
1917
|
+
source: "local",
|
|
1918
|
+
});
|
|
1111
1919
|
ctx.emit({
|
|
1920
|
+
source: "local",
|
|
1112
1921
|
verdict,
|
|
1113
1922
|
exitCode,
|
|
1114
1923
|
gate,
|
|
@@ -1117,16 +1926,19 @@ export const testsCommand: CommandSpec = {
|
|
|
1117
1926
|
reports: `${logArg}.reports`,
|
|
1118
1927
|
log: logArg,
|
|
1119
1928
|
logBytes: fs.statSync(log).size,
|
|
1120
|
-
artifact: artifact ?? null,
|
|
1929
|
+
artifact: lockWait ? null : (artifact ?? null),
|
|
1121
1930
|
clamp,
|
|
1931
|
+
row,
|
|
1932
|
+
lockHolder,
|
|
1122
1933
|
});
|
|
1123
1934
|
if (artifactFailed !== null) {
|
|
1124
1935
|
ctx.log(`--artifact ${artifact}: ${artifactFailed}`);
|
|
1125
1936
|
return EXIT_ARTIFACT_STEP;
|
|
1126
1937
|
}
|
|
1938
|
+
if (lockWait) return EXIT.AGENT_WAIT;
|
|
1127
1939
|
if (couldNotRun) {
|
|
1128
1940
|
ctx.log(
|
|
1129
|
-
`${scope} could not run (exit ${exitCode}: ${exitCode === 126 ? "not executable" : "command not found"}) — no tests executed; this is not a FAIL verdict`,
|
|
1941
|
+
`${scope} could not run (exit ${exitCode}: ${lockExit === "could-not-run" ? "tests lock refused" : exitCode === 126 ? "not executable" : "command not found"}) — no tests executed; this is not a FAIL verdict`,
|
|
1130
1942
|
);
|
|
1131
1943
|
return EXIT_COULD_NOT_RUN;
|
|
1132
1944
|
}
|
|
@@ -1136,130 +1948,287 @@ export const testsCommand: CommandSpec = {
|
|
|
1136
1948
|
},
|
|
1137
1949
|
red: {
|
|
1138
1950
|
summary:
|
|
1139
|
-
"Run
|
|
1951
|
+
"Run every TEST FIRST test of one group's open tasks before any task code exists, and write one build-tests-red row holding a state and blob per planned test; a test an earlier row of the group proved red, its file unchanged since, is not run again. Exit 0 every test red, 1 a gap (a test passing, absent from its report, failing lint, or its file missing), 3 bkit:test:file undeclared (blobs recorded, nothing ran), 4 a script could not run.",
|
|
1952
|
+
use: "workflow",
|
|
1140
1953
|
flags: [
|
|
1141
1954
|
FILE_FLAG,
|
|
1142
1955
|
WORKFLOW_FLAG,
|
|
1143
|
-
GROUP_FLAG,
|
|
1144
1956
|
{
|
|
1145
|
-
|
|
1146
|
-
type: "string",
|
|
1957
|
+
...GROUP_FLAG,
|
|
1147
1958
|
required: true,
|
|
1148
|
-
help: "
|
|
1959
|
+
help: "the group whose planned tests run (positive integer)",
|
|
1149
1960
|
},
|
|
1150
1961
|
{
|
|
1151
|
-
name: "
|
|
1962
|
+
name: "plan",
|
|
1152
1963
|
type: "string",
|
|
1153
1964
|
required: true,
|
|
1154
|
-
|
|
1965
|
+
repeatable: true,
|
|
1966
|
+
help: "candidate plan file; the first that exists is read",
|
|
1155
1967
|
},
|
|
1156
1968
|
{
|
|
1157
|
-
name: "
|
|
1969
|
+
name: "log",
|
|
1158
1970
|
type: "string",
|
|
1159
1971
|
required: true,
|
|
1160
|
-
help: "the
|
|
1972
|
+
help: "file the red report is written to (directories are created)",
|
|
1161
1973
|
},
|
|
1162
1974
|
{
|
|
1163
|
-
name: "log",
|
|
1975
|
+
name: "writer-log",
|
|
1164
1976
|
type: "string",
|
|
1165
|
-
|
|
1166
|
-
help: "file the runner output is saved to; the test file's copy lands beside it as .snapshot",
|
|
1977
|
+
help: "the tests step's log: a `PASSES — TASK N: `file` `name`: reason` line there accepts that test's pass as passed-accepted",
|
|
1167
1978
|
},
|
|
1168
1979
|
],
|
|
1169
1980
|
output: {
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1981
|
+
enforced:
|
|
1982
|
+
"false when bkit:test:file is not declared (the row records the blobs, every state unknown, exit 3) or the plan could not be read (no row, exit 1)",
|
|
1983
|
+
reason: "why test-first is not enforced",
|
|
1984
|
+
attempt: "this group's build-tests-red rows so far, this one included",
|
|
1985
|
+
tests:
|
|
1986
|
+
"[{task, test, testName, state, blob, reason?}] in plan order, the tests run now: state red | red-not-loaded | passed-accepted | vacuous | missing | no-report | lint | unknown; blob the test file's git blob as it stood (null when the file is missing); reason the tests step's, on passed-accepted",
|
|
1987
|
+
skipped:
|
|
1988
|
+
"planned tests an earlier row of the group proved red, their file unchanged since: not run again, and not on this row",
|
|
1989
|
+
gaps: "[{task, test, name, kind, detail}]: every planned test whose state is not red, red-not-loaded or passed-accepted; kind is the state",
|
|
1990
|
+
row: "the build-tests-red row written",
|
|
1991
|
+
log: "where the report was written (as given)",
|
|
1178
1992
|
},
|
|
1179
1993
|
exitCodes: {
|
|
1180
|
-
[EXIT_NO_GATE]: `${BKIT_TEST_FILE_SCRIPT} is not declared in package.json`,
|
|
1181
|
-
[EXIT_COULD_NOT_RUN]:
|
|
1994
|
+
[EXIT_NO_GATE]: `${BKIT_TEST_FILE_SCRIPT} is not declared in package.json: the row holds the blobs with enforced false, and no test ran`,
|
|
1995
|
+
[EXIT_COULD_NOT_RUN]:
|
|
1996
|
+
"bkit:lint:file or bkit:test:file could not run (exit 126 / 127); nothing written",
|
|
1182
1997
|
},
|
|
1183
1998
|
run: (ctx) => {
|
|
1184
1999
|
const workflow = String(ctx.flags.workflow);
|
|
1185
|
-
const group = positiveInt(ctx.flags.group, "group")
|
|
1186
|
-
const task = positiveInt(ctx.flags.task, "task")!;
|
|
1187
|
-
const name = String(ctx.flags.name).trim();
|
|
1188
|
-
if (name === "") throw new CommandError(EXIT.USAGE, "--name must be non-empty");
|
|
2000
|
+
const group = positiveInt(ctx.flags.group, "group")!;
|
|
1189
2001
|
const logArg = String(ctx.flags.log);
|
|
1190
2002
|
if (logArg === "") throw new CommandError(EXIT.USAGE, "--log must be non-empty");
|
|
2003
|
+
const log = path.resolve(ctx.cwd, logArg);
|
|
1191
2004
|
const top = toplevel(ctx.cwd);
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
const test = repoTestPath(top, String(ctx.flags.test));
|
|
1197
|
-
const blob = git(top, ["hash-object", "--", test]);
|
|
2005
|
+
const exists = (p: string) => fs.existsSync(path.resolve(ctx.cwd, p));
|
|
2006
|
+
const plan = (ctx.flags.plan as string[]).find(exists);
|
|
2007
|
+
const planText = plan ? fs.readFileSync(path.resolve(ctx.cwd, plan), "utf-8") : "";
|
|
2008
|
+
const parsed = parsePlanBlocks(planText);
|
|
1198
2009
|
const status = statusFile(ctx);
|
|
1199
|
-
const
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
if (COULD_NOT_RUN_EXIT_CODES.has(exitCode)) {
|
|
1213
|
-
ctx.log(couldNotRunMessage(exitCode));
|
|
1214
|
-
return EXIT_COULD_NOT_RUN;
|
|
1215
|
-
}
|
|
1216
|
-
const refusal = judgeRed(exitCode, cases, name);
|
|
1217
|
-
if (refusal !== null) {
|
|
1218
|
-
ctx.emit({
|
|
1219
|
-
recorded: false,
|
|
1220
|
-
refused: refusal.kind,
|
|
1221
|
-
reason: refusal.reason,
|
|
1222
|
-
exitCode,
|
|
1223
|
-
log: logArg,
|
|
2010
|
+
const attempt = redAttempt(status, workflow, group);
|
|
2011
|
+
const refusal = planRefusal(plan, parsed);
|
|
2012
|
+
if (refusal !== undefined) {
|
|
2013
|
+
writeRedReport(log, {
|
|
2014
|
+
enforced: false,
|
|
2015
|
+
reason: refusal,
|
|
2016
|
+
plan,
|
|
2017
|
+
group,
|
|
2018
|
+
attempt,
|
|
2019
|
+
tests: [],
|
|
2020
|
+
details: new Map(),
|
|
2021
|
+
gaps: [],
|
|
2022
|
+
outputs: [],
|
|
1224
2023
|
});
|
|
1225
|
-
ctx.
|
|
2024
|
+
ctx.emit({ enforced: false, reason: refusal, attempt, tests: [], gaps: [], log: logArg });
|
|
2025
|
+
ctx.log(`test-first not enforced: ${refusal}`);
|
|
1226
2026
|
return EXIT.NO;
|
|
1227
2027
|
}
|
|
1228
|
-
const
|
|
1229
|
-
|
|
1230
|
-
|
|
2028
|
+
const g = parsed.groups.find((x) => x.group === group);
|
|
2029
|
+
if (g === undefined) throw new CommandError(EXIT.USAGE, `the plan has no GROUP ${group}`);
|
|
2030
|
+
if (g.tasks.length === 0)
|
|
2031
|
+
throw new CommandError(EXIT.USAGE, `GROUP ${group} has no tasks; nothing to run`);
|
|
2032
|
+
const done = new Set(parsed.tasks.filter((t) => t.done).map((t) => t.task));
|
|
2033
|
+
// A red an earlier row proved stands while the test's own lines are unchanged, so a
|
|
2034
|
+
// retry, a handover or a task added to the group runs only what is left, and an edit
|
|
2035
|
+
// to another test of the same file costs no proof. A test whose lines could not be
|
|
2036
|
+
// found, and rows from before per-test locking, stand while the whole file is unchanged.
|
|
2037
|
+
const proven = redEntries(status, workflow, group);
|
|
2038
|
+
const textOf = (test: string) => {
|
|
2039
|
+
const abs = path.join(top, test);
|
|
2040
|
+
return fs.existsSync(abs) ? fs.readFileSync(abs, "utf-8") : null;
|
|
2041
|
+
};
|
|
2042
|
+
const standing = (t: RedTest) => {
|
|
2043
|
+
const earlier = proven.get(redKey(t.task, t.test, t.testName));
|
|
2044
|
+
if (earlier === undefined || !RED_PROVEN.has(earlier.state)) return false;
|
|
2045
|
+
if (earlier.block === undefined || earlier.block === null)
|
|
2046
|
+
return git(top, ["hash-object", "--", t.test]) === earlier.blob;
|
|
2047
|
+
return blockHash(textOf(t.test), t.testName) === earlier.block;
|
|
2048
|
+
};
|
|
2049
|
+
const startSha = groupStartSha(status, workflow, group);
|
|
2050
|
+
/** The test's lines at the group's start commit: it existed before this group's tests step. */
|
|
2051
|
+
const blockAtStart = (t: RedTest) => {
|
|
2052
|
+
if (startSha === undefined) return null;
|
|
2053
|
+
const shown = runGit(top, ["show", `${startSha}:${t.test}`]);
|
|
2054
|
+
return shown.ok ? blockHash(shown.stdout, t.testName) : null;
|
|
2055
|
+
};
|
|
2056
|
+
const tests: RedTest[] = [];
|
|
2057
|
+
let skipped = 0;
|
|
2058
|
+
for (const s of readTestFirst(planText, group)) {
|
|
2059
|
+
if (s.kind !== "test" || done.has(s.task)) continue;
|
|
2060
|
+
for (const t of s.tests) {
|
|
2061
|
+
const planned: RedTest = {
|
|
2062
|
+
task: s.task,
|
|
2063
|
+
test: t.test,
|
|
2064
|
+
testName: t.name,
|
|
2065
|
+
state: "unknown",
|
|
2066
|
+
blob: null,
|
|
2067
|
+
};
|
|
2068
|
+
if (standing(planned)) skipped += 1;
|
|
2069
|
+
else tests.push(planned);
|
|
2070
|
+
}
|
|
2071
|
+
}
|
|
2072
|
+
if (skipped > 0)
|
|
2073
|
+
ctx.log(`tests red: ${skipped} planned test(s) already red, not run again`);
|
|
2074
|
+
const enforced = resolveGate(top, [BKIT_TEST_FILE_SCRIPT]) !== null;
|
|
2075
|
+
const lint = resolveGate(top, [BKIT_LINT_FILE_SCRIPT]) !== null;
|
|
2076
|
+
if (enforced && !lint && tests.length > 0)
|
|
2077
|
+
ctx.log(
|
|
2078
|
+
`${BKIT_LINT_FILE_SCRIPT} is not declared — the tests are not linted before their red`,
|
|
2079
|
+
);
|
|
2080
|
+
const writerLog = str(ctx.flags.writerLog);
|
|
2081
|
+
const writerPath = writerLog === undefined ? undefined : path.resolve(ctx.cwd, writerLog);
|
|
2082
|
+
const passes =
|
|
2083
|
+
writerPath !== undefined && fs.existsSync(writerPath)
|
|
2084
|
+
? passesLines(fs.readFileSync(writerPath, "utf-8"))
|
|
2085
|
+
: [];
|
|
2086
|
+
const details = new Map<RedTest, string>();
|
|
2087
|
+
const outputs: Array<{ title: string; text: string }> = [];
|
|
2088
|
+
const couldNotRun = (script: string, exitCode: number) =>
|
|
2089
|
+
`${gateScope(script)} could not run (exit ${exitCode}: ${exitCode === 126 ? "not executable" : "command not found"}) — no test executed`;
|
|
2090
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bkit-tests-red-"));
|
|
2091
|
+
try {
|
|
2092
|
+
for (const file of [...new Set(tests.map((t) => t.test))]) {
|
|
2093
|
+
const mine = tests.filter((t) => t.test === file);
|
|
2094
|
+
if (!fs.existsSync(path.join(top, file))) {
|
|
2095
|
+
for (const t of mine) {
|
|
2096
|
+
t.state = "missing";
|
|
2097
|
+
details.set(t, `\`${file}\` does not exist`);
|
|
2098
|
+
}
|
|
2099
|
+
continue;
|
|
2100
|
+
}
|
|
2101
|
+
// `-w` stores the file in git: green and `steps finish` read the blob back later.
|
|
2102
|
+
const blob = git(top, ["hash-object", "-w", "--", file]);
|
|
2103
|
+
const text = textOf(file);
|
|
2104
|
+
for (const t of mine) {
|
|
2105
|
+
t.blob = blob;
|
|
2106
|
+
t.block = blockHash(text, t.testName);
|
|
2107
|
+
}
|
|
2108
|
+
if (!enforced) continue;
|
|
2109
|
+
const runLog = path.join(dir, "run.log");
|
|
2110
|
+
if (lint) {
|
|
2111
|
+
const linted = runToLog([...scriptCommand(BKIT_LINT_FILE_SCRIPT), file], top, runLog);
|
|
2112
|
+
outputs.push({
|
|
2113
|
+
title: `Lint output — \`${file}\``,
|
|
2114
|
+
text: fs.readFileSync(runLog, "utf-8"),
|
|
2115
|
+
});
|
|
2116
|
+
if (COULD_NOT_RUN_EXIT_CODES.has(linted)) {
|
|
2117
|
+
ctx.log(couldNotRun(BKIT_LINT_FILE_SCRIPT, linted));
|
|
2118
|
+
return EXIT_COULD_NOT_RUN;
|
|
2119
|
+
}
|
|
2120
|
+
if (linted !== 0) {
|
|
2121
|
+
for (const t of mine) {
|
|
2122
|
+
t.state = "lint";
|
|
2123
|
+
details.set(
|
|
2124
|
+
t,
|
|
2125
|
+
`${BKIT_LINT_FILE_SCRIPT} exited ${linted} on \`${file}\`: fix the test until it lints clean (output below)`,
|
|
2126
|
+
);
|
|
2127
|
+
}
|
|
2128
|
+
continue;
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
const { exitCode, cases } = runTestFile(top, file, runLog);
|
|
2132
|
+
outputs.push({
|
|
2133
|
+
title: `Runner output — \`${file}\``,
|
|
2134
|
+
text: fs.readFileSync(runLog, "utf-8"),
|
|
2135
|
+
});
|
|
2136
|
+
if (COULD_NOT_RUN_EXIT_CODES.has(exitCode)) {
|
|
2137
|
+
ctx.log(couldNotRun(BKIT_TEST_FILE_SCRIPT, exitCode));
|
|
2138
|
+
return EXIT_COULD_NOT_RUN;
|
|
2139
|
+
}
|
|
2140
|
+
for (const t of mine) {
|
|
2141
|
+
const judged = judgeRed(cases, t.testName, file);
|
|
2142
|
+
const accepted =
|
|
2143
|
+
judged.state === "vacuous"
|
|
2144
|
+
? passes.find(
|
|
2145
|
+
(p) => p.task === t.task && p.test === t.test && p.name === t.testName,
|
|
2146
|
+
)
|
|
2147
|
+
: undefined;
|
|
2148
|
+
const before = judged.state === "vacuous" ? blockAtStart(t) : null;
|
|
2149
|
+
if (accepted !== undefined) {
|
|
2150
|
+
t.state = "passed-accepted";
|
|
2151
|
+
t.reason = accepted.reason;
|
|
2152
|
+
details.set(t, `passes; accepted by the tests step: ${accepted.reason}`);
|
|
2153
|
+
} else if (before !== null && before !== t.block) {
|
|
2154
|
+
t.state = "updated";
|
|
2155
|
+
t.base = startSha;
|
|
2156
|
+
details.set(
|
|
2157
|
+
t,
|
|
2158
|
+
`an existing test, changed since the group started (\`${startSha!.slice(0, 9)}\`) and passing: the verifier judges its diff — it may add expectations, never remove or loosen one`,
|
|
2159
|
+
);
|
|
2160
|
+
} else {
|
|
2161
|
+
t.state = judged.state;
|
|
2162
|
+
details.set(t, judged.detail);
|
|
2163
|
+
}
|
|
2164
|
+
}
|
|
2165
|
+
}
|
|
2166
|
+
} finally {
|
|
2167
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
2168
|
+
}
|
|
2169
|
+
const gaps: RedGap[] = [];
|
|
2170
|
+
for (const t of tests) {
|
|
2171
|
+
if (RED_PROVEN.has(t.state) || t.state === "unknown") continue;
|
|
2172
|
+
const gap: RedGap = {
|
|
2173
|
+
task: t.task,
|
|
2174
|
+
test: t.test,
|
|
2175
|
+
name: t.testName,
|
|
2176
|
+
kind: t.state as RedGap["kind"],
|
|
2177
|
+
detail: details.get(t) ?? t.state,
|
|
2178
|
+
};
|
|
2179
|
+
gaps.push(gap);
|
|
2180
|
+
ctx.log(
|
|
2181
|
+
`not a red (${gap.kind}): TASK ${gap.task} \`${gap.test}\` "${gap.name}" — ${gap.detail}`,
|
|
2182
|
+
);
|
|
2183
|
+
}
|
|
2184
|
+
writeRedReport(log, {
|
|
2185
|
+
enforced,
|
|
2186
|
+
reason: enforced
|
|
2187
|
+
? undefined
|
|
2188
|
+
: `\`${BKIT_TEST_FILE_SCRIPT}\` is not declared in package.json`,
|
|
2189
|
+
plan,
|
|
2190
|
+
group,
|
|
2191
|
+
attempt,
|
|
2192
|
+
tests,
|
|
2193
|
+
details,
|
|
2194
|
+
gaps,
|
|
2195
|
+
outputs,
|
|
2196
|
+
});
|
|
1231
2197
|
const written = writeStepRow(
|
|
1232
2198
|
ctx.cwd,
|
|
1233
2199
|
{
|
|
1234
|
-
file:
|
|
2200
|
+
file: status,
|
|
1235
2201
|
workflow,
|
|
1236
2202
|
step: RED_STEP,
|
|
1237
2203
|
group,
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
test,
|
|
1242
|
-
testName: name,
|
|
1243
|
-
testBlob: blob,
|
|
1244
|
-
exitCode,
|
|
1245
|
-
failure: failed.message.slice(0, 200),
|
|
1246
|
-
},
|
|
2204
|
+
attempt,
|
|
2205
|
+
artifacts: [path.relative(ctx.cwd, log)],
|
|
2206
|
+
extras: { enforced, tests },
|
|
1247
2207
|
commit: "none",
|
|
1248
2208
|
},
|
|
1249
2209
|
ctx.log,
|
|
1250
2210
|
);
|
|
1251
|
-
ctx.emit({
|
|
1252
|
-
|
|
2211
|
+
ctx.emit({ enforced, attempt, tests, skipped, gaps, row: written.row, log: logArg });
|
|
2212
|
+
if (gaps.length > 0) {
|
|
2213
|
+
ctx.log(`tests red: ${gaps.length} gap(s) — report in ${logArg}`);
|
|
2214
|
+
return EXIT.NO;
|
|
2215
|
+
}
|
|
2216
|
+
if (!enforced) {
|
|
2217
|
+
ctx.log(`${BKIT_TEST_FILE_SCRIPT} is not declared — ${NO_TEST_FILE_HINT}`);
|
|
2218
|
+
return EXIT_NO_GATE;
|
|
2219
|
+
}
|
|
2220
|
+
ctx.log(`tests red: ${tests.length} planned test(s) red — report in ${logArg}`);
|
|
1253
2221
|
return EXIT.OK;
|
|
1254
2222
|
},
|
|
1255
2223
|
},
|
|
1256
2224
|
green: {
|
|
1257
2225
|
summary:
|
|
1258
|
-
"Audit a plan's test-first evidence (
|
|
2226
|
+
"Audit a plan's test-first evidence (each TEST FIRST test red at its group's tests step, by the build-tests-red row, or accepted there as passing before any task code; the named test passing now); exit 0 no gaps, 1 gaps or a plan with no task markers.",
|
|
2227
|
+
use: "workflow",
|
|
1259
2228
|
flags: [
|
|
1260
2229
|
FILE_FLAG,
|
|
1261
2230
|
WORKFLOW_FLAG,
|
|
1262
|
-
{ ...GROUP_FLAG, help: "audit only this
|
|
2231
|
+
{ ...GROUP_FLAG, help: "audit only this task group's tasks (positive integer)" },
|
|
1263
2232
|
{
|
|
1264
2233
|
name: "plan",
|
|
1265
2234
|
type: "string",
|
|
@@ -1279,11 +2248,15 @@ export const testsCommand: CommandSpec = {
|
|
|
1279
2248
|
"false when the audit could not run: no plan file, a plan with no task markers (exit 1), or the test-file script undeclared (exit 0)",
|
|
1280
2249
|
reason: "why test-first is not enforced",
|
|
1281
2250
|
plan: "the plan file audited",
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
"
|
|
2251
|
+
verdict:
|
|
2252
|
+
"PASS no gaps | FAIL a gap's fixable is false | CONCERNS every gap's fixable is true | null when a gap needs a diff read, or the audit could not run",
|
|
2253
|
+
gaps: "[{task, kind, detail, fixable, logLine}]: no-test-first | test-first-malformed | test-first-unread (task null: a plan-level format problem) | not-green | exception-refused (a task whose only test field is a TEST EXCEPTION not honoured) | delete-still-referenced (a DELETE line's file still exists, or its name is still found in the tree); fixable is null when the gap needs a diff read; logLine is the line the verifier appends",
|
|
2254
|
+
alreadyLanded:
|
|
2255
|
+
"[{task, test, name, reason}]: tests the tests step accepted as passing before any task code (passed-accepted on the red row); no red is due for them",
|
|
2256
|
+
unproven:
|
|
2257
|
+
"[{task, test, name, detail}]: tests with no proven red (no build-tests-red row, or a state other than red, red-not-loaded, updated or passed-accepted) and no honoured TEST EXCEPTION; never a gap — the owner answers them once, at the end of the build",
|
|
2258
|
+
updated:
|
|
2259
|
+
"[{task, test, name, base, additive, diff}]: existing tests the tests step changed (updated on the red row); base is the group's start commit; additive = the test's own lines since base gained lines and lost none (null when a side cannot be read), diff = their unified line diff; the verifier judges each diff — it may add expectations, never remove or loosen one",
|
|
1287
2260
|
log: "where the report was written (as given)",
|
|
1288
2261
|
},
|
|
1289
2262
|
exitCodes: {
|
|
@@ -1303,14 +2276,7 @@ export const testsCommand: CommandSpec = {
|
|
|
1303
2276
|
const specs = readTestFirst(planText, group);
|
|
1304
2277
|
// A plan with no task markers is an old-format plan: nothing it says can be audited,
|
|
1305
2278
|
// and passing it would let its tests go unchecked.
|
|
1306
|
-
const refusal =
|
|
1307
|
-
plan === undefined
|
|
1308
|
-
? `no plan file; ${NO_MARKERS_REASON}`
|
|
1309
|
-
: parsed.tasks.length === 0
|
|
1310
|
-
? parsed.fenceOpen === undefined
|
|
1311
|
-
? NO_MARKERS_REASON
|
|
1312
|
-
: fenceNeverClosed(parsed.fenceOpen)
|
|
1313
|
-
: undefined;
|
|
2279
|
+
const refusal = planRefusal(plan, parsed);
|
|
1314
2280
|
if (refusal === undefined && group !== undefined) {
|
|
1315
2281
|
const g = parsed.groups.find((x) => x.group === group);
|
|
1316
2282
|
if (g === undefined) throw new CommandError(EXIT.USAGE, `the plan has no GROUP ${group}`);
|
|
@@ -1323,12 +2289,13 @@ export const testsCommand: CommandSpec = {
|
|
|
1323
2289
|
? `\`${BKIT_TEST_FILE_SCRIPT}\` is not declared in package.json`
|
|
1324
2290
|
: undefined);
|
|
1325
2291
|
const empty = {
|
|
2292
|
+
verdict: null,
|
|
1326
2293
|
rows: [],
|
|
1327
2294
|
gaps: [],
|
|
1328
|
-
testOrderOnly: false,
|
|
1329
2295
|
exceptions: [],
|
|
1330
|
-
|
|
1331
|
-
|
|
2296
|
+
landed: [],
|
|
2297
|
+
unproven: [],
|
|
2298
|
+
updated: [],
|
|
1332
2299
|
outputs: [],
|
|
1333
2300
|
};
|
|
1334
2301
|
if (reason !== undefined) {
|
|
@@ -1337,23 +2304,26 @@ export const testsCommand: CommandSpec = {
|
|
|
1337
2304
|
enforced: false,
|
|
1338
2305
|
reason,
|
|
1339
2306
|
plan: plan ?? null,
|
|
2307
|
+
verdict: null,
|
|
1340
2308
|
gaps: [],
|
|
1341
|
-
|
|
1342
|
-
|
|
2309
|
+
alreadyLanded: [],
|
|
2310
|
+
unproven: [],
|
|
2311
|
+
updated: [],
|
|
1343
2312
|
log: logArg,
|
|
1344
2313
|
});
|
|
1345
2314
|
ctx.log(`test-first not enforced: ${reason}`);
|
|
1346
2315
|
return refusal === undefined ? EXIT.OK : EXIT.NO;
|
|
1347
2316
|
}
|
|
1348
2317
|
|
|
1349
|
-
const reds =
|
|
2318
|
+
const reds = redEntries(statusFile(ctx), workflow);
|
|
2319
|
+
const groupOf = (task: number) => parsed.tasks.find((b) => b.task === task)?.group;
|
|
2320
|
+
const landed: Landed[] = [];
|
|
2321
|
+
const unproven: Unproven[] = [];
|
|
2322
|
+
const updated: Updated[] = [];
|
|
1350
2323
|
const statusRel = repoRel(top, ctx.cwd, statusFile(ctx));
|
|
1351
|
-
const codeCommits = taskCodeCommits(top, ctx.cwd, statusFile(ctx), workflow);
|
|
1352
2324
|
const exceptions: ExceptionResult[] = [];
|
|
1353
|
-
const lateUnchecked = new Set<number>();
|
|
1354
2325
|
const gaps: Gap[] = [];
|
|
1355
2326
|
const rows: AuditRow[] = [];
|
|
1356
|
-
const diffs: Array<{ task: number; test: string; diff: string }> = [];
|
|
1357
2327
|
const outputs: Array<{ test: string; tail: string }> = [];
|
|
1358
2328
|
const greens = new Map<string, { cases: JunitCase[] | null; tail: string }>();
|
|
1359
2329
|
/** One run per test file; null when the script could not run. */
|
|
@@ -1389,22 +2359,25 @@ export const testsCommand: CommandSpec = {
|
|
|
1389
2359
|
cleared: refused === null ? ["the task's `TEST FIRST` check"] : [],
|
|
1390
2360
|
});
|
|
1391
2361
|
if (refused !== null)
|
|
1392
|
-
gaps.push(
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
2362
|
+
gaps.push(
|
|
2363
|
+
gapOf(
|
|
2364
|
+
s.task,
|
|
2365
|
+
EXCEPTION_REFUSED,
|
|
2366
|
+
`the task's only test field is a \`**TEST EXCEPTION:**\`, not honoured: ${refused}`,
|
|
2367
|
+
),
|
|
2368
|
+
);
|
|
1397
2369
|
continue;
|
|
1398
2370
|
}
|
|
1399
2371
|
if (s.kind !== "test") {
|
|
1400
|
-
gaps.push(
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
2372
|
+
gaps.push(
|
|
2373
|
+
gapOf(
|
|
2374
|
+
s.task,
|
|
2375
|
+
s.kind === "missing" ? "no-test-first" : "test-first-malformed",
|
|
1404
2376
|
s.kind === "missing"
|
|
1405
2377
|
? "the task has neither a `**TEST FIRST:**` nor a `**TEST EXCEPTION:**` field"
|
|
1406
2378
|
: "`**TEST FIRST:**` is neither `n/a — {reason}` nor one sub-bullet per test naming a backticked test file and test name",
|
|
1407
|
-
|
|
2379
|
+
),
|
|
2380
|
+
);
|
|
1408
2381
|
continue;
|
|
1409
2382
|
}
|
|
1410
2383
|
const excepted = s.exception;
|
|
@@ -1422,15 +2395,11 @@ export const testsCommand: CommandSpec = {
|
|
|
1422
2395
|
cleared: [],
|
|
1423
2396
|
};
|
|
1424
2397
|
if (exception !== undefined) exceptions.push(exception);
|
|
1425
|
-
const taskReds = reds.filter((r) => r.task === s.task);
|
|
1426
|
-
const planned = (r: Record<string, unknown>) =>
|
|
1427
|
-
s.tests.some((t) => t.test === r.test && t.name === r.testName);
|
|
1428
2398
|
for (const t of s.tests) {
|
|
1429
2399
|
const row: AuditRow = {
|
|
1430
2400
|
task: s.task,
|
|
1431
2401
|
test: `\`${t.test}\` "${t.name}"`,
|
|
1432
|
-
red: "
|
|
1433
|
-
changed: "—",
|
|
2402
|
+
red: "no row",
|
|
1434
2403
|
green: "—",
|
|
1435
2404
|
};
|
|
1436
2405
|
rows.push(row);
|
|
@@ -1439,85 +2408,28 @@ export const testsCommand: CommandSpec = {
|
|
|
1439
2408
|
excepted!.tests.some((e) => e.test === t.test && e.name === t.name)
|
|
1440
2409
|
? exception
|
|
1441
2410
|
: undefined;
|
|
1442
|
-
const gap = (kind:
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
2411
|
+
const gap = (kind: GapKind, detail: string) =>
|
|
2412
|
+
gaps.push(gapOf(s.task, kind, detail, { file: t.test, name: t.name }));
|
|
2413
|
+
const noRed = (detail: string) => {
|
|
2414
|
+
if (clearing !== undefined)
|
|
2415
|
+
clearing.cleared.push(`the red for \`${t.test}\` "${t.name}"`);
|
|
2416
|
+
else unproven.push({ task: s.task, test: t.test, name: t.name, detail });
|
|
1446
2417
|
};
|
|
1447
|
-
const red =
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
if (red === undefined && unplanned.length > 0) {
|
|
1452
|
-
const recorded: string[] = [];
|
|
1453
|
-
for (const r of unplanned) {
|
|
1454
|
-
const rTest = String(r.test);
|
|
1455
|
-
const rName = String(r.testName);
|
|
1456
|
-
let now = "file missing";
|
|
1457
|
-
if (fs.existsSync(path.join(top, rTest))) {
|
|
1458
|
-
const run = greenRun(rTest);
|
|
1459
|
-
if (run === null) return EXIT_COULD_NOT_RUN;
|
|
1460
|
-
const why = judgeGreen(run.cases, rName);
|
|
1461
|
-
now = why === null ? "passes now" : `not passing: ${why}`;
|
|
1462
|
-
}
|
|
1463
|
-
recorded.push(`\`${rTest}\` "${rName}" (${now})`);
|
|
1464
|
-
}
|
|
1465
|
-
gap(
|
|
1466
|
-
"red-mismatch",
|
|
1467
|
-
`no red for \`${t.test}\` "${t.name}"; reds recorded for ${recorded.join(", ")}`,
|
|
1468
|
-
);
|
|
1469
|
-
} else if (red === undefined) {
|
|
1470
|
-
gap(
|
|
1471
|
-
"no-red",
|
|
1472
|
-
`no red recorded for "${t.name}" — the test was never seen failing before the code`,
|
|
2418
|
+
const red = reds.get(redKey(s.task, t.test, t.name));
|
|
2419
|
+
if (red === undefined)
|
|
2420
|
+
noRed(
|
|
2421
|
+
`no tests step recorded a red for it (\`bkit tests red --group=${groupOf(s.task) ?? "N"}\`)`,
|
|
1473
2422
|
);
|
|
2423
|
+
else if (red.state === "passed-accepted") {
|
|
2424
|
+
row.red = red.state;
|
|
2425
|
+
landed.push({ task: s.task, test: t.test, name: t.name, reason: red.reason ?? "" });
|
|
2426
|
+
} else if (RED_PROVEN.has(red.state)) {
|
|
2427
|
+
row.red = red.state;
|
|
2428
|
+
if (red.state === "updated")
|
|
2429
|
+
updated.push(updatedEntry(top, s.task, t, red.base ?? null));
|
|
1474
2430
|
} else {
|
|
1475
|
-
row.red =
|
|
1476
|
-
|
|
1477
|
-
const on = typeof red.parentSha === "string" ? red.parentSha : undefined;
|
|
1478
|
-
if (code === undefined || on === undefined) lateUnchecked.add(s.task);
|
|
1479
|
-
else if (runGit(top, ["merge-base", "--is-ancestor", code, on]).ok) {
|
|
1480
|
-
row.red = `late — ${row.red}`;
|
|
1481
|
-
gap(
|
|
1482
|
-
"late-red",
|
|
1483
|
-
`the red for "${t.name}" was recorded on ${on.slice(0, 7)}, on or after ${code.slice(0, 7)}, the first commit whose plan-execute log records TASK ${s.task} DONE — the task's code already existed`,
|
|
1484
|
-
);
|
|
1485
|
-
}
|
|
1486
|
-
const snap = (Array.isArray(red.artifacts) ? red.artifacts : [])
|
|
1487
|
-
.map(String)
|
|
1488
|
-
.find((a) => a.endsWith(".snapshot"));
|
|
1489
|
-
const snapAbs = snap === undefined ? undefined : path.resolve(ctx.cwd, snap);
|
|
1490
|
-
const testAbs = path.join(top, t.test);
|
|
1491
|
-
if (snapAbs === undefined || !fs.existsSync(snapAbs)) {
|
|
1492
|
-
row.changed = "unknown";
|
|
1493
|
-
gap(
|
|
1494
|
-
"red-evidence-missing",
|
|
1495
|
-
`the red's test snapshot ${snap ?? "(none recorded)"} is missing`,
|
|
1496
|
-
);
|
|
1497
|
-
} else if (!fs.existsSync(testAbs)) {
|
|
1498
|
-
row.changed = "deleted";
|
|
1499
|
-
gap("test-changed", `\`${t.test}\` no longer exists`);
|
|
1500
|
-
} else if (!fs.readFileSync(snapAbs).equals(fs.readFileSync(testAbs))) {
|
|
1501
|
-
const added = onlyAddsLines(
|
|
1502
|
-
fs.readFileSync(snapAbs, "utf-8"),
|
|
1503
|
-
fs.readFileSync(testAbs, "utf-8"),
|
|
1504
|
-
);
|
|
1505
|
-
row.changed = added ? "added lines only" : "yes";
|
|
1506
|
-
if (!added)
|
|
1507
|
-
gap(
|
|
1508
|
-
"test-changed",
|
|
1509
|
-
`\`${t.test}\` differs from its red snapshot for "${t.name}" (diff below)`,
|
|
1510
|
-
);
|
|
1511
|
-
const diff = runGit(top, [
|
|
1512
|
-
"diff",
|
|
1513
|
-
"--no-index",
|
|
1514
|
-
"--no-color",
|
|
1515
|
-
"--",
|
|
1516
|
-
path.relative(top, snapAbs),
|
|
1517
|
-
t.test,
|
|
1518
|
-
]);
|
|
1519
|
-
diffs.push({ task: s.task, test: t.test, diff: diff.stdout });
|
|
1520
|
-
} else row.changed = "no";
|
|
2431
|
+
row.red = red.state;
|
|
2432
|
+
noRed(`the tests step left it as \`${red.state}\`, not red`);
|
|
1521
2433
|
}
|
|
1522
2434
|
|
|
1523
2435
|
if (!fs.existsSync(path.join(top, t.test))) {
|
|
@@ -1545,37 +2457,45 @@ export const testsCommand: CommandSpec = {
|
|
|
1545
2457
|
),
|
|
1546
2458
|
);
|
|
1547
2459
|
for (const e of parsed.errors)
|
|
1548
|
-
gaps.push(
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
2460
|
+
gaps.push(
|
|
2461
|
+
gapOf(
|
|
2462
|
+
null,
|
|
2463
|
+
"test-first-unread",
|
|
2464
|
+
`\`${plan}:${e.line}\` ${e.message} — tests there were not audited; the plan's blocks need fixing`,
|
|
2465
|
+
),
|
|
2466
|
+
);
|
|
2467
|
+
const verdict = verdictOf(gaps);
|
|
1554
2468
|
writeAuditReport(log, {
|
|
1555
2469
|
enforced: true,
|
|
1556
2470
|
plan,
|
|
2471
|
+
verdict,
|
|
1557
2472
|
rows,
|
|
1558
2473
|
gaps,
|
|
1559
|
-
testOrderOnly,
|
|
1560
2474
|
exceptions,
|
|
1561
|
-
|
|
1562
|
-
|
|
2475
|
+
landed,
|
|
2476
|
+
unproven,
|
|
2477
|
+
updated,
|
|
1563
2478
|
outputs,
|
|
1564
2479
|
});
|
|
1565
2480
|
ctx.emit({
|
|
1566
2481
|
enforced: true,
|
|
1567
2482
|
plan,
|
|
2483
|
+
verdict,
|
|
1568
2484
|
gaps,
|
|
1569
|
-
|
|
1570
|
-
|
|
2485
|
+
alreadyLanded: landed,
|
|
2486
|
+
unproven,
|
|
2487
|
+
updated,
|
|
1571
2488
|
log: logArg,
|
|
1572
2489
|
});
|
|
1573
|
-
ctx.log(
|
|
2490
|
+
ctx.log(
|
|
2491
|
+
`test-first audit: ${gaps.length} gap(s), ${unproven.length} unproven — report in ${logArg}`,
|
|
2492
|
+
);
|
|
1574
2493
|
return gaps.length === 0 ? EXIT.OK : EXIT.NO;
|
|
1575
2494
|
},
|
|
1576
2495
|
},
|
|
1577
2496
|
"plan-check": {
|
|
1578
|
-
summary: `Check a plan's group and task blocks: the block format, DONE WHEN and TEST FIRST on every task, DEPENDS ON order, the caps from package.json \`bkit.workflows.tasks\` (defaults: ${capsText(CONFIG_DEFAULTS.tasks)}), a Reuse Inventory NEW row per CREATE file, and that each READ / UPDATE / DELETE file and symbol exists; exit 0 no problems, 1 refused.`,
|
|
2497
|
+
summary: `Check a plan's group and task blocks: the block format, DONE WHEN and TEST FIRST on every task, DEPENDS ON order, the caps from package.json \`bkit.workflows.tasks\` (defaults: ${capsText(CONFIG_DEFAULTS.tasks)}), a Reuse Inventory NEW row per CREATE file, a Supersession Inventory row per DELETE, a \`Task\` on every Derived Test Matrix row and a row per test-bearing task, and that each READ / UPDATE / DELETE file and symbol exists; exit 0 no problems, 1 refused.`,
|
|
2498
|
+
use: "by-hand",
|
|
1579
2499
|
flags: [
|
|
1580
2500
|
{
|
|
1581
2501
|
name: "plan",
|
|
@@ -1601,9 +2521,11 @@ export const testsCommand: CommandSpec = {
|
|
|
1601
2521
|
"[{group, title, tasks, tests, kb}]: each group's tasks, planned TEST FIRST tests and size",
|
|
1602
2522
|
tasks: "[{task, group, tests, files}]: each task's planned tests and distinct test files",
|
|
1603
2523
|
problems:
|
|
1604
|
-
"[{line, kind, message}]: format | stray | field | depends | cap | inventory | markers — each refuses the plan",
|
|
2524
|
+
"[{line, kind, message}]: format | stray | field | depends | cap | inventory | matrix | supersession | markers — each refuses the plan",
|
|
1605
2525
|
existence:
|
|
1606
2526
|
"[{line, task, message}]: READ / UPDATE / DELETE files or symbols not found; refuses only with --existence=refuse",
|
|
2527
|
+
existing:
|
|
2528
|
+
"[{line, task, test, name}]: planned tests of open tasks whose name a test in their file already carries; reported only, never refuses",
|
|
1607
2529
|
log: "where the report was written (as given)",
|
|
1608
2530
|
},
|
|
1609
2531
|
run: (ctx) => {
|
|
@@ -1623,6 +2545,7 @@ export const testsCommand: CommandSpec = {
|
|
|
1623
2545
|
const caps = planCaps(ctx.cwd);
|
|
1624
2546
|
const { groups, sizes, problems } = checkPlanText(text, caps);
|
|
1625
2547
|
const existence = planExistence(text, toplevel(ctx.cwd));
|
|
2548
|
+
const existing = existingTestNames(text, toplevel(ctx.cwd));
|
|
1626
2549
|
const refused = problems.length > 0 || (existenceArg === "refuse" && existence.length > 0);
|
|
1627
2550
|
|
|
1628
2551
|
const out = [
|
|
@@ -1667,10 +2590,16 @@ export const testsCommand: CommandSpec = {
|
|
|
1667
2590
|
);
|
|
1668
2591
|
if (existence.length === 0) out.push("None.");
|
|
1669
2592
|
for (const e of existence) out.push(`- \`${plan}:${e.line}\` — ${e.message}`);
|
|
2593
|
+
if (existing.length > 0)
|
|
2594
|
+
out.push("", "## Planned test names already in use (reported only)", "");
|
|
2595
|
+
for (const e of existing)
|
|
2596
|
+
out.push(
|
|
2597
|
+
`- \`${plan}:${e.line}\` TASK ${e.task}: \`${e.test}\` already holds a test named "${e.name}" — name the update to it in TEST FIRST, or give the new test a new name`,
|
|
2598
|
+
);
|
|
1670
2599
|
const log = path.resolve(ctx.cwd, logArg);
|
|
1671
2600
|
fs.mkdirSync(path.dirname(log), { recursive: true });
|
|
1672
2601
|
fs.writeFileSync(log, `${out.join("\n")}\n`);
|
|
1673
|
-
ctx.emit({ plan, groups, tasks: sizes, problems, existence, log: logArg });
|
|
2602
|
+
ctx.emit({ plan, groups, tasks: sizes, problems, existence, existing, log: logArg });
|
|
1674
2603
|
ctx.log(
|
|
1675
2604
|
`plan check: ${problems.length} problem(s), ${existence.length} existence finding(s)${refused ? " — refused" : ""}; report in ${logArg}`,
|
|
1676
2605
|
);
|