@cbartens/bkit 6.0.0 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bkit/README.md +154 -151
- package/.bkit/core/.agents/context.md +95 -29
- package/.bkit/core/.agents/docs.md +17 -15
- package/.bkit/core/.agents/plugins/bkit/hooks/bkit-gateguard.cjs +1 -0
- package/.bkit/core/.agents/plugins/bkit/hooks/bkit-guardrails.cjs +6 -5
- package/.bkit/core/.agents/plugins/bkit/hooks/bkit-readguard.cjs +1 -0
- package/.bkit/core/.agents/plugins/bkit/hooks/bkit-rules.cjs +65 -0
- package/.bkit/core/.agents/rules.md +108 -107
- package/.bkit/core/.agents/skills.md +63 -15
- package/.bkit/core/.bkit/README.md +154 -151
- package/.bkit/core/.bkit/authoring.md +714 -84
- package/.bkit/core/.bkit/cli.md +407 -0
- package/.bkit/core/.bkit/cloud.md +281 -193
- package/.bkit/core/.bkit/comparison.md +43 -41
- package/.bkit/core/.bkit/install.md +154 -84
- package/.bkit/core/.bkit/jsonl.md +193 -130
- package/.bkit/core/.bkit/pipeline.md +114 -114
- package/.bkit/core/.bkit/prompts.md +272 -165
- package/.bkit/core/.bkit/providers.md +457 -271
- package/.bkit/core/.bkit/router.md +69 -0
- package/.bkit/core/.bkit/templates.md +529 -256
- package/.bkit/core/.bkit/trackers.md +80 -52
- package/.bkit/core/.bkit/workflows.md +589 -504
- package/.bkit/core/.claude/hooks/bkit-gateguard.cjs +2 -2
- package/.bkit/core/.claude/hooks/bkit-guardrails.cjs +3 -3
- package/.bkit/core/.claude/hooks/bkit-preflight.sh +45 -14
- package/.bkit/core/.codex/hooks/bkit-guardrails.cjs +3 -3
- package/.bkit/core/.codex/hooks/bkit-rules.cjs +57 -0
- package/.bkit/core/.cursor/hooks/bkit-guardrails.cjs +7 -7
- package/.bkit/core/.cursor/hooks/bkit-preflight.sh +47 -12
- package/.bkit/core/.opencode/plugin/bkit-rules.js +89 -0
- package/.bkit/core/AGENTS.md +5 -73
- package/.bkit/core/docs/templates/bkit-audit-accessibility.md +127 -105
- package/.bkit/core/docs/templates/bkit-audit-adoption.md +104 -70
- package/.bkit/core/docs/templates/bkit-audit-agents.md +94 -72
- package/.bkit/core/docs/templates/bkit-audit-aio-seo.md +99 -78
- package/.bkit/core/docs/templates/bkit-audit-analytics.md +140 -82
- package/.bkit/core/docs/templates/bkit-audit-annotations.md +98 -72
- package/.bkit/core/docs/templates/bkit-audit-app-performance.md +121 -96
- package/.bkit/core/docs/templates/bkit-audit-app-store.md +157 -76
- package/.bkit/core/docs/templates/bkit-audit-architecture.md +113 -71
- package/.bkit/core/docs/templates/bkit-audit-ci-performance.md +75 -72
- package/.bkit/core/docs/templates/bkit-audit-config.md +122 -76
- package/.bkit/core/docs/templates/bkit-audit-cost.md +85 -74
- package/.bkit/core/docs/templates/bkit-audit-coverage.md +75 -68
- package/.bkit/core/docs/templates/bkit-audit-database.md +101 -70
- package/.bkit/core/docs/templates/bkit-audit-dead-code.md +164 -83
- package/.bkit/core/docs/templates/bkit-audit-deferrals.md +115 -86
- package/.bkit/core/docs/templates/bkit-audit-dependencies.md +264 -88
- package/.bkit/core/docs/templates/bkit-audit-documentation.md +118 -93
- package/.bkit/core/docs/templates/bkit-audit-duplication.md +139 -86
- package/.bkit/core/docs/templates/bkit-audit-failures.md +110 -81
- package/.bkit/core/docs/templates/bkit-audit-flakes.md +99 -74
- package/.bkit/core/docs/templates/bkit-audit-gates.md +105 -77
- package/.bkit/core/docs/templates/bkit-audit-issues.md +104 -80
- package/.bkit/core/docs/templates/bkit-audit-llm.md +131 -58
- package/.bkit/core/docs/templates/bkit-audit-logs.md +123 -100
- package/.bkit/core/docs/templates/bkit-audit-models.md +98 -53
- package/.bkit/core/docs/templates/bkit-audit-privacy.md +56 -52
- package/.bkit/core/docs/templates/bkit-audit-prompts.md +63 -54
- package/.bkit/core/docs/templates/bkit-audit-red-team.md +124 -68
- package/.bkit/core/docs/templates/bkit-audit-reliability.md +95 -70
- package/.bkit/core/docs/templates/bkit-audit-repo.md +169 -73
- package/.bkit/core/docs/templates/bkit-audit-security.md +206 -96
- package/.bkit/core/docs/templates/bkit-audit-slop.md +77 -70
- package/.bkit/core/docs/templates/bkit-audit-testing.md +162 -114
- package/.bkit/core/docs/templates/bkit-audit-usability.md +113 -71
- package/.bkit/core/docs/templates/bkit-plan.md +138 -132
- package/.bkit/core/engine/checks/after/auth.ts +40 -37
- package/.bkit/core/engine/checks/after/ci-yml.ts +65 -2
- package/.bkit/core/engine/checks/after/index.ts +12 -2
- package/.bkit/core/engine/checks/after/tools.ts +80 -33
- package/.bkit/core/engine/checks/before/context-fetch.ts +227 -33
- package/.bkit/core/engine/checks/before/env.ts +1 -1
- package/.bkit/core/engine/checks/before/index.ts +7 -0
- package/.bkit/core/engine/checks/before/repo.ts +40 -0
- package/.bkit/core/engine/checks/status-holes.ts +916 -123
- package/.bkit/core/engine/checks/tips/catalogue.ts +68 -17
- package/.bkit/core/engine/checks/tips/detect.ts +32 -0
- package/.bkit/core/engine/checks/tips/index.ts +47 -6
- package/.bkit/core/engine/cli.ts +238 -84
- package/.bkit/core/engine/commands/analysis.ts +128 -76
- package/.bkit/core/engine/commands/ci/ci.ts +363 -29
- package/.bkit/core/engine/commands/ci/pins.ts +172 -0
- package/.bkit/core/engine/commands/ci/push-gate.ts +223 -0
- package/.bkit/core/engine/commands/install/uninstall.ts +21 -27
- package/.bkit/core/engine/commands/install/upgrade.ts +32 -15
- package/.bkit/core/engine/commands/land.ts +126 -11
- package/.bkit/core/engine/commands/lib.ts +303 -6
- package/.bkit/core/engine/commands/registry.ts +6 -2
- package/.bkit/core/engine/commands/setup/setup.ts +1 -0
- package/.bkit/core/engine/commands/stats.ts +91 -35
- package/.bkit/core/engine/commands/workflow/branches.ts +1477 -117
- package/.bkit/core/engine/commands/workflow/challenge.ts +114 -39
- package/.bkit/core/engine/commands/workflow/completeness.ts +109 -0
- package/.bkit/core/engine/commands/workflow/context.ts +433 -55
- package/.bkit/core/engine/commands/workflow/counters.ts +244 -39
- package/.bkit/core/engine/commands/workflow/decisions.ts +61 -17
- package/.bkit/core/engine/commands/workflow/disk.ts +237 -55
- package/.bkit/core/engine/commands/workflow/fids.ts +19 -1
- package/.bkit/core/engine/commands/workflow/issues.ts +6 -0
- package/.bkit/core/engine/commands/workflow/ledgers.ts +458 -18
- package/.bkit/core/engine/commands/workflow/locks.ts +2 -0
- package/.bkit/core/engine/commands/workflow/plan-blocks.ts +124 -15
- package/.bkit/core/engine/commands/workflow/plans.ts +257 -0
- package/.bkit/core/engine/commands/workflow/prs.ts +12 -3
- package/.bkit/core/engine/commands/workflow/repo.ts +8 -0
- package/.bkit/core/engine/commands/workflow/roadmap.ts +56 -15
- package/.bkit/core/engine/commands/workflow/run-facts.ts +22 -31
- package/.bkit/core/engine/commands/workflow/runs.ts +652 -34
- package/.bkit/core/engine/commands/workflow/runtime.ts +3 -0
- package/.bkit/core/engine/commands/workflow/skills.ts +1 -0
- package/.bkit/core/engine/commands/workflow/status.ts +200 -17
- package/.bkit/core/engine/commands/workflow/step-table.ts +75 -33
- package/.bkit/core/engine/commands/workflow/steps.ts +635 -24
- package/.bkit/core/engine/commands/workflow/summary-findings.ts +177 -0
- package/.bkit/core/engine/commands/workflow/summary.ts +192 -46
- package/.bkit/core/engine/commands/workflow/tasks.ts +409 -0
- package/.bkit/core/engine/commands/workflow/test-blocks.ts +254 -0
- package/.bkit/core/engine/commands/workflow/tests-changed.ts +201 -0
- package/.bkit/core/engine/commands/workflow/tests.ts +1285 -356
- package/.bkit/core/engine/commands/workflow/workflows.ts +228 -43
- package/.bkit/core/engine/commands/workflow/worktrees.ts +389 -47
- package/.bkit/core/engine/install/cloud-env.ts +251 -121
- package/.bkit/core/engine/install/context-ci-workflow.ts +102 -0
- package/.bkit/core/engine/install/manifest.ts +60 -12
- package/.bkit/core/engine/install/modules/antigravity.ts +3 -2
- package/.bkit/core/engine/install/modules/cloud.ts +3 -3
- package/.bkit/core/engine/install/modules/codex.ts +15 -2
- package/.bkit/core/engine/install/modules/core.ts +956 -52
- package/.bkit/core/engine/install/modules/cursor.ts +6 -6
- package/.bkit/core/engine/install/modules/opencode.ts +16 -4
- package/.bkit/core/engine/install/template-notes.ts +130 -0
- package/.bkit/core/engine/install/version-file.ts +3 -2
- package/.bkit/core/engine/install/write.ts +5 -0
- package/.bkit/core/engine/ledgers/compact.ts +124 -0
- package/.bkit/core/engine/ledgers/journal.ts +291 -0
- package/.bkit/core/engine/ledgers/status.ts +160 -9
- package/.bkit/core/engine/ledgers/telemetry.ts +158 -21
- package/.bkit/core/engine/render/hosts.ts +114 -14
- package/.bkit/core/engine/shared/bkit-latest.ts +26 -4
- package/.bkit/core/engine/shared/dep-updater.ts +60 -0
- package/.bkit/core/engine/shared/egress.ts +1 -1
- package/.bkit/core/engine/shared/package-manager.ts +9 -0
- package/.bkit/core/engine/shared/settings.ts +92 -17
- package/.bkit/core/engine/shared/timeouts.ts +2 -0
- package/.bkit/core/engine/tools/cli-doc.ts +111 -0
- package/.bkit/core/engine/tools/grammar-density.ts +1 -1
- package/.bkit/core/engine/tools/instructions-sources.ts +15 -17
- package/.bkit/core/engine/tools/instructions.ts +94 -43
- package/.bkit/core/hooks/antigravity-hook.js +25 -10
- package/.bkit/core/hooks/gateguard-engine.js +53 -29
- package/.bkit/core/hooks/guardrails-shell.js +4 -4
- package/.bkit/core/hooks/guardrails-tiers.js +3 -3
- package/.bkit/core/hooks/preflight-linear.sh +9 -9
- package/.bkit/core/hooks/preflight-rules.sh +8 -0
- package/.bkit/core/hooks/readguard-engine.js +29 -10
- package/.bkit/core/hooks/session-rules-advisory.js +61 -0
- package/.bkit/core/prompts/bkit-audit-generate.md +187 -116
- package/.bkit/core/prompts/bkit-build-challenge-1.md +56 -37
- package/.bkit/core/prompts/bkit-build-challenge-2.md +65 -45
- package/.bkit/core/prompts/bkit-build-install-skills.md +38 -29
- package/.bkit/core/prompts/bkit-build-plan-convert.md +43 -32
- package/.bkit/core/prompts/bkit-build-plan-current.md +49 -39
- package/.bkit/core/prompts/bkit-build-plan-gaps-remediate.md +79 -55
- package/.bkit/core/prompts/bkit-build-plan-tasks.md +149 -0
- package/.bkit/core/prompts/bkit-build-plan-tests.md +120 -0
- package/.bkit/core/prompts/bkit-build-plan-verify.md +87 -50
- package/.bkit/core/prompts/bkit-debug-abort-handover.md +36 -35
- package/.bkit/core/prompts/bkit-debug-failure-diagnose.md +96 -64
- package/.bkit/core/prompts/bkit-debug-failure-remediate.md +54 -45
- package/.bkit/core/prompts/bkit-debug-failure-reproduce.md +42 -33
- package/.bkit/core/prompts/bkit-debug-failure-soak.md +53 -37
- package/.bkit/core/prompts/bkit-design-brainstorm-ideate-1.md +35 -30
- package/.bkit/core/prompts/bkit-design-brainstorm-ideate-2.md +34 -29
- package/.bkit/core/prompts/bkit-design-brainstorm-ideate-3.md +32 -28
- package/.bkit/core/prompts/bkit-design-generate.md +68 -52
- package/.bkit/core/prompts/bkit-design-organise.md +73 -35
- package/.bkit/core/prompts/bkit-design-review-1.md +58 -54
- package/.bkit/core/prompts/bkit-design-review-2.md +54 -50
- package/.bkit/core/prompts/bkit-design-spike.md +54 -47
- package/.bkit/core/prompts/bkit-design-update.md +42 -31
- package/.bkit/core/prompts/bkit-plan-generate.md +76 -58
- package/.bkit/core/prompts/bkit-plan-review-1.md +63 -49
- package/.bkit/core/prompts/bkit-plan-review-2.md +60 -48
- package/.bkit/core/prompts/bkit-plan-scope-check.md +53 -46
- package/.bkit/core/prompts/bkit-plan-update.md +70 -52
- package/.bkit/core/prompts/bkit-ship-conflict-resolve.md +82 -50
- package/.bkit/core/prompts/bkit-ship-deferred-sweep.md +80 -45
- package/.bkit/core/prompts/bkit-ship-update-docs.md +35 -29
- package/.bkit/core/prompts/bkit-utility-compress-markdown.md +55 -49
- package/.bkit/core/prompts/bkit-utility-update-decisions.md +76 -49
- package/.bkit/core/prompts/bkit-utility-update-deferrals.md +70 -41
- package/.bkit/core/prompts/bkit-utility-update-documentation.md +33 -30
- package/.bkit/core/prompts/bkit-utility-update-failures.md +73 -43
- package/.bkit/core/prompts/bkit-utility-update-knowledge.md +77 -50
- package/.bkit/core/prompts/bkit-utility-update-roadmap.md +77 -45
- package/.bkit/core/prompts/bkit-utility-update-tracker.md +144 -68
- package/.bkit/core/workflows/bkit-auto.md +53 -30
- package/.bkit/core/workflows/bkit-build.md +635 -304
- package/.bkit/core/workflows/bkit-debug.md +301 -148
- package/.bkit/core/workflows/bkit-design.md +167 -95
- package/.bkit/core/workflows/bkit-plan.md +172 -84
- package/.bkit/core/workflows/bkit-ship.md +299 -152
- package/.bkit/shared/.agents/plugins/bkit/hooks.json +10 -0
- package/.bkit/shared/.claude/bkit-cloud.md +12 -10
- package/.bkit/shared/.claude/settings.json +3 -0
- package/.bkit/shared/.codex/bkit-cloud.md +98 -0
- package/.bkit/shared/.codex/hooks.json +20 -0
- package/.bkit/shared/.cursor/bkit-cloud.md +3 -3
- package/.bkit/shared/.opencode/opencode.json +1 -1
- package/.bkit/shared/CLAUDE.md +5 -1
- package/.bkit/shared/GEMINI.md +4 -1
- package/.bkit/shared/docs/architecture.md +4 -7
- package/.bkit/shared/docs/data-schema.md +4 -7
- package/.bkit/shared/docs/deployments.md +239 -166
- package/.bkit/shared/docs/design.md +11 -12
- package/.bkit/shared/docs/noise.md +8 -7
- package/.bkit/shared/docs/observability.md +5 -8
- package/.bkit/shared/docs/security.md +35 -17
- package/.bkit/shared/docs/setup.md +29 -18
- package/.bkit/shared/docs/technology.md +3 -6
- package/.bkit/shared/docs/testing.md +282 -176
- package/.bkit/shared/gitignore +6 -2
- package/.bkit/templates/.claude/hooks/bkit-cloud-bootstrap.sh +15 -16
- package/.bkit/templates/.cursor/Dockerfile +12 -8
- package/.bkit/templates/.cursor/environment.json +1 -1
- package/.bkit/templates/.github/ISSUE_TEMPLATE/bkit-bug.yml +10 -8
- package/.bkit/templates/.github/ISSUE_TEMPLATE/bkit-change.yml +24 -12
- package/.bkit/templates/.github/ISSUE_TEMPLATE/bkit-idea.yml +3 -3
- package/.bkit/templates/.github/renovate.json5 +34 -6
- package/.bkit/templates/.github/workflows/bkit-ci.yml +53 -50
- package/.bkit/templates/.github/workflows/bkit-cloud.yml +13 -14
- package/.bkit/templates/.github/workflows/bkit-health.yml +97 -0
- package/.bkit/templates/.github/workflows/bkit-renovate.yml +5 -4
- package/.bkit/templates/.github/workflows/bkit-trivy.yml +5 -4
- package/.bkit/templates/.husky/pre-commit +2 -2
- package/.bkit/templates/.husky/pre-push +95 -35
- package/.bkit/templates/.prettierignore +6 -7
- package/.bkit/templates/.vale.ini +27 -27
- package/.bkit/templates/context/bkit-cloudrun.md +23 -15
- package/.bkit/templates/context/bkit-expo.md +70 -52
- package/.bkit/templates/context/bkit-github.md +93 -56
- package/.bkit/templates/context/bkit-inngest.md +21 -15
- package/.bkit/templates/context/bkit-maestro.md +22 -16
- package/.bkit/templates/context/bkit-playwright.md +23 -17
- package/.bkit/templates/context/bkit-resend.md +12 -10
- package/.bkit/templates/context/bkit-sentry.md +30 -20
- package/.bkit/templates/context/bkit-serena.md +8 -7
- package/.bkit/templates/context/bkit-stripe.md +14 -13
- package/.bkit/templates/context/bkit-supabase.md +16 -13
- package/.bkit/templates/context/bkit-tavily.md +14 -11
- package/.bkit/templates/context/bkit-vercel.md +27 -18
- package/.bkit/templates/docs/capabilities.md +15 -14
- package/.bkit/templates/scripts/ci/{check-accepted-divergences.mjs → bkit-check-accepted-divergences.mjs} +6 -3
- package/.bkit/templates/scripts/ci/{check-accepted-divergences.test.mjs → bkit-check-accepted-divergences.test.mjs} +3 -3
- package/.bkit/templates/scripts/ci/{check-capstone.mjs → bkit-check-capstone.mjs} +2 -2
- package/.bkit/templates/scripts/ci/{check-capstone.test.mjs → bkit-check-capstone.test.mjs} +4 -4
- package/.bkit/templates/scripts/ci/{check-dependency-audit.mjs → bkit-check-dependency-audit.mjs} +103 -18
- package/.bkit/templates/scripts/ci/{check-dependency-audit.test.mjs → bkit-check-dependency-audit.test.mjs} +31 -3
- package/.bkit/templates/scripts/ci/bkit-machine-lock.sh +387 -0
- package/.bkit/templates/scripts/ci/bkit-test-awake.mjs +75 -0
- package/.bkit/templates/scripts/ci/bkit-test-awake.test.mjs +160 -0
- package/.bkit/templates/scripts/ci/lanes/{resolve-push-base.mjs → bkit-resolve-push-base.mjs} +3 -3
- package/.bkit/templates/scripts/ci/lanes/{resolve-push-base.test.mjs → bkit-resolve-push-base.test.mjs} +4 -4
- package/.bkit/templates/scripts/cleanup/bkit-disk-prune.mjs +9 -1
- package/.bkit/templates/scripts/graphify/bkit-graph-drift.py +1 -1
- package/.bkit/templates/scripts/graphify/bkit-graph-eager.py +3 -3
- package/.bkit/templates/scripts/graphify/bkit-graph-embed.py +63 -0
- package/.bkit/templates/scripts/graphify/bkit-graph-imports.py +2 -2
- package/.bkit/templates/scripts/graphify/bkit-graph-pre-push.sh +69 -18
- package/.bkit/templates/scripts/graphify/bkit-graph-repair.py +3 -3
- package/.bkit/templates/scripts/graphify/bkit-graph-surface.py +1 -1
- package/.bkit/templates/scripts/graphify/bkit-graphify-graphml.py +1 -1
- package/.bkit/templates/scripts/graphify/bkit-graphify-rollup.py +197 -126
- package/.bkit/templates/scripts/graphify/bkit-graphify.sh +64 -41
- package/.bkit/templates/scripts/slop/bkit-slop-type-suppression.mjs +23 -7
- package/.bkit/templates/slop-scan.config.json +9 -2
- package/CHANGELOG.md +119 -0
- package/README.md +250 -76
- package/dist/checks/after/auth.js +36 -33
- package/dist/checks/after/ci-yml.js +52 -2
- package/dist/checks/after/index.js +12 -4
- package/dist/checks/after/tools.js +73 -33
- package/dist/checks/before/context-fetch.js +218 -33
- package/dist/checks/before/env.js +1 -1
- package/dist/checks/before/index.js +6 -2
- package/dist/checks/before/repo.js +35 -1
- package/dist/checks/status-holes.js +739 -114
- package/dist/checks/tips/catalogue.js +59 -17
- package/dist/checks/tips/detect.js +30 -0
- package/dist/checks/tips/index.js +48 -6
- package/dist/cli.js +222 -84
- package/dist/commands/analysis.js +77 -53
- package/dist/commands/ci/ci.js +302 -25
- package/dist/commands/ci/pins.js +142 -0
- package/dist/commands/ci/push-gate.js +194 -0
- package/dist/commands/install/uninstall.js +23 -27
- package/dist/commands/install/upgrade.js +21 -11
- package/dist/commands/land.js +126 -10
- package/dist/commands/lib.js +244 -5
- package/dist/commands/registry.js +6 -2
- package/dist/commands/setup/setup.js +1 -0
- package/dist/commands/stats.js +69 -30
- package/dist/commands/workflow/branches.js +1232 -111
- package/dist/commands/workflow/challenge.js +90 -37
- package/dist/commands/workflow/completeness.js +97 -0
- package/dist/commands/workflow/context.js +371 -51
- package/dist/commands/workflow/counters.js +206 -40
- package/dist/commands/workflow/decisions.js +60 -17
- package/dist/commands/workflow/disk.js +213 -55
- package/dist/commands/workflow/fids.js +18 -1
- package/dist/commands/workflow/issues.js +6 -0
- package/dist/commands/workflow/ledgers.js +372 -21
- package/dist/commands/workflow/locks.js +2 -0
- package/dist/commands/workflow/plan-blocks.js +78 -14
- package/dist/commands/workflow/plans.js +205 -0
- package/dist/commands/workflow/prs.js +12 -3
- package/dist/commands/workflow/repo.js +6 -0
- package/dist/commands/workflow/roadmap.js +50 -17
- package/dist/commands/workflow/run-facts.js +23 -28
- package/dist/commands/workflow/runs.js +530 -33
- package/dist/commands/workflow/runtime.js +3 -0
- package/dist/commands/workflow/skills.js +1 -0
- package/dist/commands/workflow/status.js +166 -19
- package/dist/commands/workflow/step-table.js +61 -29
- package/dist/commands/workflow/steps.js +542 -26
- package/dist/commands/workflow/summary-findings.js +117 -0
- package/dist/commands/workflow/summary.js +154 -41
- package/dist/commands/workflow/tasks.js +378 -0
- package/dist/commands/workflow/test-blocks.js +231 -0
- package/dist/commands/workflow/tests-changed.js +172 -0
- package/dist/commands/workflow/tests.js +1013 -321
- package/dist/commands/workflow/workflows.js +191 -38
- package/dist/commands/workflow/worktrees.js +310 -42
- package/dist/install/cloud-env.js +233 -99
- package/dist/install/context-ci-workflow.js +85 -0
- package/dist/install/manifest.js +57 -12
- package/dist/install/modules/antigravity.js +3 -2
- package/dist/install/modules/cloud.js +3 -3
- package/dist/install/modules/codex.js +12 -2
- package/dist/install/modules/core.js +851 -53
- package/dist/install/modules/cursor.js +6 -6
- package/dist/install/modules/opencode.js +11 -5
- package/dist/install/template-notes.js +116 -0
- package/dist/install/version-file.js +3 -2
- package/dist/ledgers/compact.js +87 -0
- package/dist/ledgers/journal.js +233 -0
- package/dist/ledgers/status.js +115 -5
- package/dist/ledgers/telemetry.js +121 -17
- package/dist/render/hosts.js +100 -14
- package/dist/shared/bkit-latest.js +26 -4
- package/dist/shared/dep-updater.js +49 -0
- package/dist/shared/egress.js +1 -1
- package/dist/shared/package-manager.js +6 -0
- package/dist/shared/settings.js +59 -10
- package/dist/shared/timeouts.js +2 -0
- package/dist/tools/cli-doc.js +97 -0
- package/dist/tools/grammar-density.js +1 -1
- package/dist/tools/instructions-sources.js +15 -16
- package/dist/tools/instructions.js +83 -39
- package/package.json +58 -42
- package/.bkit/core/.bkit/lite.md +0 -73
- package/.bkit/core/engine/commands/workflow/workstreams.ts +0 -172
- package/.bkit/core/prompts/bkit-build-plan-execute.md +0 -124
- package/.bkit/templates/scripts/ci/bkit-local-stack-lock.sh +0 -194
- package/dist/commands/workflow/workstreams.js +0 -149
- /package/.bkit/templates/.vale/styles/{House → Bkit}/Filler.yml +0 -0
- /package/.bkit/templates/.vale/styles/{House → Bkit}/Hedging.yml +0 -0
- /package/.bkit/templates/.vale/styles/{House → Bkit}/LLMSlop.yml +0 -0
- /package/.bkit/templates/.vale/styles/{House → Bkit}/Verbose.yml +0 -0
- /package/.bkit/templates/scripts/ci/{accepted-advisories.json → bkit-accepted-advisories.json} +0 -0
- /package/.bkit/templates/scripts/ci/{accepted-divergences.json → bkit-accepted-divergences.json} +0 -0
- /package/.bkit/templates/scripts/launchd/{launchd.env.example → bkit-launchd.env.example} +0 -0
- /package/.bkit/templates/scripts/launchd/{ai.bkit.launchd.plist → bkit-launchd.plist} +0 -0
|
@@ -8,35 +8,42 @@
|
|
|
8
8
|
// ran (not executable / not found), so it is not recorded as a failing test.
|
|
9
9
|
// The failure count comes from the JUnit reports the gate's runners write into
|
|
10
10
|
// `{log}.reports/` (`BKIT_TEST_REPORT_DIR`), never from reading the log.
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
// file through the consumer's `bkit:test:file` script
|
|
14
|
-
// never
|
|
15
|
-
// unwritten code all exit non-zero exactly like a real failure. A red
|
|
16
|
-
//
|
|
11
|
+
// bkit tests red / green — test-first evidence for build tasks. `red` runs every TEST FIRST
|
|
12
|
+
// test of one group's open tasks after the tests step wrote them and before any task code
|
|
13
|
+
// exists: one run per test file through the consumer's `bkit:test:file` script, each test
|
|
14
|
+
// judged by the JUnit report, never the exit code alone (a misspelled path, a syntax error
|
|
15
|
+
// and an import of unwritten code all exit non-zero exactly like a real failure). A red is
|
|
16
|
+
// the NAMED test failing, or its file failing to load; `green` reads the group's red row and
|
|
17
|
+
// needs the named test to run and pass.
|
|
17
18
|
import { spawnSync } from "node:child_process";
|
|
19
|
+
import { createHash } from "node:crypto";
|
|
18
20
|
import * as fs from "node:fs";
|
|
19
21
|
import * as os from "node:os";
|
|
20
22
|
import * as path from "node:path";
|
|
21
|
-
import { BKIT_TEST_FILE_SCRIPT } from "../../install/manifest.js";
|
|
23
|
+
import { BKIT_LINT_FILE_SCRIPT, BKIT_TEST_FILE_SCRIPT } from "../../install/manifest.js";
|
|
22
24
|
import { clampLogPath } from "../../ledgers/clamp-log.js";
|
|
23
|
-
import {
|
|
25
|
+
import { appendLine, statusTimestamp, validateCiSelection, testRunProblem, } from "../../ledgers/status.js";
|
|
26
|
+
import { classifyLockExit, CommandError, EXIT, git, runGit, testsLockHolder, wholeNumber, } from "../lib.js";
|
|
24
27
|
import { parseJson } from "../../shared/json.js";
|
|
25
28
|
import { readPackageJson } from "../../shared/package-json.js";
|
|
26
|
-
import { COULD_NOT_RUN_EXIT_CODES, scriptCommand } from "../../shared/package-manager.js";
|
|
29
|
+
import { COULD_NOT_RUN_EXIT_CODES, scriptCommand, testCommand, } from "../../shared/package-manager.js";
|
|
27
30
|
import { CONFIG_DEFAULTS, readBkitConfig } from "../../shared/settings.js";
|
|
28
|
-
import { FILE_FLAG, GROUP_FLAG, WORKFLOW_FLAG, positiveInt, readStatusRows, statusFile, str, } from "./status.js";
|
|
29
|
-
import { fenceNeverClosed, parsePlanBlocks, reuseInventory, } from "./plan-blocks.js";
|
|
31
|
+
import { FILE_FLAG, GROUP_FLAG, WORKFLOW_FLAG, positiveInt, readStatusRows, readRuns, statusFile, str, } from "./status.js";
|
|
32
|
+
import { NO_MARKERS_REASON, derivedTestMatrix, fenceNeverClosed, parsePlanBlocks, reuseInventory, supersessionInventory, } from "./plan-blocks.js";
|
|
30
33
|
import { writeStepRow } from "./steps.js";
|
|
31
34
|
import { appendRecord, clampFile } from "./context.js";
|
|
35
|
+
import { testsChangedVerb } from "./tests-changed.js";
|
|
32
36
|
/** Exit code: no candidate gate is declared in package.json. */
|
|
33
37
|
export const EXIT_NO_GATE = 3;
|
|
34
38
|
/** Exit code: the gate could not run (exit 126 / 127, or the package runner is missing). */
|
|
35
39
|
export const EXIT_COULD_NOT_RUN = 4;
|
|
36
40
|
/** Exit code: --artifact could not be written, trimmed or recorded. */
|
|
37
41
|
export const EXIT_ARTIFACT_STEP = 5;
|
|
42
|
+
/** The status row type `tests run --file` appends, one per run. */
|
|
43
|
+
export const TEST_RUN_ROW = "test-run";
|
|
38
44
|
/** The env var naming the folder every runner a gate calls writes its JUnit XML into. */
|
|
39
45
|
export const REPORT_DIR_ENV = "BKIT_TEST_REPORT_DIR";
|
|
46
|
+
export { classifyLockExit } from "../lib.js";
|
|
40
47
|
const NO_REPORT_HINT = `no JUnit report in $${REPORT_DIR_ENV} — failures unknown; turn on each test runner's JUnit ` +
|
|
41
48
|
`reporter in the gate and point its output into $${REPORT_DIR_ENV} ` +
|
|
42
49
|
"(.bkit/workflows.md § Local gate scripts)";
|
|
@@ -67,7 +74,8 @@ export const gateScope = (gate) => scriptCommand(gate).join(" ");
|
|
|
67
74
|
export function runToLog(argv, cwd, log, env) {
|
|
68
75
|
fs.mkdirSync(path.dirname(log), { recursive: true });
|
|
69
76
|
const fd = fs.openSync(log, "w");
|
|
70
|
-
const
|
|
77
|
+
const protectedArgv = testCommand(argv);
|
|
78
|
+
const res = spawnSync(protectedArgv[0], protectedArgv.slice(1), {
|
|
71
79
|
cwd,
|
|
72
80
|
stdio: ["ignore", fd, fd],
|
|
73
81
|
env: env === undefined ? process.env : { ...process.env, ...env },
|
|
@@ -102,13 +110,286 @@ const tailLines = (file, n) => fs
|
|
|
102
110
|
.replace(/\r?\n$/, "")
|
|
103
111
|
.split(/\r?\n/)
|
|
104
112
|
.slice(-n);
|
|
113
|
+
/**
|
|
114
|
+
* Append one `test-run` row to `file`. `run` numbers this step's test-run rows in the
|
|
115
|
+
* workflow's active run (since its last workflow-started row), this one included, so a
|
|
116
|
+
* gate re-run under the same attempt still shows.
|
|
117
|
+
*/
|
|
118
|
+
function appendTestRun(file, r) {
|
|
119
|
+
const rows = readStatusRows(file);
|
|
120
|
+
let start = -1;
|
|
121
|
+
rows.forEach((x, i) => {
|
|
122
|
+
if (x.type === "workflow-started" && x.workflow === r.workflow)
|
|
123
|
+
start = i;
|
|
124
|
+
});
|
|
125
|
+
const run = rows
|
|
126
|
+
.slice(start + 1)
|
|
127
|
+
.filter((x) => x.type === TEST_RUN_ROW && x.workflow === r.workflow && x.step === r.step)
|
|
128
|
+
.length + 1;
|
|
129
|
+
const row = {
|
|
130
|
+
type: TEST_RUN_ROW,
|
|
131
|
+
workflow: r.workflow,
|
|
132
|
+
at: statusTimestamp(),
|
|
133
|
+
step: r.step,
|
|
134
|
+
...(r.attempt === undefined ? {} : { attempt: r.attempt }),
|
|
135
|
+
run,
|
|
136
|
+
verdict: r.verdict,
|
|
137
|
+
exitCode: r.exitCode,
|
|
138
|
+
failures: r.failures,
|
|
139
|
+
scope: r.scope,
|
|
140
|
+
log: r.log,
|
|
141
|
+
...(r.artifact === undefined ? {} : { artifact: r.artifact }),
|
|
142
|
+
...(r.source === undefined ? {} : { source: r.source }),
|
|
143
|
+
...r.evidence,
|
|
144
|
+
};
|
|
145
|
+
const problem = testRunProblem(row);
|
|
146
|
+
if (problem !== null)
|
|
147
|
+
throw new CommandError(EXIT.USAGE, problem);
|
|
148
|
+
fs.appendFileSync(file, appendLine(row));
|
|
149
|
+
return row;
|
|
150
|
+
}
|
|
105
151
|
// ---------------------------------------------------------------------------
|
|
106
152
|
// Test-first: JUnit reading, plan `Test first:` lines, red / green judgement
|
|
107
153
|
// ---------------------------------------------------------------------------
|
|
108
154
|
/** The env var naming the JUnit report path `bkit:test:file` must write. */
|
|
109
155
|
export const REPORT_ENV = "BKIT_TEST_REPORT";
|
|
110
|
-
/** The status row step a
|
|
111
|
-
export const RED_STEP = "build-
|
|
156
|
+
/** The status row step a group's red writes: one row per `tests red` run, a state and blob per planned test. */
|
|
157
|
+
export const RED_STEP = "build-tests-red";
|
|
158
|
+
/**
|
|
159
|
+
* The states that count as a red: the test failed, its file did not load, the tests step
|
|
160
|
+
* vouched for its pass, or it is an existing test updated and passing, whose diff the
|
|
161
|
+
* verifier judges (it may add expectations, never remove or loosen one).
|
|
162
|
+
*/
|
|
163
|
+
export const RED_PROVEN = new Set([
|
|
164
|
+
"red",
|
|
165
|
+
"red-not-loaded",
|
|
166
|
+
"passed-accepted",
|
|
167
|
+
"updated",
|
|
168
|
+
]);
|
|
169
|
+
/** The `build-tests-red` rows of `workflow` in status.jsonl, in file order; `group` narrows. */
|
|
170
|
+
function redRows(file, workflow, group) {
|
|
171
|
+
if (!fs.existsSync(file))
|
|
172
|
+
return [];
|
|
173
|
+
return readStatusRows(file).filter((r) => r.type === "step-done" &&
|
|
174
|
+
r.workflow === workflow &&
|
|
175
|
+
r.step === RED_STEP &&
|
|
176
|
+
typeof r.group === "number" &&
|
|
177
|
+
(group === undefined || r.group === group));
|
|
178
|
+
}
|
|
179
|
+
/** A red row's `tests`, as written. */
|
|
180
|
+
export const redTestsOf = (row) => Array.isArray(row.tests) ? row.tests : [];
|
|
181
|
+
/** The key a planned test is looked up by: task numbers are unique across the plan. */
|
|
182
|
+
export const redKey = (task, test, name) => JSON.stringify([task, test, name]);
|
|
183
|
+
/** The number this group's next red row carries: its rows so far, plus one. */
|
|
184
|
+
function redAttempt(file, workflow, group) {
|
|
185
|
+
if (!fs.existsSync(file))
|
|
186
|
+
return 1;
|
|
187
|
+
return (readStatusRows(file).filter((r) => {
|
|
188
|
+
const row = r;
|
|
189
|
+
return (row.type === "step-done" &&
|
|
190
|
+
row.workflow === workflow &&
|
|
191
|
+
row.step === RED_STEP &&
|
|
192
|
+
row.group === group);
|
|
193
|
+
}).length + 1);
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Every planned test the red rows recorded, keyed by task, file and name. A later row's entry
|
|
197
|
+
* wins, so a retry or a later task of the group overrides only the tests it ran, except that a
|
|
198
|
+
* proven red is never replaced by an unproven result for the same test lines. `group` narrows
|
|
199
|
+
* to one group.
|
|
200
|
+
*/
|
|
201
|
+
export function redEntries(file, workflow, group) {
|
|
202
|
+
const out = new Map();
|
|
203
|
+
for (const row of redRows(file, workflow, group))
|
|
204
|
+
for (const t of redTestsOf(row)) {
|
|
205
|
+
const key = redKey(t.task, t.test, t.testName);
|
|
206
|
+
const earlier = out.get(key);
|
|
207
|
+
const keepsProof = earlier !== undefined &&
|
|
208
|
+
RED_PROVEN.has(earlier.state) &&
|
|
209
|
+
!RED_PROVEN.has(t.state) &&
|
|
210
|
+
typeof earlier.block === "string" &&
|
|
211
|
+
earlier.block === t.block;
|
|
212
|
+
if (!keepsProof)
|
|
213
|
+
out.set(key, t);
|
|
214
|
+
}
|
|
215
|
+
return out;
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* The planned tests of `group` a code step changed since their red: a test whose own lines
|
|
219
|
+
* (`testBlock`) no longer hash to the recorded block. A row with no block (written before
|
|
220
|
+
* per-test locking, or when no line carried the name) derives the block from the recorded
|
|
221
|
+
* blob; when that holds no block either, the whole file's blob is compared. Other edits to a
|
|
222
|
+
* test file are free.
|
|
223
|
+
*/
|
|
224
|
+
export function touchedTests(top, file, workflow, group) {
|
|
225
|
+
const touched = [];
|
|
226
|
+
const texts = new Map();
|
|
227
|
+
const textOf = (test) => {
|
|
228
|
+
if (!texts.has(test)) {
|
|
229
|
+
const abs = path.join(top, test);
|
|
230
|
+
texts.set(test, fs.existsSync(abs) ? fs.readFileSync(abs, "utf-8") : null);
|
|
231
|
+
}
|
|
232
|
+
return texts.get(test);
|
|
233
|
+
};
|
|
234
|
+
for (const t of redEntries(file, workflow, group).values()) {
|
|
235
|
+
let block = t.block ?? null;
|
|
236
|
+
if (block === null)
|
|
237
|
+
block = blockHash(blobText(top, t.blob), t.testName);
|
|
238
|
+
if (block === null) {
|
|
239
|
+
if (git(top, ["hash-object", "--", t.test]) !== t.blob)
|
|
240
|
+
touched.push({ test: t.test, name: t.testName });
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
if (blockHash(textOf(t.test), t.testName) !== block)
|
|
244
|
+
touched.push({ test: t.test, name: t.testName });
|
|
245
|
+
}
|
|
246
|
+
return touched;
|
|
247
|
+
}
|
|
248
|
+
/** The text a recorded blob holds, untrimmed; null when the object store lacks it. */
|
|
249
|
+
function blobText(top, blob) {
|
|
250
|
+
if (blob === null)
|
|
251
|
+
return null;
|
|
252
|
+
const res = spawnSync("git", ["cat-file", "-p", blob], { cwd: top, encoding: "utf-8" });
|
|
253
|
+
return res.status === 0 ? res.stdout : null;
|
|
254
|
+
}
|
|
255
|
+
/** Leading whitespace width, a tab counting as one column. */
|
|
256
|
+
const indentOf = (line) => line.length - line.trimStart().length;
|
|
257
|
+
/** A line that only closes the block opened at its own indent. */
|
|
258
|
+
const CLOSER = /^(?:[)\]}]|end\b)/;
|
|
259
|
+
const escapeRegExp = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
260
|
+
/** A comment line: a name in it is prose, not the test. */
|
|
261
|
+
const COMMENT = /^(?:\/\/|\/\*|\*|#|--|<!--)/;
|
|
262
|
+
/** A line opening a suite (`describe`, `context`, `suite`, a class), not a test. */
|
|
263
|
+
const SUITE = /^(?:[\w$]+\.)?(?:describe|context|suite|class)\b/;
|
|
264
|
+
/**
|
|
265
|
+
* The first line `matches` that is no comment and opens no suite; else the first matching
|
|
266
|
+
* suite line, so a suite that alone carries the name still yields a block; else -1.
|
|
267
|
+
*/
|
|
268
|
+
function testStart(lines, matches) {
|
|
269
|
+
let suite = -1;
|
|
270
|
+
for (let i = 0; i < lines.length; i++) {
|
|
271
|
+
const line = lines[i].trimStart();
|
|
272
|
+
if (!matches(lines[i]) || COMMENT.test(line))
|
|
273
|
+
continue;
|
|
274
|
+
if (!SUITE.test(line))
|
|
275
|
+
return i;
|
|
276
|
+
if (suite < 0)
|
|
277
|
+
suite = i;
|
|
278
|
+
}
|
|
279
|
+
return suite;
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* The lines of the test named `name` in `text`, or null when no line names it. The test
|
|
283
|
+
* starts on the first line naming it as a quoted string (else as a whole word) that is no
|
|
284
|
+
* comment and opens no suite, or on the line before when the name opens its line and that
|
|
285
|
+
* line ends with `(`; it runs through every deeper-indented line after the start, and one
|
|
286
|
+
* closing line (`}`, `)`, `]`, `end`) at the start's indent. Framework-agnostic: it needs
|
|
287
|
+
* only a name and indentation.
|
|
288
|
+
*/
|
|
289
|
+
export function testBlock(text, name) {
|
|
290
|
+
if (text === null)
|
|
291
|
+
return null;
|
|
292
|
+
const span = testSpan(text, name);
|
|
293
|
+
return span === null
|
|
294
|
+
? null
|
|
295
|
+
: text
|
|
296
|
+
.split("\n")
|
|
297
|
+
.slice(span.start - 1, span.end)
|
|
298
|
+
.join("\n");
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* The 1-based first and last line of the test named `name` in `text` (the lines
|
|
302
|
+
* `testBlock` returns), or null when no line names it.
|
|
303
|
+
*/
|
|
304
|
+
export function testSpan(text, name) {
|
|
305
|
+
const lines = text.split("\n");
|
|
306
|
+
const quoted = [`"${name}"`, `'${name}'`, `\`${name}\``];
|
|
307
|
+
let at = testStart(lines, (l) => quoted.some((q) => l.includes(q)));
|
|
308
|
+
if (at < 0) {
|
|
309
|
+
const word = new RegExp(`(^|[^\\w])${escapeRegExp(name)}([^\\w]|$)`);
|
|
310
|
+
at = testStart(lines, (l) => word.test(l));
|
|
311
|
+
}
|
|
312
|
+
if (at < 0) {
|
|
313
|
+
// A suite path (`suite > test`, as a JUnit report names it): the last segment is the test.
|
|
314
|
+
const last = name.split(" > ").pop();
|
|
315
|
+
return last === name ? null : testSpan(text, last);
|
|
316
|
+
}
|
|
317
|
+
let start = at;
|
|
318
|
+
while (start > 0 &&
|
|
319
|
+
/^["'`]/.test(lines[start].trimStart()) &&
|
|
320
|
+
/\($/.test(lines[start - 1].trimEnd()))
|
|
321
|
+
start -= 1;
|
|
322
|
+
const base = indentOf(lines[start]);
|
|
323
|
+
let end = start;
|
|
324
|
+
for (let i = start + 1; i < lines.length; i++) {
|
|
325
|
+
const line = lines[i];
|
|
326
|
+
if (line.trim() === "")
|
|
327
|
+
continue;
|
|
328
|
+
if (indentOf(line) > base) {
|
|
329
|
+
end = i;
|
|
330
|
+
continue;
|
|
331
|
+
}
|
|
332
|
+
if (indentOf(line) === base && CLOSER.test(line.trimStart()))
|
|
333
|
+
end = i;
|
|
334
|
+
break;
|
|
335
|
+
}
|
|
336
|
+
return { start: start + 1, end: end + 1 };
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* The line diff of a test's own lines, `before` to `after`: unified lines (` `, `-`, `+`)
|
|
340
|
+
* over a longest common subsequence, and the counts of lines removed and added. A change
|
|
341
|
+
* that removes no line cannot have dropped an expectation; it can still have disabled one,
|
|
342
|
+
* which only a read of the added lines tells.
|
|
343
|
+
*/
|
|
344
|
+
export function lineDiff(before, after) {
|
|
345
|
+
const a = before.split("\n");
|
|
346
|
+
const b = after.split("\n");
|
|
347
|
+
const lcs = Array.from({ length: a.length + 1 }, () => new Array(b.length + 1).fill(0));
|
|
348
|
+
for (let i = a.length - 1; i >= 0; i--)
|
|
349
|
+
for (let j = b.length - 1; j >= 0; j--)
|
|
350
|
+
lcs[i][j] =
|
|
351
|
+
a[i] === b[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
|
|
352
|
+
const out = [];
|
|
353
|
+
let removed = 0;
|
|
354
|
+
let added = 0;
|
|
355
|
+
let i = 0;
|
|
356
|
+
let j = 0;
|
|
357
|
+
while (i < a.length || j < b.length) {
|
|
358
|
+
if (i < a.length && j < b.length && a[i] === b[j]) {
|
|
359
|
+
out.push(` ${a[i]}`);
|
|
360
|
+
i++;
|
|
361
|
+
j++;
|
|
362
|
+
}
|
|
363
|
+
else if (i < a.length && (j >= b.length || lcs[i + 1][j] >= lcs[i][j + 1])) {
|
|
364
|
+
out.push(`-${a[i]}`);
|
|
365
|
+
removed++;
|
|
366
|
+
i++;
|
|
367
|
+
}
|
|
368
|
+
else {
|
|
369
|
+
out.push(`+${b[j]}`);
|
|
370
|
+
added++;
|
|
371
|
+
j++;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
return { text: out.join("\n"), removed, added };
|
|
375
|
+
}
|
|
376
|
+
/** The sha-1 of a test's own lines (`testBlock`), or null when `text` holds no such test. */
|
|
377
|
+
export function blockHash(text, name) {
|
|
378
|
+
const block = testBlock(text, name);
|
|
379
|
+
return block === null ? null : createHash("sha1").update(block).digest("hex");
|
|
380
|
+
}
|
|
381
|
+
/** The commit the group started at: its `build-group-start` row's `parentSha`. */
|
|
382
|
+
function groupStartSha(file, workflow, group) {
|
|
383
|
+
if (!fs.existsSync(file))
|
|
384
|
+
return undefined;
|
|
385
|
+
const row = readStatusRows(file)
|
|
386
|
+
.filter((r) => r.type === "step-done" &&
|
|
387
|
+
r.workflow === workflow &&
|
|
388
|
+
r.step === "build-group-start" &&
|
|
389
|
+
r.group === group)
|
|
390
|
+
.at(-1);
|
|
391
|
+
return typeof row?.parentSha === "string" ? row.parentSha : undefined;
|
|
392
|
+
}
|
|
112
393
|
const NO_TEST_FILE_HINT = `declare "${BKIT_TEST_FILE_SCRIPT}" in package.json: it runs the one test file passed as its ` +
|
|
113
394
|
`argument and writes a JUnit report to $${REPORT_ENV} ` +
|
|
114
395
|
"(.bkit/workflows.md § Local gate scripts)";
|
|
@@ -132,18 +413,21 @@ function attr(attrs, name) {
|
|
|
132
413
|
const m = new RegExp(String.raw `\s${name}\s*=\s*(?:"([^"]*)"|'([^']*)')`).exec(attrs);
|
|
133
414
|
return m ? decodeXml(m[1] ?? m[2] ?? "") : undefined;
|
|
134
415
|
}
|
|
135
|
-
/**
|
|
416
|
+
/** CDATA text as escaped character data, so a tag inside it never ends an element early. */
|
|
417
|
+
const escapeCdata = (xml) => xml.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, (_, text) => text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">"));
|
|
418
|
+
/** Every `<testcase>` in a JUnit report: its name, outcome, first failure message and detail. */
|
|
136
419
|
export function parseJunit(xml) {
|
|
137
|
-
const body = xml
|
|
420
|
+
const body = escapeCdata(xml).replace(/<!--[\s\S]*?-->/g, "");
|
|
138
421
|
const cases = [];
|
|
139
422
|
const testcase = new RegExp(String.raw `<testcase${ATTRS}\s*(?:\/>|>([\s\S]*?)<\/testcase>)`, "g");
|
|
140
|
-
const outcome = new RegExp(String.raw `<(failure|error|skipped)${ATTRS}`);
|
|
423
|
+
const outcome = new RegExp(String.raw `<(failure|error|skipped)${ATTRS}\s*(?:\/>|>([\s\S]*?)<\/\1>)`);
|
|
141
424
|
for (const m of body.matchAll(testcase)) {
|
|
142
425
|
const child = outcome.exec(m[2] ?? "");
|
|
143
426
|
cases.push({
|
|
144
427
|
name: attr(m[1] ?? "", "name") ?? "",
|
|
145
428
|
state: child ? child[1] : "passed",
|
|
146
429
|
message: child ? (attr(child[2] ?? "", "message") ?? "") : "",
|
|
430
|
+
detail: child ? decodeXml((child[3] ?? "").replace(/<[^>]*>/g, "")).trim() : "",
|
|
147
431
|
});
|
|
148
432
|
}
|
|
149
433
|
return cases;
|
|
@@ -156,29 +440,40 @@ const caseNames = (cases) => cases.length === 0
|
|
|
156
440
|
.slice(0, 5)
|
|
157
441
|
.map((c) => `"${c.name}"`)
|
|
158
442
|
.join(", ")}${cases.length > 5 ? ", …" : ""}`;
|
|
159
|
-
/**
|
|
160
|
-
|
|
443
|
+
/** The stem a load error names: the test file's base name without its extension. */
|
|
444
|
+
const fileStem = (file) => path.posix.basename(file).replace(/\.[^.]*$/, "");
|
|
445
|
+
/**
|
|
446
|
+
* How `want` stood in a red run of `file`, from its JUnit report: `red` (the named test
|
|
447
|
+
* failed or errored), `red-not-loaded` (the file ran no test, or the only cases are failures
|
|
448
|
+
* named after the file itself: a load error, which an import of unwritten code is),
|
|
449
|
+
* `vacuous` (the named test passed, so it proves nothing before the task's code), `missing`
|
|
450
|
+
* (it did not run, or was skipped) or `no-report`.
|
|
451
|
+
*/
|
|
452
|
+
export function judgeRed(cases, want, file) {
|
|
161
453
|
if (cases === null)
|
|
162
454
|
return {
|
|
163
|
-
|
|
164
|
-
|
|
455
|
+
state: "no-report",
|
|
456
|
+
detail: `no JUnit report at $${REPORT_ENV} — ${BKIT_TEST_FILE_SCRIPT} must write one`,
|
|
165
457
|
};
|
|
166
458
|
const named = cases.filter((c) => isNamedCase(c.name, want));
|
|
167
|
-
if (named.length === 0)
|
|
168
|
-
|
|
459
|
+
if (named.length === 0) {
|
|
460
|
+
const stem = fileStem(file);
|
|
461
|
+
if (cases.length === 0 || cases.every((c) => c.state !== "passed" && c.name.includes(stem)))
|
|
462
|
+
return {
|
|
463
|
+
state: "red-not-loaded",
|
|
464
|
+
detail: `\`${file}\` did not load${cases[0]?.message ? `: ${cases[0].message}` : ""}`,
|
|
465
|
+
};
|
|
466
|
+
return { state: "missing", detail: `"${want}" did not run; ${caseNames(cases)}` };
|
|
467
|
+
}
|
|
169
468
|
if (named.some((c) => c.state === "passed"))
|
|
170
469
|
return {
|
|
171
|
-
|
|
172
|
-
|
|
470
|
+
state: "vacuous",
|
|
471
|
+
detail: `"${want}" passes at its red, so it proves nothing`,
|
|
173
472
|
};
|
|
174
473
|
if (named.some((c) => c.state === "skipped"))
|
|
175
|
-
return {
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
kind: "exit-mismatch",
|
|
179
|
-
reason: `"${want}" failed in the report but the script exited 0`,
|
|
180
|
-
};
|
|
181
|
-
return null;
|
|
474
|
+
return { state: "missing", detail: `"${want}" was skipped, not run` };
|
|
475
|
+
const failed = named.find((c) => c.state !== "passed");
|
|
476
|
+
return { state: "red", detail: `${failed.state}${failed.message ? `: ${failed.message}` : ""}` };
|
|
182
477
|
}
|
|
183
478
|
/** Why a run is not green for `want`, or null when every case of the named test passed. */
|
|
184
479
|
export function judgeGreen(cases, want) {
|
|
@@ -192,16 +487,37 @@ export function judgeGreen(cases, want) {
|
|
|
192
487
|
? null
|
|
193
488
|
: `"${want}" ${bad.state}${bad.message ? `: ${bad.message}` : ""}`;
|
|
194
489
|
}
|
|
195
|
-
/** The task field naming a task's tests, and the one recording an accepted
|
|
490
|
+
/** The task field naming a task's tests, and the one recording an accepted missing red. */
|
|
196
491
|
export const TEST_FIRST = "**TEST FIRST:**";
|
|
197
492
|
export const TEST_EXCEPTION = "**TEST EXCEPTION:**";
|
|
198
|
-
/** Why `tests green` refuses a plan with no group and task blocks. */
|
|
199
|
-
export const NO_MARKERS_REASON = "this plan has no task markers; run /bkit-build again to convert it";
|
|
200
493
|
/** The plan caps `package.json` `bkit.workflows.tasks` sets for the checkout `cwd` sits in. */
|
|
201
494
|
export const planCaps = (cwd) => readBkitConfig(toplevel(cwd)).tasks;
|
|
202
495
|
/** The caps as the plan-check report and help state them. */
|
|
203
496
|
export const capsText = (c) => `${c.maxPerGroup} tasks and ${c.maxTestsPerGroup} TEST FIRST tests per group; ${c.maxTestsPerTask} tests across ${c.maxTestFilesPerTask} test files per task`;
|
|
204
497
|
/** A task block's TEST FIRST read as one `TestFirst`; a TEST EXCEPTION with no TEST FIRST is its `n/a`. */
|
|
498
|
+
/**
|
|
499
|
+
* Planned tests of open tasks whose name a test in their file already carries: the tests
|
|
500
|
+
* step cannot write a new test under that name, so it either updates the existing one or
|
|
501
|
+
* finds it passing. Reported only — the plan names the update, or a new name.
|
|
502
|
+
*/
|
|
503
|
+
function existingTestNames(text, top) {
|
|
504
|
+
const out = [];
|
|
505
|
+
for (const t of parsePlanBlocks(text).tasks) {
|
|
506
|
+
if (t.done)
|
|
507
|
+
continue;
|
|
508
|
+
const s = testFirstOf(t);
|
|
509
|
+
if (s.kind !== "test")
|
|
510
|
+
continue;
|
|
511
|
+
for (const x of s.tests) {
|
|
512
|
+
const abs = path.join(top, x.test);
|
|
513
|
+
if (!fs.existsSync(abs))
|
|
514
|
+
continue;
|
|
515
|
+
if (testBlock(fs.readFileSync(abs, "utf-8"), x.name) !== null)
|
|
516
|
+
out.push({ line: t.testFirst?.line ?? t.line, task: t.task, test: x.test, name: x.name });
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
return out;
|
|
520
|
+
}
|
|
205
521
|
function testFirstOf(t) {
|
|
206
522
|
const { task, testFirst: tf, testException: exc } = t;
|
|
207
523
|
const plain = (tests) => tests.map(({ test, name }) => ({ test, name }));
|
|
@@ -220,7 +536,7 @@ function testFirstOf(t) {
|
|
|
220
536
|
* Each task block's TEST FIRST field (`plan-blocks.ts`): one test per sub-bullet (first
|
|
221
537
|
* backticked span = file, second = test name), `n/a — {reason}`, missing, or malformed
|
|
222
538
|
* (an entry with fewer than two spans, or no entry). A TEST EXCEPTION field names, in
|
|
223
|
-
* the same entry form, the tests whose
|
|
539
|
+
* the same entry form, the tests whose missing reds it accepts. `group` narrows to
|
|
224
540
|
* one group's tasks.
|
|
225
541
|
*/
|
|
226
542
|
export function readTestFirst(plan, group) {
|
|
@@ -246,7 +562,8 @@ const kb1 = (bytes) => Math.round((bytes / 1024) * 10) / 10;
|
|
|
246
562
|
* task-like text outside every block, a task without DONE WHEN or without TEST FIRST and
|
|
247
563
|
* TEST EXCEPTION, a malformed TEST FIRST, a DEPENDS ON naming a missing task or one in a
|
|
248
564
|
* later group, a group or task over a cap in `caps`, a CREATE file with no NEW Reuse
|
|
249
|
-
* Inventory row
|
|
565
|
+
* Inventory row, a `Derived Test Matrix` row naming no task / an undefined task / two
|
|
566
|
+
* tasks or a test-bearing task no row names, and a DELETE file with no Supersession row.
|
|
250
567
|
*/
|
|
251
568
|
export function checkPlanText(text, caps = CONFIG_DEFAULTS.tasks) {
|
|
252
569
|
const blocks = parsePlanBlocks(text);
|
|
@@ -320,6 +637,61 @@ export function checkPlanText(text, caps = CONFIG_DEFAULTS.tasks) {
|
|
|
320
637
|
message: `TASK ${t.task}: CREATE in \`${c.file}\` has no NEW row naming it in the Reuse Inventory's File column`,
|
|
321
638
|
});
|
|
322
639
|
}
|
|
640
|
+
// The matrix maps each row to one task and each test-bearing task to a row. A plan with no
|
|
641
|
+
// matrix section yields no problem: the refusal is on a row, and issue-form plans carry none.
|
|
642
|
+
const matrix = derivedTestMatrix(text);
|
|
643
|
+
if (matrix !== null) {
|
|
644
|
+
if (matrix.taskColumn === null)
|
|
645
|
+
problems.push({
|
|
646
|
+
line: matrix.line,
|
|
647
|
+
kind: "matrix",
|
|
648
|
+
message: "the Derived Test Matrix header has no `Task` column; add one naming the task each row's test belongs to",
|
|
649
|
+
});
|
|
650
|
+
else {
|
|
651
|
+
const mapped = new Set();
|
|
652
|
+
for (const row of matrix.rows) {
|
|
653
|
+
const named = row.tasks.map((n) => `TASK ${n}`);
|
|
654
|
+
if (row.tasks.length === 0)
|
|
655
|
+
problems.push({
|
|
656
|
+
line: row.number,
|
|
657
|
+
kind: "matrix",
|
|
658
|
+
message: `Derived Test Matrix row maps to no task: name one in its \`Task\` cell`,
|
|
659
|
+
});
|
|
660
|
+
else if (row.tasks.length > 1)
|
|
661
|
+
problems.push({
|
|
662
|
+
line: row.number,
|
|
663
|
+
kind: "matrix",
|
|
664
|
+
message: `Derived Test Matrix row maps to ${named.join(" and ")}; name exactly one task`,
|
|
665
|
+
});
|
|
666
|
+
else {
|
|
667
|
+
const n = row.tasks[0];
|
|
668
|
+
if (!byTask.has(n))
|
|
669
|
+
problems.push({
|
|
670
|
+
line: row.number,
|
|
671
|
+
kind: "matrix",
|
|
672
|
+
message: `Derived Test Matrix row maps to TASK ${n}, which the plan does not define`,
|
|
673
|
+
});
|
|
674
|
+
mapped.add(n);
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
for (const t of blocks.tasks)
|
|
678
|
+
if (specs.get(t.task).kind === "test" && !mapped.has(t.task))
|
|
679
|
+
problems.push({
|
|
680
|
+
line: t.line,
|
|
681
|
+
kind: "matrix",
|
|
682
|
+
message: `TASK ${t.task} names TEST FIRST tests but no Derived Test Matrix row maps to it`,
|
|
683
|
+
});
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
const superseded = new Set((supersessionInventory(text) ?? []).flatMap((r) => r.files));
|
|
687
|
+
for (const t of blocks.tasks)
|
|
688
|
+
for (const d of t.delete)
|
|
689
|
+
if (!superseded.has(path.posix.normalize(d.file)))
|
|
690
|
+
problems.push({
|
|
691
|
+
line: d.line,
|
|
692
|
+
kind: "supersession",
|
|
693
|
+
message: `TASK ${t.task}: DELETE in \`${d.file}\` has no row naming it in the Supersession Inventory`,
|
|
694
|
+
});
|
|
323
695
|
const groups = blocks.groups.map((g) => {
|
|
324
696
|
const tests = sizes.filter((s) => s.group === g.group).reduce((n, s) => n + s.tests, 0);
|
|
325
697
|
const bytes = Buffer.byteLength(lines.slice(g.line - 1, g.end).join("\n"));
|
|
@@ -475,28 +847,6 @@ export function planExistence(text, root) {
|
|
|
475
847
|
}
|
|
476
848
|
return out.sort((a, b) => a.line - b.line);
|
|
477
849
|
}
|
|
478
|
-
/** True when `after` keeps every non-blank line of `before`, in order, ignoring whitespace: lines were only added. */
|
|
479
|
-
export function onlyAddsLines(before, after) {
|
|
480
|
-
const norm = (text) => text
|
|
481
|
-
.split("\n")
|
|
482
|
-
.map((l) => l.replace(/\s+/g, ""))
|
|
483
|
-
.filter(Boolean);
|
|
484
|
-
const want = norm(before);
|
|
485
|
-
let kept = 0;
|
|
486
|
-
for (const line of norm(after))
|
|
487
|
-
if (kept < want.length && line === want[kept])
|
|
488
|
-
kept++;
|
|
489
|
-
return kept === want.length;
|
|
490
|
-
}
|
|
491
|
-
/** `test` as a repo-relative path inside `top`; usage error when absolute, outside, or absent. */
|
|
492
|
-
function repoTestPath(top, test) {
|
|
493
|
-
const rel = path.relative(top, path.resolve(top, test));
|
|
494
|
-
if (path.isAbsolute(test) || rel.startsWith("..") || rel === "")
|
|
495
|
-
throw new CommandError(EXIT.USAGE, `--test must be a path inside the repo, got "${test}"`);
|
|
496
|
-
if (!fs.statSync(path.join(top, rel), { throwIfNoEntry: false })?.isFile())
|
|
497
|
-
throw new CommandError(EXIT.USAGE, `--test ${rel} is not a file`);
|
|
498
|
-
return rel.split(path.sep).join("/");
|
|
499
|
-
}
|
|
500
850
|
/** Run one test file through `bkit:test:file`; the report is read and removed. */
|
|
501
851
|
function runTestFile(top, test, log) {
|
|
502
852
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bkit-test-report-"));
|
|
@@ -509,54 +859,56 @@ function runTestFile(top, test, log) {
|
|
|
509
859
|
return { exitCode, cases };
|
|
510
860
|
}
|
|
511
861
|
const couldNotRunMessage = (exitCode) => `${gateScope(BKIT_TEST_FILE_SCRIPT)} could not run (exit ${exitCode}: ${exitCode === 126 ? "not executable" : "command not found"}) — no test executed`;
|
|
512
|
-
/** The copy of the test file a red keeps beside its log. */
|
|
513
|
-
export const snapshotPath = (log) => `${log.replace(/\.md$/, "")}.snapshot`;
|
|
514
862
|
const toplevel = (cwd) => git(cwd, ["rev-parse", "--show-toplevel"]) ?? cwd;
|
|
515
863
|
/** A fence longer than any backtick run in `text`. */
|
|
516
864
|
const fenceFor = (text) => "`".repeat(Math.max(3, ...[...text.matchAll(/`+/g)].map((m) => m[0].length + 1)));
|
|
517
|
-
/**
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
865
|
+
/**
|
|
866
|
+
* Every gap the audit emits, and the verifier's `Fixable` for it: `null` = the
|
|
867
|
+
* answer needs a diff read, which only the verifier can make. A test with no proven red
|
|
868
|
+
* is no gap: the audit lists it as unproven, and the owner answers every unproven test
|
|
869
|
+
* once, at the end of the build.
|
|
870
|
+
*/
|
|
871
|
+
const GAP_FIXABLE = {
|
|
872
|
+
"not-green": true,
|
|
873
|
+
"delete-still-referenced": null,
|
|
874
|
+
"no-test-first": false,
|
|
875
|
+
"test-first-malformed": false,
|
|
876
|
+
"test-first-unread": false,
|
|
877
|
+
"exception-refused": false,
|
|
878
|
+
};
|
|
879
|
+
/** One gap, with the `fixable` its kind settles and the ready-to-append log line. */
|
|
880
|
+
function gapOf(task, kind, detail, test) {
|
|
881
|
+
const fixable = GAP_FIXABLE[kind];
|
|
882
|
+
const subject = test === undefined ? "" : `\`${test.file}\` \`${test.name}\` — `;
|
|
883
|
+
return {
|
|
884
|
+
task,
|
|
885
|
+
kind,
|
|
886
|
+
detail,
|
|
887
|
+
fixable,
|
|
888
|
+
logLine: `${task === null ? "Plan" : `TASK ${task}`}: ${subject}${kind} — Fixable: ${fixable}`,
|
|
889
|
+
};
|
|
890
|
+
}
|
|
891
|
+
/** The report's verdict from its gaps: a decision one gap still needs (`null`) outranks one it refuses. */
|
|
892
|
+
const verdictOf = (gaps) => gaps.length === 0
|
|
893
|
+
? "PASS"
|
|
894
|
+
: gaps.some((g) => g.fixable === null)
|
|
895
|
+
? null
|
|
896
|
+
: gaps.some((g) => g.fixable === false)
|
|
897
|
+
? "FAIL"
|
|
898
|
+
: "CONCERNS";
|
|
525
899
|
/** The gap for a task whose only test field is a TEST EXCEPTION that is not honoured. */
|
|
526
900
|
export const EXCEPTION_REFUSED = "exception-refused";
|
|
527
901
|
/** The status row step plan-execute's finish writes; its commit carries the code of each task its log records DONE. */
|
|
528
|
-
export const
|
|
902
|
+
export const PLAN_TASKS_STEP = "build-plan-tasks";
|
|
529
903
|
const repoRel = (top, cwd, p) => path.relative(top, path.resolve(cwd, p)).split(path.sep).join("/");
|
|
530
904
|
/**
|
|
531
|
-
*
|
|
532
|
-
*
|
|
533
|
-
*
|
|
905
|
+
* The task numbers a plan-execute log records `DONE — TASK N` for, in log order. `verbs`
|
|
906
|
+
* widens the reader to the log's other closing lines (`BLOCKED`), so one parser serves
|
|
907
|
+
* every caller; the default reads `DONE` only, which is what "the task is done" means.
|
|
534
908
|
*/
|
|
535
|
-
function
|
|
536
|
-
const
|
|
537
|
-
|
|
538
|
-
for (const r of readStatusRows(file)) {
|
|
539
|
-
if (r.type !== "step-done" || r.workflow !== workflow || r.step !== PLAN_EXECUTE_STEP)
|
|
540
|
-
continue;
|
|
541
|
-
const log = Array.isArray(r.artifacts) ? r.artifacts.map(String)[0] : undefined;
|
|
542
|
-
if (typeof r.parentSha !== "string" || log === undefined)
|
|
543
|
-
continue;
|
|
544
|
-
const range = `${r.parentSha}..HEAD`;
|
|
545
|
-
const commit = git(top, ["rev-list", "--reverse", "--ancestry-path", range, "--", statusRel])
|
|
546
|
-
?.split("\n")
|
|
547
|
-
.find(Boolean);
|
|
548
|
-
const text = commit === undefined ? null : git(top, ["show", `${commit}:${repoRel(top, cwd, log)}`]);
|
|
549
|
-
if (commit === undefined || text === null)
|
|
550
|
-
continue;
|
|
551
|
-
for (const n of doneTasks(text))
|
|
552
|
-
if (!commits.has(n))
|
|
553
|
-
commits.set(n, commit);
|
|
554
|
-
}
|
|
555
|
-
return commits;
|
|
556
|
-
}
|
|
557
|
-
/** The task numbers a plan-execute log records `DONE — TASK N` for, in log order. */
|
|
558
|
-
export function doneTasks(log) {
|
|
559
|
-
return [...log.matchAll(/^DONE\s*[—–-]+\s*TASK\s+(\d+)\b/gim)].map((m) => Number(m[1]));
|
|
909
|
+
export function doneTasks(log, verbs = ["DONE"]) {
|
|
910
|
+
const closing = verbs.map((v) => v.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|");
|
|
911
|
+
return [...log.matchAll(new RegExp(`^(?:${closing})\\s*[—–-]+\\s*TASK\\s+(\\d+)\\b`, "gim"))].map((m) => Number(m[1]));
|
|
560
912
|
}
|
|
561
913
|
/**
|
|
562
914
|
* Why a task's `TEST EXCEPTION` lines do not count, or null when they do: every
|
|
@@ -665,7 +1017,7 @@ function deleteGaps(top, planRel, tasks, planTasks) {
|
|
|
665
1017
|
path.posix.basename(path.posix.normalize(c.file)) === path.posix.basename(file))
|
|
666
1018
|
.map((c) => path.posix.normalize(c.file))
|
|
667
1019
|
: [];
|
|
668
|
-
const gap = (detail) => gaps.push(
|
|
1020
|
+
const gap = (detail) => gaps.push(gapOf(t.task, DELETE_STILL_REFERENCED, detail));
|
|
669
1021
|
const abs = path.join(top, file);
|
|
670
1022
|
const exists = fs.statSync(abs, { throwIfNoEntry: false })?.isFile() === true;
|
|
671
1023
|
let what;
|
|
@@ -698,21 +1050,87 @@ function deleteGaps(top, planRel, tasks, planTasks) {
|
|
|
698
1050
|
}
|
|
699
1051
|
return gaps;
|
|
700
1052
|
}
|
|
701
|
-
/**
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
1053
|
+
/** The `Updated` entry for a planned test: its own lines at `base` against the file now. */
|
|
1054
|
+
function updatedEntry(top, task, t, base) {
|
|
1055
|
+
const entry = { task, test: t.test, name: t.name, base, additive: null, diff: null };
|
|
1056
|
+
if (base === null)
|
|
1057
|
+
return entry;
|
|
1058
|
+
const shown = runGit(top, ["show", `${base}:${t.test}`]);
|
|
1059
|
+
const abs = path.join(top, t.test);
|
|
1060
|
+
if (!shown.ok || !fs.existsSync(abs))
|
|
1061
|
+
return entry;
|
|
1062
|
+
const was = testBlock(shown.stdout, t.name);
|
|
1063
|
+
const now = testBlock(fs.readFileSync(abs, "utf-8"), t.name);
|
|
1064
|
+
if (was === null || now === null)
|
|
1065
|
+
return entry;
|
|
1066
|
+
const d = lineDiff(was, now);
|
|
1067
|
+
return { ...entry, additive: d.removed === 0, diff: d.text };
|
|
1068
|
+
}
|
|
1069
|
+
/** Every PASSES line of a tests step's log: the tests the writer saw passing and vouches for. */
|
|
1070
|
+
function passesLines(log) {
|
|
1071
|
+
return [
|
|
1072
|
+
...log.matchAll(/^PASSES\s*[—–-]+\s*TASK\s+(\d+):\s*`([^`]+)`\s+`([^`]+)`\s*:\s*(.+)$/gm),
|
|
1073
|
+
].map((m) => ({
|
|
1074
|
+
task: Number(m[1]),
|
|
1075
|
+
test: m[2].trim(),
|
|
1076
|
+
name: m[3].trim(),
|
|
1077
|
+
reason: m[4].trim(),
|
|
1078
|
+
}));
|
|
1079
|
+
}
|
|
1080
|
+
/** Why a plan cannot be read for its tests (no file, no task markers, a fence never closed), or undefined. */
|
|
1081
|
+
const planRefusal = (plan, parsed) => plan === undefined
|
|
1082
|
+
? `no plan file; ${NO_MARKERS_REASON}`
|
|
1083
|
+
: parsed.tasks.length === 0
|
|
1084
|
+
? parsed.fenceOpen === undefined
|
|
1085
|
+
? NO_MARKERS_REASON
|
|
1086
|
+
: fenceNeverClosed(parsed.fenceOpen)
|
|
1087
|
+
: undefined;
|
|
1088
|
+
/** A table cell: pipes escaped, line breaks folded. */
|
|
1089
|
+
const cell = (s) => s.replace(/\s+/g, " ").replace(/\|/g, "\\|").trim();
|
|
1090
|
+
function writeRedReport(log, o) {
|
|
1091
|
+
const red = o.tests.filter((t) => t.state === "red" || t.state === "red-not-loaded").length;
|
|
1092
|
+
const accepted = o.tests.filter((t) => t.state === "passed-accepted").length;
|
|
1093
|
+
const updated = o.tests.filter((t) => t.state === "updated").length;
|
|
1094
|
+
const out = [
|
|
1095
|
+
`Enforced: ${o.enforced}`,
|
|
1096
|
+
`Red: ${red}`,
|
|
1097
|
+
`Accepted: ${accepted}`,
|
|
1098
|
+
`Updated: ${updated}`,
|
|
1099
|
+
`Gaps: ${o.gaps.length}`,
|
|
1100
|
+
"",
|
|
1101
|
+
"<!-- verdict:end -->",
|
|
1102
|
+
"",
|
|
1103
|
+
`# Tests Red — group ${o.group}, attempt ${o.attempt}`,
|
|
1104
|
+
"",
|
|
1105
|
+
];
|
|
1106
|
+
if (o.plan)
|
|
1107
|
+
out.push(`Plan: \`${o.plan}\``, "");
|
|
1108
|
+
if (!o.enforced)
|
|
1109
|
+
out.push(`Not enforced: ${o.reason}.`, "");
|
|
1110
|
+
if (o.tests.length > 0) {
|
|
1111
|
+
out.push("| Task | Test | State | Detail |", "| --- | --- | --- | --- |");
|
|
1112
|
+
for (const t of o.tests)
|
|
1113
|
+
out.push(`| ${t.task} | \`${t.test}\` "${cell(t.testName)}" | ${t.state} | ${cell(o.details.get(t) ?? "")} |`);
|
|
1114
|
+
out.push("");
|
|
1115
|
+
}
|
|
1116
|
+
out.push("## Gaps", "");
|
|
1117
|
+
if (o.gaps.length === 0)
|
|
1118
|
+
out.push("None.");
|
|
1119
|
+
for (const g of o.gaps)
|
|
1120
|
+
out.push(`- TASK ${g.task} \`${g.test}\` "${g.name}" \`${g.kind}\` — ${g.detail}`);
|
|
1121
|
+
for (const t of o.outputs) {
|
|
1122
|
+
const fence = fenceFor(t.text);
|
|
1123
|
+
out.push("", `## ${t.title}`, "", fence, t.text.replace(/\n$/, ""), fence);
|
|
1124
|
+
}
|
|
1125
|
+
fs.mkdirSync(path.dirname(log), { recursive: true });
|
|
1126
|
+
fs.writeFileSync(log, `${out.join("\n")}\n`);
|
|
710
1127
|
}
|
|
711
1128
|
function writeAuditReport(log, o) {
|
|
712
1129
|
const out = [
|
|
713
1130
|
`Enforced: ${o.enforced}`,
|
|
1131
|
+
`Verdict: ${o.verdict}`,
|
|
714
1132
|
`Gaps: ${o.gaps.length}`,
|
|
715
|
-
`
|
|
1133
|
+
`Unproven: ${o.unproven.length}`,
|
|
716
1134
|
"",
|
|
717
1135
|
"<!-- verdict:end -->",
|
|
718
1136
|
"",
|
|
@@ -723,30 +1141,45 @@ function writeAuditReport(log, o) {
|
|
|
723
1141
|
if (!o.enforced)
|
|
724
1142
|
out.push(`Not enforced: ${o.reason}.`, "");
|
|
725
1143
|
if (o.rows.length > 0) {
|
|
726
|
-
out.push("| Task | Test | Red |
|
|
1144
|
+
out.push("| Task | Test | Red | Green |", "| --- | --- | --- | --- |");
|
|
727
1145
|
for (const r of o.rows)
|
|
728
|
-
out.push(`| ${r.task} | ${r.test} | ${r.red} | ${r.
|
|
1146
|
+
out.push(`| ${r.task} | ${r.test} | ${r.red} | ${r.green} |`);
|
|
729
1147
|
out.push("");
|
|
730
1148
|
}
|
|
731
1149
|
out.push("## Gaps", "");
|
|
732
1150
|
if (o.gaps.length === 0)
|
|
733
1151
|
out.push("None.");
|
|
734
1152
|
for (const g of o.gaps)
|
|
735
|
-
out.push(`- ${g.task === null ? "Plan" : `Task ${g.task}`} \`${g.kind}\` — ${g.detail}`);
|
|
1153
|
+
out.push(`- ${g.task === null ? "Plan" : `Task ${g.task}`} \`${g.kind}\` — ${g.detail} — Fixable: ${g.fixable} — log: ${g.logLine}`);
|
|
736
1154
|
if (o.exceptions.length > 0)
|
|
737
1155
|
out.push("", "## Test exceptions", "");
|
|
738
1156
|
for (const e of o.exceptions)
|
|
739
1157
|
out.push(e.refusal === null
|
|
740
1158
|
? `- Task ${e.task} — honoured; cleared ${e.cleared.length > 0 ? e.cleared.join(", ") : "nothing"}`
|
|
741
1159
|
: `- Task ${e.task} — not honoured: ${e.refusal}`);
|
|
742
|
-
if (o.
|
|
743
|
-
out.push("", "##
|
|
744
|
-
for (const
|
|
745
|
-
out.push(`- Task ${task}
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
1160
|
+
if (o.landed.length > 0)
|
|
1161
|
+
out.push("", "## Already landed", "");
|
|
1162
|
+
for (const l of o.landed)
|
|
1163
|
+
out.push(`- Task ${l.task} \`${l.test}\` "${l.name}" — passed at the group's red, and the tests step accepted it: ${l.reason}; no red is due`);
|
|
1164
|
+
if (o.updated.length > 0)
|
|
1165
|
+
out.push("", "## Updated tests", "");
|
|
1166
|
+
for (const u of o.updated) {
|
|
1167
|
+
const since = u.base === null ? "" : ` since \`${u.base.slice(0, 9)}\``;
|
|
1168
|
+
const judged = u.additive === null
|
|
1169
|
+
? "its lines at the base commit could not be read, so read the group diff"
|
|
1170
|
+
: u.additive
|
|
1171
|
+
? "additive: no line of the test was removed, so no expectation was dropped — read the added lines for one that disables an earlier expectation"
|
|
1172
|
+
: "not additive: a line of the test was removed or changed — read the diff for an expectation removed or loosened";
|
|
1173
|
+
out.push(`- Task ${u.task} \`${u.test}\` "${u.name}" — an existing test the tests step changed${since}; the update may add expectations, never remove or loosen one. ${judged}`);
|
|
1174
|
+
if (u.diff !== null) {
|
|
1175
|
+
const fence = fenceFor(u.diff);
|
|
1176
|
+
out.push("", ` ${fence}diff`, ...u.diff.split("\n").map((l) => ` ${l}`), ` ${fence}`, "");
|
|
1177
|
+
}
|
|
749
1178
|
}
|
|
1179
|
+
if (o.unproven.length > 0)
|
|
1180
|
+
out.push("", "## Unproven", "");
|
|
1181
|
+
for (const u of o.unproven)
|
|
1182
|
+
out.push(`- Task ${u.task} \`${u.test}\` "${u.name}" — ${u.detail}; the owner answers it at the end of the build`);
|
|
750
1183
|
for (const t of o.outputs) {
|
|
751
1184
|
const fence = fenceFor(t.tail);
|
|
752
1185
|
out.push("", `## Runner output — \`${t.test}\``, "", fence, t.tail, fence);
|
|
@@ -758,8 +1191,10 @@ export const testsCommand = {
|
|
|
758
1191
|
noun: "tests",
|
|
759
1192
|
summary: "Run the deepest declared local gate, or one test file to record a red and audit a green.",
|
|
760
1193
|
verbs: {
|
|
1194
|
+
changed: testsChangedVerb,
|
|
761
1195
|
run: {
|
|
762
|
-
summary: "Run the first declared --gate; exit 0 PASS, 1 FAIL, 4 could not run.",
|
|
1196
|
+
summary: "Run the first declared --gate, and with --file append a test-run status row for the run; exit 0 PASS, 1 FAIL, 4 could not run, 81 tests lock held.",
|
|
1197
|
+
use: "by-hand",
|
|
763
1198
|
flags: [
|
|
764
1199
|
{
|
|
765
1200
|
name: "gate",
|
|
@@ -779,16 +1214,46 @@ export const testsCommand = {
|
|
|
779
1214
|
type: "string",
|
|
780
1215
|
help: `lines of the log echoed to stderr afterwards (default ${DEFAULT_TAIL}; 0 for none)`,
|
|
781
1216
|
},
|
|
1217
|
+
{
|
|
1218
|
+
name: "lock-wait-s",
|
|
1219
|
+
type: "string",
|
|
1220
|
+
help: "whole-number lock wait >= 0, passed as LOCAL_STACK_LOCK_WAIT_S (default BKIT_LOCK_WAIT_S, else 60)",
|
|
1221
|
+
},
|
|
782
1222
|
{
|
|
783
1223
|
name: "artifact",
|
|
784
1224
|
type: "string",
|
|
785
1225
|
help: "copy the log here, trim the copy (debug-evidence) and record the clamp; needs --fid and --step",
|
|
786
1226
|
},
|
|
787
1227
|
{ name: "fid", type: "string", help: "--artifact: feature id, for the clamp log" },
|
|
788
|
-
{
|
|
789
|
-
|
|
1228
|
+
{
|
|
1229
|
+
name: "step",
|
|
1230
|
+
type: "string",
|
|
1231
|
+
help: "step name: the test-run row's (--file) and the clamp log's (--artifact)",
|
|
1232
|
+
},
|
|
1233
|
+
{
|
|
1234
|
+
name: "attempt",
|
|
1235
|
+
type: "string",
|
|
1236
|
+
help: "attempt: on the test-run row (--file) and in the clamp log's name (--artifact)",
|
|
1237
|
+
},
|
|
1238
|
+
{
|
|
1239
|
+
name: "file",
|
|
1240
|
+
type: "string",
|
|
1241
|
+
help: "status.jsonl: append one test-run row for this run, pass or fail; needs --workflow and --step",
|
|
1242
|
+
},
|
|
1243
|
+
{ name: "workflow", type: "string", help: "--file: workflow name, for the test-run row" },
|
|
1244
|
+
{ name: "ci-run", type: "string", help: "selected GitHub evidence run ID" },
|
|
1245
|
+
{ name: "ci-url", type: "string", help: "selected GitHub evidence attempt URL" },
|
|
790
1246
|
],
|
|
791
1247
|
output: {
|
|
1248
|
+
source: "ci for pinned red evidence; local for every executed gate",
|
|
1249
|
+
runAttempt: "selected CI attempt for a CI evidence round",
|
|
1250
|
+
ciGeneration: "active selection generation for a CI evidence round",
|
|
1251
|
+
ciRun: "selected CI run ID for a CI evidence round",
|
|
1252
|
+
ciUrl: "selected CI attempt URL for a CI evidence round",
|
|
1253
|
+
repo: "selected repository for a CI evidence round",
|
|
1254
|
+
workflowId: "selected workflow ID for a CI evidence round",
|
|
1255
|
+
workflowPath: "selected workflow path for a CI evidence round",
|
|
1256
|
+
headSha: "selected commit for a CI evidence round",
|
|
792
1257
|
verdict: "PASS | FAIL (null when the gate could not run)",
|
|
793
1258
|
exitCode: "the gate's exit code",
|
|
794
1259
|
gate: "the script that ran",
|
|
@@ -799,14 +1264,20 @@ export const testsCommand = {
|
|
|
799
1264
|
logBytes: "size of the saved output",
|
|
800
1265
|
artifact: "--artifact as given, else null",
|
|
801
1266
|
clamp: "the artifact's clamp record (raw = the log, source = scope), else null",
|
|
1267
|
+
row: `the ${TEST_RUN_ROW} row appended to --file ({type, workflow, at, step, attempt?, run, verdict, exitCode, failures, scope, log, artifact?}; run = this step's test-run rows in the workflow's active run, this one included), else null; a tests-lock wait writes no row`,
|
|
1268
|
+
lockHolder: "tests-lock holder when the wait gave up, else null",
|
|
802
1269
|
},
|
|
803
|
-
textOmit: ["clamp"],
|
|
1270
|
+
textOmit: ["clamp", "row"],
|
|
804
1271
|
exitCodes: {
|
|
805
1272
|
[EXIT_NO_GATE]: "no candidate gate is declared in package.json",
|
|
806
1273
|
[EXIT_COULD_NOT_RUN]: "the gate could not run (exit 126 / 127): not a test failure",
|
|
807
1274
|
[EXIT_ARTIFACT_STEP]: "--artifact could not be written, trimmed or recorded",
|
|
808
1275
|
},
|
|
809
1276
|
run: (ctx) => {
|
|
1277
|
+
const waitSource = ctx.flags.lockWaitS ?? process.env.BKIT_LOCK_WAIT_S;
|
|
1278
|
+
const waitS = wholeNumber(waitSource, ctx.flags.lockWaitS === undefined && process.env.BKIT_LOCK_WAIT_S !== undefined
|
|
1279
|
+
? "lock-wait-s (BKIT_LOCK_WAIT_S)"
|
|
1280
|
+
: "lock-wait-s", 60);
|
|
810
1281
|
const candidates = ctx.flags.gate;
|
|
811
1282
|
const logArg = String(ctx.flags.log);
|
|
812
1283
|
if (logArg === "")
|
|
@@ -817,6 +1288,98 @@ export const testsCommand = {
|
|
|
817
1288
|
const attempt = positiveInt(ctx.flags.attempt, "attempt");
|
|
818
1289
|
if (artifact !== undefined && (!fid || !step))
|
|
819
1290
|
throw new CommandError(EXIT.USAGE, "--artifact needs --fid and --step");
|
|
1291
|
+
const statusArg = str(ctx.flags.file);
|
|
1292
|
+
const workflow = str(ctx.flags.workflow);
|
|
1293
|
+
if (statusArg !== undefined && (!workflow || !step))
|
|
1294
|
+
throw new CommandError(EXIT.USAGE, "--file needs --workflow and --step");
|
|
1295
|
+
const status = statusArg === undefined ? undefined : path.resolve(ctx.cwd, statusArg);
|
|
1296
|
+
if (status !== undefined && !fs.existsSync(status))
|
|
1297
|
+
throw new CommandError(EXIT.USAGE, `--file ${statusArg} does not exist (a test-run row never starts a log)`);
|
|
1298
|
+
if (ctx.flags.ciRun !== undefined || ctx.flags.ciUrl !== undefined) {
|
|
1299
|
+
const active = status === undefined ? undefined : readRuns(status).at(-1);
|
|
1300
|
+
const selection = active?.ciSelection;
|
|
1301
|
+
if (!status ||
|
|
1302
|
+
workflow !== "debug" ||
|
|
1303
|
+
active?.workflow !== "debug" ||
|
|
1304
|
+
active.status !== "IN_PROGRESS" ||
|
|
1305
|
+
!validateCiSelection(selection) ||
|
|
1306
|
+
selection.mode !== "ci" ||
|
|
1307
|
+
ctx.flags.ciRun !== selection.runId ||
|
|
1308
|
+
ctx.flags.ciUrl !== selection.runUrl ||
|
|
1309
|
+
!Number.isSafeInteger(active.ciGeneration) ||
|
|
1310
|
+
Number(active.ciGeneration) < 1 ||
|
|
1311
|
+
!artifact)
|
|
1312
|
+
throw new CommandError(EXIT.USAGE, "CI round needs an active matching debug selection and artifact");
|
|
1313
|
+
const fixed = active.steps.some((s) => s.status === "DONE" &&
|
|
1314
|
+
s.ciGeneration === active.ciGeneration &&
|
|
1315
|
+
["debug-failure-remediate", "debug-failure-soak"].includes(s.step));
|
|
1316
|
+
if (!fixed) {
|
|
1317
|
+
const scope = `ci:${selection.runId}`;
|
|
1318
|
+
const text = [
|
|
1319
|
+
"Verdict: FAIL",
|
|
1320
|
+
"Source: ci",
|
|
1321
|
+
`Scope: ${scope}`,
|
|
1322
|
+
`Repository: ${selection.repo}`,
|
|
1323
|
+
`Workflow: ${selection.workflowPath} (${selection.workflowId})`,
|
|
1324
|
+
`Run: ${selection.runId}`,
|
|
1325
|
+
`Run attempt: ${selection.runAttempt}`,
|
|
1326
|
+
`CI generation: ${active.ciGeneration}`,
|
|
1327
|
+
`Commit: ${selection.headSha}`,
|
|
1328
|
+
`URL: ${selection.runUrl}`,
|
|
1329
|
+
"",
|
|
1330
|
+
"<!-- verdict:end -->",
|
|
1331
|
+
"",
|
|
1332
|
+
"# CI evidence",
|
|
1333
|
+
"",
|
|
1334
|
+
"The local gate was not run.",
|
|
1335
|
+
"",
|
|
1336
|
+
].join("\n");
|
|
1337
|
+
for (const dest of new Set([artifact, logArg])) {
|
|
1338
|
+
const out = path.resolve(ctx.cwd, dest);
|
|
1339
|
+
fs.mkdirSync(path.dirname(out), { recursive: true });
|
|
1340
|
+
fs.writeFileSync(out, text);
|
|
1341
|
+
}
|
|
1342
|
+
const evidence = {
|
|
1343
|
+
gate: null,
|
|
1344
|
+
ciRun: selection.runId,
|
|
1345
|
+
ciUrl: selection.runUrl,
|
|
1346
|
+
runAttempt: selection.runAttempt,
|
|
1347
|
+
ciGeneration: active.ciGeneration,
|
|
1348
|
+
repo: selection.repo,
|
|
1349
|
+
workflowId: selection.workflowId,
|
|
1350
|
+
workflowPath: selection.workflowPath,
|
|
1351
|
+
headSha: selection.headSha,
|
|
1352
|
+
};
|
|
1353
|
+
const row = appendTestRun(status, {
|
|
1354
|
+
workflow,
|
|
1355
|
+
step: step,
|
|
1356
|
+
attempt,
|
|
1357
|
+
source: "ci",
|
|
1358
|
+
verdict: "FAIL",
|
|
1359
|
+
exitCode: null,
|
|
1360
|
+
failures: null,
|
|
1361
|
+
scope,
|
|
1362
|
+
log: logArg,
|
|
1363
|
+
artifact,
|
|
1364
|
+
evidence,
|
|
1365
|
+
});
|
|
1366
|
+
ctx.emit({
|
|
1367
|
+
source: "ci",
|
|
1368
|
+
verdict: "FAIL",
|
|
1369
|
+
scope,
|
|
1370
|
+
exitCode: null,
|
|
1371
|
+
failures: null,
|
|
1372
|
+
...evidence,
|
|
1373
|
+
artifact,
|
|
1374
|
+
log: logArg,
|
|
1375
|
+
logBytes: Buffer.byteLength(text),
|
|
1376
|
+
reports: null,
|
|
1377
|
+
clamp: null,
|
|
1378
|
+
row,
|
|
1379
|
+
});
|
|
1380
|
+
return EXIT.NO;
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
820
1383
|
const tailRaw = str(ctx.flags.tail) ?? String(DEFAULT_TAIL);
|
|
821
1384
|
if (!/^[0-9]+$/.test(tailRaw)) {
|
|
822
1385
|
throw new CommandError(EXIT.USAGE, `--tail must be a non-negative integer, got "${tailRaw}"`);
|
|
@@ -837,20 +1400,34 @@ export const testsCommand = {
|
|
|
837
1400
|
const reports = freshReportDir(log);
|
|
838
1401
|
ctx.log(`running ${scope} — full output in ${logArg}`);
|
|
839
1402
|
const fetchedAt = new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
840
|
-
const
|
|
841
|
-
|
|
1403
|
+
const who = ["bkit tests run", gate, fid, step].filter(Boolean).join(" ");
|
|
1404
|
+
const exitCode = runToLog(scriptCommand(gate), top, log, {
|
|
1405
|
+
[REPORT_DIR_ENV]: reports,
|
|
1406
|
+
LOCAL_STACK_LOCK_WAIT_S: String(waitS),
|
|
1407
|
+
LOCAL_STACK_LOCK_WHO: who,
|
|
1408
|
+
});
|
|
1409
|
+
const logText = fs.readFileSync(log, "utf-8");
|
|
1410
|
+
const lockExit = classifyLockExit({ exitCode, log: logText, who });
|
|
1411
|
+
const lockWait = lockExit === "lock-wait";
|
|
1412
|
+
const lockHolder = lockWait ? testsLockHolder(logText, who) : null;
|
|
1413
|
+
const lines = lockExit === "gate"
|
|
1414
|
+
? Number(tailRaw) > 0
|
|
1415
|
+
? tailLines(log, Number(tailRaw))
|
|
1416
|
+
: []
|
|
1417
|
+
: logText.split(/\r?\n/).filter((line) => line.startsWith("[machine-lock]"));
|
|
1418
|
+
for (const line of lines)
|
|
842
1419
|
ctx.log(line);
|
|
843
|
-
const couldNotRun = COULD_NOT_RUN_EXIT_CODES.has(exitCode);
|
|
844
|
-
const verdict = couldNotRun ? null : exitCode === 0 ? "PASS" : "FAIL";
|
|
845
|
-
const counted = countReportFailures(reports);
|
|
846
|
-
if (counted === null)
|
|
1420
|
+
const couldNotRun = lockExit === "could-not-run" || COULD_NOT_RUN_EXIT_CODES.has(exitCode);
|
|
1421
|
+
const verdict = couldNotRun || lockWait ? null : exitCode === 0 ? "PASS" : "FAIL";
|
|
1422
|
+
const counted = lockExit === "gate" ? countReportFailures(reports) : null;
|
|
1423
|
+
if (counted === null && lockExit === "gate")
|
|
847
1424
|
ctx.log(NO_REPORT_HINT);
|
|
848
1425
|
else if (verdict === "FAIL" && counted === 0)
|
|
849
1426
|
ctx.log(`the JUnit reports in $${REPORT_DIR_ENV} hold no failing case — the gate failed outside its reported tests; failures unknown`);
|
|
850
1427
|
const failures = verdict === "PASS" ? 0 : counted !== null && counted > 0 ? counted : null;
|
|
851
1428
|
let clamp = null;
|
|
852
1429
|
let artifactFailed = null;
|
|
853
|
-
if (artifact !== undefined) {
|
|
1430
|
+
if (artifact !== undefined && !lockWait) {
|
|
854
1431
|
try {
|
|
855
1432
|
const out = path.resolve(ctx.cwd, artifact);
|
|
856
1433
|
fs.mkdirSync(path.dirname(out), { recursive: true });
|
|
@@ -872,7 +1449,22 @@ export const testsCommand = {
|
|
|
872
1449
|
artifactFailed = err.message;
|
|
873
1450
|
}
|
|
874
1451
|
}
|
|
1452
|
+
const row = status === undefined || lockWait
|
|
1453
|
+
? null
|
|
1454
|
+
: appendTestRun(status, {
|
|
1455
|
+
workflow: workflow,
|
|
1456
|
+
step: step,
|
|
1457
|
+
attempt,
|
|
1458
|
+
verdict,
|
|
1459
|
+
exitCode,
|
|
1460
|
+
failures,
|
|
1461
|
+
scope,
|
|
1462
|
+
log: logArg,
|
|
1463
|
+
artifact,
|
|
1464
|
+
source: "local",
|
|
1465
|
+
});
|
|
875
1466
|
ctx.emit({
|
|
1467
|
+
source: "local",
|
|
876
1468
|
verdict,
|
|
877
1469
|
exitCode,
|
|
878
1470
|
gate,
|
|
@@ -881,15 +1473,19 @@ export const testsCommand = {
|
|
|
881
1473
|
reports: `${logArg}.reports`,
|
|
882
1474
|
log: logArg,
|
|
883
1475
|
logBytes: fs.statSync(log).size,
|
|
884
|
-
artifact: artifact ?? null,
|
|
1476
|
+
artifact: lockWait ? null : (artifact ?? null),
|
|
885
1477
|
clamp,
|
|
1478
|
+
row,
|
|
1479
|
+
lockHolder,
|
|
886
1480
|
});
|
|
887
1481
|
if (artifactFailed !== null) {
|
|
888
1482
|
ctx.log(`--artifact ${artifact}: ${artifactFailed}`);
|
|
889
1483
|
return EXIT_ARTIFACT_STEP;
|
|
890
1484
|
}
|
|
1485
|
+
if (lockWait)
|
|
1486
|
+
return EXIT.AGENT_WAIT;
|
|
891
1487
|
if (couldNotRun) {
|
|
892
|
-
ctx.log(`${scope} could not run (exit ${exitCode}: ${exitCode === 126 ? "not executable" : "command not found"}) — no tests executed; this is not a FAIL verdict`);
|
|
1488
|
+
ctx.log(`${scope} could not run (exit ${exitCode}: ${lockExit === "could-not-run" ? "tests lock refused" : exitCode === 126 ? "not executable" : "command not found"}) — no tests executed; this is not a FAIL verdict`);
|
|
893
1489
|
return EXIT_COULD_NOT_RUN;
|
|
894
1490
|
}
|
|
895
1491
|
ctx.log(`verdict ${verdict} (exit ${exitCode})`);
|
|
@@ -897,126 +1493,275 @@ export const testsCommand = {
|
|
|
897
1493
|
},
|
|
898
1494
|
},
|
|
899
1495
|
red: {
|
|
900
|
-
summary: "Run
|
|
1496
|
+
summary: "Run every TEST FIRST test of one group's open tasks before any task code exists, and write one build-tests-red row holding a state and blob per planned test; a test an earlier row of the group proved red, its file unchanged since, is not run again. Exit 0 every test red, 1 a gap (a test passing, absent from its report, failing lint, or its file missing), 3 bkit:test:file undeclared (blobs recorded, nothing ran), 4 a script could not run.",
|
|
1497
|
+
use: "workflow",
|
|
901
1498
|
flags: [
|
|
902
1499
|
FILE_FLAG,
|
|
903
1500
|
WORKFLOW_FLAG,
|
|
904
|
-
GROUP_FLAG,
|
|
905
1501
|
{
|
|
906
|
-
|
|
907
|
-
type: "string",
|
|
1502
|
+
...GROUP_FLAG,
|
|
908
1503
|
required: true,
|
|
909
|
-
help: "
|
|
1504
|
+
help: "the group whose planned tests run (positive integer)",
|
|
910
1505
|
},
|
|
911
1506
|
{
|
|
912
|
-
name: "
|
|
1507
|
+
name: "plan",
|
|
913
1508
|
type: "string",
|
|
914
1509
|
required: true,
|
|
915
|
-
|
|
1510
|
+
repeatable: true,
|
|
1511
|
+
help: "candidate plan file; the first that exists is read",
|
|
916
1512
|
},
|
|
917
1513
|
{
|
|
918
|
-
name: "
|
|
1514
|
+
name: "log",
|
|
919
1515
|
type: "string",
|
|
920
1516
|
required: true,
|
|
921
|
-
help: "the
|
|
1517
|
+
help: "file the red report is written to (directories are created)",
|
|
922
1518
|
},
|
|
923
1519
|
{
|
|
924
|
-
name: "log",
|
|
1520
|
+
name: "writer-log",
|
|
925
1521
|
type: "string",
|
|
926
|
-
|
|
927
|
-
help: "file the runner output is saved to; the test file's copy lands beside it as .snapshot",
|
|
1522
|
+
help: "the tests step's log: a `PASSES — TASK N: `file` `name`: reason` line there accepts that test's pass as passed-accepted",
|
|
928
1523
|
},
|
|
929
1524
|
],
|
|
930
1525
|
output: {
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
1526
|
+
enforced: "false when bkit:test:file is not declared (the row records the blobs, every state unknown, exit 3) or the plan could not be read (no row, exit 1)",
|
|
1527
|
+
reason: "why test-first is not enforced",
|
|
1528
|
+
attempt: "this group's build-tests-red rows so far, this one included",
|
|
1529
|
+
tests: "[{task, test, testName, state, blob, reason?}] in plan order, the tests run now: state red | red-not-loaded | passed-accepted | vacuous | missing | no-report | lint | unknown; blob the test file's git blob as it stood (null when the file is missing); reason the tests step's, on passed-accepted",
|
|
1530
|
+
skipped: "planned tests an earlier row of the group proved red, their file unchanged since: not run again, and not on this row",
|
|
1531
|
+
gaps: "[{task, test, name, kind, detail}]: every planned test whose state is not red, red-not-loaded or passed-accepted; kind is the state",
|
|
1532
|
+
row: "the build-tests-red row written",
|
|
1533
|
+
log: "where the report was written (as given)",
|
|
938
1534
|
},
|
|
939
1535
|
exitCodes: {
|
|
940
|
-
[EXIT_NO_GATE]: `${BKIT_TEST_FILE_SCRIPT} is not declared in package.json`,
|
|
941
|
-
[EXIT_COULD_NOT_RUN]: "
|
|
1536
|
+
[EXIT_NO_GATE]: `${BKIT_TEST_FILE_SCRIPT} is not declared in package.json: the row holds the blobs with enforced false, and no test ran`,
|
|
1537
|
+
[EXIT_COULD_NOT_RUN]: "bkit:lint:file or bkit:test:file could not run (exit 126 / 127); nothing written",
|
|
942
1538
|
},
|
|
943
1539
|
run: (ctx) => {
|
|
944
1540
|
const workflow = String(ctx.flags.workflow);
|
|
945
1541
|
const group = positiveInt(ctx.flags.group, "group");
|
|
946
|
-
const task = positiveInt(ctx.flags.task, "task");
|
|
947
|
-
const name = String(ctx.flags.name).trim();
|
|
948
|
-
if (name === "")
|
|
949
|
-
throw new CommandError(EXIT.USAGE, "--name must be non-empty");
|
|
950
1542
|
const logArg = String(ctx.flags.log);
|
|
951
1543
|
if (logArg === "")
|
|
952
1544
|
throw new CommandError(EXIT.USAGE, "--log must be non-empty");
|
|
1545
|
+
const log = path.resolve(ctx.cwd, logArg);
|
|
953
1546
|
const top = toplevel(ctx.cwd);
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
const test = repoTestPath(top, String(ctx.flags.test));
|
|
959
|
-
const blob = git(top, ["hash-object", "--", test]);
|
|
1547
|
+
const exists = (p) => fs.existsSync(path.resolve(ctx.cwd, p));
|
|
1548
|
+
const plan = ctx.flags.plan.find(exists);
|
|
1549
|
+
const planText = plan ? fs.readFileSync(path.resolve(ctx.cwd, plan), "utf-8") : "";
|
|
1550
|
+
const parsed = parsePlanBlocks(planText);
|
|
960
1551
|
const status = statusFile(ctx);
|
|
961
|
-
const
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
if (COULD_NOT_RUN_EXIT_CODES.has(exitCode)) {
|
|
975
|
-
ctx.log(couldNotRunMessage(exitCode));
|
|
976
|
-
return EXIT_COULD_NOT_RUN;
|
|
977
|
-
}
|
|
978
|
-
const refusal = judgeRed(exitCode, cases, name);
|
|
979
|
-
if (refusal !== null) {
|
|
980
|
-
ctx.emit({
|
|
981
|
-
recorded: false,
|
|
982
|
-
refused: refusal.kind,
|
|
983
|
-
reason: refusal.reason,
|
|
984
|
-
exitCode,
|
|
985
|
-
log: logArg,
|
|
1552
|
+
const attempt = redAttempt(status, workflow, group);
|
|
1553
|
+
const refusal = planRefusal(plan, parsed);
|
|
1554
|
+
if (refusal !== undefined) {
|
|
1555
|
+
writeRedReport(log, {
|
|
1556
|
+
enforced: false,
|
|
1557
|
+
reason: refusal,
|
|
1558
|
+
plan,
|
|
1559
|
+
group,
|
|
1560
|
+
attempt,
|
|
1561
|
+
tests: [],
|
|
1562
|
+
details: new Map(),
|
|
1563
|
+
gaps: [],
|
|
1564
|
+
outputs: [],
|
|
986
1565
|
});
|
|
987
|
-
ctx.
|
|
1566
|
+
ctx.emit({ enforced: false, reason: refusal, attempt, tests: [], gaps: [], log: logArg });
|
|
1567
|
+
ctx.log(`test-first not enforced: ${refusal}`);
|
|
988
1568
|
return EXIT.NO;
|
|
989
1569
|
}
|
|
990
|
-
const
|
|
991
|
-
|
|
992
|
-
|
|
1570
|
+
const g = parsed.groups.find((x) => x.group === group);
|
|
1571
|
+
if (g === undefined)
|
|
1572
|
+
throw new CommandError(EXIT.USAGE, `the plan has no GROUP ${group}`);
|
|
1573
|
+
if (g.tasks.length === 0)
|
|
1574
|
+
throw new CommandError(EXIT.USAGE, `GROUP ${group} has no tasks; nothing to run`);
|
|
1575
|
+
const done = new Set(parsed.tasks.filter((t) => t.done).map((t) => t.task));
|
|
1576
|
+
// A red an earlier row proved stands while the test's own lines are unchanged, so a
|
|
1577
|
+
// retry, a handover or a task added to the group runs only what is left, and an edit
|
|
1578
|
+
// to another test of the same file costs no proof. A test whose lines could not be
|
|
1579
|
+
// found, and rows from before per-test locking, stand while the whole file is unchanged.
|
|
1580
|
+
const proven = redEntries(status, workflow, group);
|
|
1581
|
+
const textOf = (test) => {
|
|
1582
|
+
const abs = path.join(top, test);
|
|
1583
|
+
return fs.existsSync(abs) ? fs.readFileSync(abs, "utf-8") : null;
|
|
1584
|
+
};
|
|
1585
|
+
const standing = (t) => {
|
|
1586
|
+
const earlier = proven.get(redKey(t.task, t.test, t.testName));
|
|
1587
|
+
if (earlier === undefined || !RED_PROVEN.has(earlier.state))
|
|
1588
|
+
return false;
|
|
1589
|
+
if (earlier.block === undefined || earlier.block === null)
|
|
1590
|
+
return git(top, ["hash-object", "--", t.test]) === earlier.blob;
|
|
1591
|
+
return blockHash(textOf(t.test), t.testName) === earlier.block;
|
|
1592
|
+
};
|
|
1593
|
+
const startSha = groupStartSha(status, workflow, group);
|
|
1594
|
+
/** The test's lines at the group's start commit: it existed before this group's tests step. */
|
|
1595
|
+
const blockAtStart = (t) => {
|
|
1596
|
+
if (startSha === undefined)
|
|
1597
|
+
return null;
|
|
1598
|
+
const shown = runGit(top, ["show", `${startSha}:${t.test}`]);
|
|
1599
|
+
return shown.ok ? blockHash(shown.stdout, t.testName) : null;
|
|
1600
|
+
};
|
|
1601
|
+
const tests = [];
|
|
1602
|
+
let skipped = 0;
|
|
1603
|
+
for (const s of readTestFirst(planText, group)) {
|
|
1604
|
+
if (s.kind !== "test" || done.has(s.task))
|
|
1605
|
+
continue;
|
|
1606
|
+
for (const t of s.tests) {
|
|
1607
|
+
const planned = {
|
|
1608
|
+
task: s.task,
|
|
1609
|
+
test: t.test,
|
|
1610
|
+
testName: t.name,
|
|
1611
|
+
state: "unknown",
|
|
1612
|
+
blob: null,
|
|
1613
|
+
};
|
|
1614
|
+
if (standing(planned))
|
|
1615
|
+
skipped += 1;
|
|
1616
|
+
else
|
|
1617
|
+
tests.push(planned);
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
if (skipped > 0)
|
|
1621
|
+
ctx.log(`tests red: ${skipped} planned test(s) already red, not run again`);
|
|
1622
|
+
const enforced = resolveGate(top, [BKIT_TEST_FILE_SCRIPT]) !== null;
|
|
1623
|
+
const lint = resolveGate(top, [BKIT_LINT_FILE_SCRIPT]) !== null;
|
|
1624
|
+
if (enforced && !lint && tests.length > 0)
|
|
1625
|
+
ctx.log(`${BKIT_LINT_FILE_SCRIPT} is not declared — the tests are not linted before their red`);
|
|
1626
|
+
const writerLog = str(ctx.flags.writerLog);
|
|
1627
|
+
const writerPath = writerLog === undefined ? undefined : path.resolve(ctx.cwd, writerLog);
|
|
1628
|
+
const passes = writerPath !== undefined && fs.existsSync(writerPath)
|
|
1629
|
+
? passesLines(fs.readFileSync(writerPath, "utf-8"))
|
|
1630
|
+
: [];
|
|
1631
|
+
const details = new Map();
|
|
1632
|
+
const outputs = [];
|
|
1633
|
+
const couldNotRun = (script, exitCode) => `${gateScope(script)} could not run (exit ${exitCode}: ${exitCode === 126 ? "not executable" : "command not found"}) — no test executed`;
|
|
1634
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bkit-tests-red-"));
|
|
1635
|
+
try {
|
|
1636
|
+
for (const file of [...new Set(tests.map((t) => t.test))]) {
|
|
1637
|
+
const mine = tests.filter((t) => t.test === file);
|
|
1638
|
+
if (!fs.existsSync(path.join(top, file))) {
|
|
1639
|
+
for (const t of mine) {
|
|
1640
|
+
t.state = "missing";
|
|
1641
|
+
details.set(t, `\`${file}\` does not exist`);
|
|
1642
|
+
}
|
|
1643
|
+
continue;
|
|
1644
|
+
}
|
|
1645
|
+
// `-w` stores the file in git: green and `steps finish` read the blob back later.
|
|
1646
|
+
const blob = git(top, ["hash-object", "-w", "--", file]);
|
|
1647
|
+
const text = textOf(file);
|
|
1648
|
+
for (const t of mine) {
|
|
1649
|
+
t.blob = blob;
|
|
1650
|
+
t.block = blockHash(text, t.testName);
|
|
1651
|
+
}
|
|
1652
|
+
if (!enforced)
|
|
1653
|
+
continue;
|
|
1654
|
+
const runLog = path.join(dir, "run.log");
|
|
1655
|
+
if (lint) {
|
|
1656
|
+
const linted = runToLog([...scriptCommand(BKIT_LINT_FILE_SCRIPT), file], top, runLog);
|
|
1657
|
+
outputs.push({
|
|
1658
|
+
title: `Lint output — \`${file}\``,
|
|
1659
|
+
text: fs.readFileSync(runLog, "utf-8"),
|
|
1660
|
+
});
|
|
1661
|
+
if (COULD_NOT_RUN_EXIT_CODES.has(linted)) {
|
|
1662
|
+
ctx.log(couldNotRun(BKIT_LINT_FILE_SCRIPT, linted));
|
|
1663
|
+
return EXIT_COULD_NOT_RUN;
|
|
1664
|
+
}
|
|
1665
|
+
if (linted !== 0) {
|
|
1666
|
+
for (const t of mine) {
|
|
1667
|
+
t.state = "lint";
|
|
1668
|
+
details.set(t, `${BKIT_LINT_FILE_SCRIPT} exited ${linted} on \`${file}\`: fix the test until it lints clean (output below)`);
|
|
1669
|
+
}
|
|
1670
|
+
continue;
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
const { exitCode, cases } = runTestFile(top, file, runLog);
|
|
1674
|
+
outputs.push({
|
|
1675
|
+
title: `Runner output — \`${file}\``,
|
|
1676
|
+
text: fs.readFileSync(runLog, "utf-8"),
|
|
1677
|
+
});
|
|
1678
|
+
if (COULD_NOT_RUN_EXIT_CODES.has(exitCode)) {
|
|
1679
|
+
ctx.log(couldNotRun(BKIT_TEST_FILE_SCRIPT, exitCode));
|
|
1680
|
+
return EXIT_COULD_NOT_RUN;
|
|
1681
|
+
}
|
|
1682
|
+
for (const t of mine) {
|
|
1683
|
+
const judged = judgeRed(cases, t.testName, file);
|
|
1684
|
+
const accepted = judged.state === "vacuous"
|
|
1685
|
+
? passes.find((p) => p.task === t.task && p.test === t.test && p.name === t.testName)
|
|
1686
|
+
: undefined;
|
|
1687
|
+
const before = judged.state === "vacuous" ? blockAtStart(t) : null;
|
|
1688
|
+
if (accepted !== undefined) {
|
|
1689
|
+
t.state = "passed-accepted";
|
|
1690
|
+
t.reason = accepted.reason;
|
|
1691
|
+
details.set(t, `passes; accepted by the tests step: ${accepted.reason}`);
|
|
1692
|
+
}
|
|
1693
|
+
else if (before !== null && before !== t.block) {
|
|
1694
|
+
t.state = "updated";
|
|
1695
|
+
t.base = startSha;
|
|
1696
|
+
details.set(t, `an existing test, changed since the group started (\`${startSha.slice(0, 9)}\`) and passing: the verifier judges its diff — it may add expectations, never remove or loosen one`);
|
|
1697
|
+
}
|
|
1698
|
+
else {
|
|
1699
|
+
t.state = judged.state;
|
|
1700
|
+
details.set(t, judged.detail);
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
finally {
|
|
1706
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
1707
|
+
}
|
|
1708
|
+
const gaps = [];
|
|
1709
|
+
for (const t of tests) {
|
|
1710
|
+
if (RED_PROVEN.has(t.state) || t.state === "unknown")
|
|
1711
|
+
continue;
|
|
1712
|
+
const gap = {
|
|
1713
|
+
task: t.task,
|
|
1714
|
+
test: t.test,
|
|
1715
|
+
name: t.testName,
|
|
1716
|
+
kind: t.state,
|
|
1717
|
+
detail: details.get(t) ?? t.state,
|
|
1718
|
+
};
|
|
1719
|
+
gaps.push(gap);
|
|
1720
|
+
ctx.log(`not a red (${gap.kind}): TASK ${gap.task} \`${gap.test}\` "${gap.name}" — ${gap.detail}`);
|
|
1721
|
+
}
|
|
1722
|
+
writeRedReport(log, {
|
|
1723
|
+
enforced,
|
|
1724
|
+
reason: enforced
|
|
1725
|
+
? undefined
|
|
1726
|
+
: `\`${BKIT_TEST_FILE_SCRIPT}\` is not declared in package.json`,
|
|
1727
|
+
plan,
|
|
1728
|
+
group,
|
|
1729
|
+
attempt,
|
|
1730
|
+
tests,
|
|
1731
|
+
details,
|
|
1732
|
+
gaps,
|
|
1733
|
+
outputs,
|
|
1734
|
+
});
|
|
993
1735
|
const written = writeStepRow(ctx.cwd, {
|
|
994
|
-
file:
|
|
1736
|
+
file: status,
|
|
995
1737
|
workflow,
|
|
996
1738
|
step: RED_STEP,
|
|
997
1739
|
group,
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
test,
|
|
1002
|
-
testName: name,
|
|
1003
|
-
testBlob: blob,
|
|
1004
|
-
exitCode,
|
|
1005
|
-
failure: failed.message.slice(0, 200),
|
|
1006
|
-
},
|
|
1740
|
+
attempt,
|
|
1741
|
+
artifacts: [path.relative(ctx.cwd, log)],
|
|
1742
|
+
extras: { enforced, tests },
|
|
1007
1743
|
commit: "none",
|
|
1008
1744
|
}, ctx.log);
|
|
1009
|
-
ctx.emit({
|
|
1010
|
-
|
|
1745
|
+
ctx.emit({ enforced, attempt, tests, skipped, gaps, row: written.row, log: logArg });
|
|
1746
|
+
if (gaps.length > 0) {
|
|
1747
|
+
ctx.log(`tests red: ${gaps.length} gap(s) — report in ${logArg}`);
|
|
1748
|
+
return EXIT.NO;
|
|
1749
|
+
}
|
|
1750
|
+
if (!enforced) {
|
|
1751
|
+
ctx.log(`${BKIT_TEST_FILE_SCRIPT} is not declared — ${NO_TEST_FILE_HINT}`);
|
|
1752
|
+
return EXIT_NO_GATE;
|
|
1753
|
+
}
|
|
1754
|
+
ctx.log(`tests red: ${tests.length} planned test(s) red — report in ${logArg}`);
|
|
1011
1755
|
return EXIT.OK;
|
|
1012
1756
|
},
|
|
1013
1757
|
},
|
|
1014
1758
|
green: {
|
|
1015
|
-
summary: "Audit a plan's test-first evidence (
|
|
1759
|
+
summary: "Audit a plan's test-first evidence (each TEST FIRST test red at its group's tests step, by the build-tests-red row, or accepted there as passing before any task code; the named test passing now); exit 0 no gaps, 1 gaps or a plan with no task markers.",
|
|
1760
|
+
use: "workflow",
|
|
1016
1761
|
flags: [
|
|
1017
1762
|
FILE_FLAG,
|
|
1018
1763
|
WORKFLOW_FLAG,
|
|
1019
|
-
{ ...GROUP_FLAG, help: "audit only this
|
|
1764
|
+
{ ...GROUP_FLAG, help: "audit only this task group's tasks (positive integer)" },
|
|
1020
1765
|
{
|
|
1021
1766
|
name: "plan",
|
|
1022
1767
|
type: "string",
|
|
@@ -1035,9 +1780,11 @@ export const testsCommand = {
|
|
|
1035
1780
|
enforced: "false when the audit could not run: no plan file, a plan with no task markers (exit 1), or the test-file script undeclared (exit 0)",
|
|
1036
1781
|
reason: "why test-first is not enforced",
|
|
1037
1782
|
plan: "the plan file audited",
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1783
|
+
verdict: "PASS no gaps | FAIL a gap's fixable is false | CONCERNS every gap's fixable is true | null when a gap needs a diff read, or the audit could not run",
|
|
1784
|
+
gaps: "[{task, kind, detail, fixable, logLine}]: no-test-first | test-first-malformed | test-first-unread (task null: a plan-level format problem) | not-green | exception-refused (a task whose only test field is a TEST EXCEPTION not honoured) | delete-still-referenced (a DELETE line's file still exists, or its name is still found in the tree); fixable is null when the gap needs a diff read; logLine is the line the verifier appends",
|
|
1785
|
+
alreadyLanded: "[{task, test, name, reason}]: tests the tests step accepted as passing before any task code (passed-accepted on the red row); no red is due for them",
|
|
1786
|
+
unproven: "[{task, test, name, detail}]: tests with no proven red (no build-tests-red row, or a state other than red, red-not-loaded, updated or passed-accepted) and no honoured TEST EXCEPTION; never a gap — the owner answers them once, at the end of the build",
|
|
1787
|
+
updated: "[{task, test, name, base, additive, diff}]: existing tests the tests step changed (updated on the red row); base is the group's start commit; additive = the test's own lines since base gained lines and lost none (null when a side cannot be read), diff = their unified line diff; the verifier judges each diff — it may add expectations, never remove or loosen one",
|
|
1041
1788
|
log: "where the report was written (as given)",
|
|
1042
1789
|
},
|
|
1043
1790
|
exitCodes: {
|
|
@@ -1058,13 +1805,7 @@ export const testsCommand = {
|
|
|
1058
1805
|
const specs = readTestFirst(planText, group);
|
|
1059
1806
|
// A plan with no task markers is an old-format plan: nothing it says can be audited,
|
|
1060
1807
|
// and passing it would let its tests go unchecked.
|
|
1061
|
-
const refusal = plan
|
|
1062
|
-
? `no plan file; ${NO_MARKERS_REASON}`
|
|
1063
|
-
: parsed.tasks.length === 0
|
|
1064
|
-
? parsed.fenceOpen === undefined
|
|
1065
|
-
? NO_MARKERS_REASON
|
|
1066
|
-
: fenceNeverClosed(parsed.fenceOpen)
|
|
1067
|
-
: undefined;
|
|
1808
|
+
const refusal = planRefusal(plan, parsed);
|
|
1068
1809
|
if (refusal === undefined && group !== undefined) {
|
|
1069
1810
|
const g = parsed.groups.find((x) => x.group === group);
|
|
1070
1811
|
if (g === undefined)
|
|
@@ -1077,12 +1818,13 @@ export const testsCommand = {
|
|
|
1077
1818
|
? `\`${BKIT_TEST_FILE_SCRIPT}\` is not declared in package.json`
|
|
1078
1819
|
: undefined);
|
|
1079
1820
|
const empty = {
|
|
1821
|
+
verdict: null,
|
|
1080
1822
|
rows: [],
|
|
1081
1823
|
gaps: [],
|
|
1082
|
-
testOrderOnly: false,
|
|
1083
1824
|
exceptions: [],
|
|
1084
|
-
|
|
1085
|
-
|
|
1825
|
+
landed: [],
|
|
1826
|
+
unproven: [],
|
|
1827
|
+
updated: [],
|
|
1086
1828
|
outputs: [],
|
|
1087
1829
|
};
|
|
1088
1830
|
if (reason !== undefined) {
|
|
@@ -1091,22 +1833,25 @@ export const testsCommand = {
|
|
|
1091
1833
|
enforced: false,
|
|
1092
1834
|
reason,
|
|
1093
1835
|
plan: plan ?? null,
|
|
1836
|
+
verdict: null,
|
|
1094
1837
|
gaps: [],
|
|
1095
|
-
|
|
1096
|
-
|
|
1838
|
+
alreadyLanded: [],
|
|
1839
|
+
unproven: [],
|
|
1840
|
+
updated: [],
|
|
1097
1841
|
log: logArg,
|
|
1098
1842
|
});
|
|
1099
1843
|
ctx.log(`test-first not enforced: ${reason}`);
|
|
1100
1844
|
return refusal === undefined ? EXIT.OK : EXIT.NO;
|
|
1101
1845
|
}
|
|
1102
|
-
const reds =
|
|
1846
|
+
const reds = redEntries(statusFile(ctx), workflow);
|
|
1847
|
+
const groupOf = (task) => parsed.tasks.find((b) => b.task === task)?.group;
|
|
1848
|
+
const landed = [];
|
|
1849
|
+
const unproven = [];
|
|
1850
|
+
const updated = [];
|
|
1103
1851
|
const statusRel = repoRel(top, ctx.cwd, statusFile(ctx));
|
|
1104
|
-
const codeCommits = taskCodeCommits(top, ctx.cwd, statusFile(ctx), workflow);
|
|
1105
1852
|
const exceptions = [];
|
|
1106
|
-
const lateUnchecked = new Set();
|
|
1107
1853
|
const gaps = [];
|
|
1108
1854
|
const rows = [];
|
|
1109
|
-
const diffs = [];
|
|
1110
1855
|
const outputs = [];
|
|
1111
1856
|
const greens = new Map();
|
|
1112
1857
|
/** One run per test file; null when the script could not run. */
|
|
@@ -1139,21 +1884,13 @@ export const testsCommand = {
|
|
|
1139
1884
|
cleared: refused === null ? ["the task's `TEST FIRST` check"] : [],
|
|
1140
1885
|
});
|
|
1141
1886
|
if (refused !== null)
|
|
1142
|
-
gaps.push({
|
|
1143
|
-
task: s.task,
|
|
1144
|
-
kind: EXCEPTION_REFUSED,
|
|
1145
|
-
detail: `the task's only test field is a \`**TEST EXCEPTION:**\`, not honoured: ${refused}`,
|
|
1146
|
-
});
|
|
1887
|
+
gaps.push(gapOf(s.task, EXCEPTION_REFUSED, `the task's only test field is a \`**TEST EXCEPTION:**\`, not honoured: ${refused}`));
|
|
1147
1888
|
continue;
|
|
1148
1889
|
}
|
|
1149
1890
|
if (s.kind !== "test") {
|
|
1150
|
-
gaps.push(
|
|
1151
|
-
task
|
|
1152
|
-
|
|
1153
|
-
detail: s.kind === "missing"
|
|
1154
|
-
? "the task has neither a `**TEST FIRST:**` nor a `**TEST EXCEPTION:**` field"
|
|
1155
|
-
: "`**TEST FIRST:**` is neither `n/a — {reason}` nor one sub-bullet per test naming a backticked test file and test name",
|
|
1156
|
-
});
|
|
1891
|
+
gaps.push(gapOf(s.task, s.kind === "missing" ? "no-test-first" : "test-first-malformed", s.kind === "missing"
|
|
1892
|
+
? "the task has neither a `**TEST FIRST:**` nor a `**TEST EXCEPTION:**` field"
|
|
1893
|
+
: "`**TEST FIRST:**` is neither `n/a — {reason}` nor one sub-bullet per test naming a backticked test file and test name"));
|
|
1157
1894
|
continue;
|
|
1158
1895
|
}
|
|
1159
1896
|
const excepted = s.exception;
|
|
@@ -1166,14 +1903,11 @@ export const testsCommand = {
|
|
|
1166
1903
|
};
|
|
1167
1904
|
if (exception !== undefined)
|
|
1168
1905
|
exceptions.push(exception);
|
|
1169
|
-
const taskReds = reds.filter((r) => r.task === s.task);
|
|
1170
|
-
const planned = (r) => s.tests.some((t) => t.test === r.test && t.name === r.testName);
|
|
1171
1906
|
for (const t of s.tests) {
|
|
1172
1907
|
const row = {
|
|
1173
1908
|
task: s.task,
|
|
1174
1909
|
test: `\`${t.test}\` "${t.name}"`,
|
|
1175
|
-
red: "
|
|
1176
|
-
changed: "—",
|
|
1910
|
+
red: "no row",
|
|
1177
1911
|
green: "—",
|
|
1178
1912
|
};
|
|
1179
1913
|
rows.push(row);
|
|
@@ -1181,76 +1915,28 @@ export const testsCommand = {
|
|
|
1181
1915
|
excepted.tests.some((e) => e.test === t.test && e.name === t.name)
|
|
1182
1916
|
? exception
|
|
1183
1917
|
: undefined;
|
|
1184
|
-
const gap = (kind, detail) => {
|
|
1185
|
-
|
|
1186
|
-
|
|
1918
|
+
const gap = (kind, detail) => gaps.push(gapOf(s.task, kind, detail, { file: t.test, name: t.name }));
|
|
1919
|
+
const noRed = (detail) => {
|
|
1920
|
+
if (clearing !== undefined)
|
|
1921
|
+
clearing.cleared.push(`the red for \`${t.test}\` "${t.name}"`);
|
|
1187
1922
|
else
|
|
1188
|
-
|
|
1923
|
+
unproven.push({ task: s.task, test: t.test, name: t.name, detail });
|
|
1189
1924
|
};
|
|
1190
|
-
const red =
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
for (const r of unplanned) {
|
|
1197
|
-
const rTest = String(r.test);
|
|
1198
|
-
const rName = String(r.testName);
|
|
1199
|
-
let now = "file missing";
|
|
1200
|
-
if (fs.existsSync(path.join(top, rTest))) {
|
|
1201
|
-
const run = greenRun(rTest);
|
|
1202
|
-
if (run === null)
|
|
1203
|
-
return EXIT_COULD_NOT_RUN;
|
|
1204
|
-
const why = judgeGreen(run.cases, rName);
|
|
1205
|
-
now = why === null ? "passes now" : `not passing: ${why}`;
|
|
1206
|
-
}
|
|
1207
|
-
recorded.push(`\`${rTest}\` "${rName}" (${now})`);
|
|
1208
|
-
}
|
|
1209
|
-
gap("red-mismatch", `no red for \`${t.test}\` "${t.name}"; reds recorded for ${recorded.join(", ")}`);
|
|
1925
|
+
const red = reds.get(redKey(s.task, t.test, t.name));
|
|
1926
|
+
if (red === undefined)
|
|
1927
|
+
noRed(`no tests step recorded a red for it (\`bkit tests red --group=${groupOf(s.task) ?? "N"}\`)`);
|
|
1928
|
+
else if (red.state === "passed-accepted") {
|
|
1929
|
+
row.red = red.state;
|
|
1930
|
+
landed.push({ task: s.task, test: t.test, name: t.name, reason: red.reason ?? "" });
|
|
1210
1931
|
}
|
|
1211
|
-
else if (red
|
|
1212
|
-
|
|
1932
|
+
else if (RED_PROVEN.has(red.state)) {
|
|
1933
|
+
row.red = red.state;
|
|
1934
|
+
if (red.state === "updated")
|
|
1935
|
+
updated.push(updatedEntry(top, s.task, t, red.base ?? null));
|
|
1213
1936
|
}
|
|
1214
1937
|
else {
|
|
1215
|
-
row.red =
|
|
1216
|
-
|
|
1217
|
-
const on = typeof red.parentSha === "string" ? red.parentSha : undefined;
|
|
1218
|
-
if (code === undefined || on === undefined)
|
|
1219
|
-
lateUnchecked.add(s.task);
|
|
1220
|
-
else if (runGit(top, ["merge-base", "--is-ancestor", code, on]).ok) {
|
|
1221
|
-
row.red = `late — ${row.red}`;
|
|
1222
|
-
gap("late-red", `the red for "${t.name}" was recorded on ${on.slice(0, 7)}, on or after ${code.slice(0, 7)}, the first commit whose plan-execute log records TASK ${s.task} DONE — the task's code already existed`);
|
|
1223
|
-
}
|
|
1224
|
-
const snap = (Array.isArray(red.artifacts) ? red.artifacts : [])
|
|
1225
|
-
.map(String)
|
|
1226
|
-
.find((a) => a.endsWith(".snapshot"));
|
|
1227
|
-
const snapAbs = snap === undefined ? undefined : path.resolve(ctx.cwd, snap);
|
|
1228
|
-
const testAbs = path.join(top, t.test);
|
|
1229
|
-
if (snapAbs === undefined || !fs.existsSync(snapAbs)) {
|
|
1230
|
-
row.changed = "unknown";
|
|
1231
|
-
gap("red-evidence-missing", `the red's test snapshot ${snap ?? "(none recorded)"} is missing`);
|
|
1232
|
-
}
|
|
1233
|
-
else if (!fs.existsSync(testAbs)) {
|
|
1234
|
-
row.changed = "deleted";
|
|
1235
|
-
gap("test-changed", `\`${t.test}\` no longer exists`);
|
|
1236
|
-
}
|
|
1237
|
-
else if (!fs.readFileSync(snapAbs).equals(fs.readFileSync(testAbs))) {
|
|
1238
|
-
const added = onlyAddsLines(fs.readFileSync(snapAbs, "utf-8"), fs.readFileSync(testAbs, "utf-8"));
|
|
1239
|
-
row.changed = added ? "added lines only" : "yes";
|
|
1240
|
-
if (!added)
|
|
1241
|
-
gap("test-changed", `\`${t.test}\` differs from its red snapshot for "${t.name}" (diff below)`);
|
|
1242
|
-
const diff = runGit(top, [
|
|
1243
|
-
"diff",
|
|
1244
|
-
"--no-index",
|
|
1245
|
-
"--no-color",
|
|
1246
|
-
"--",
|
|
1247
|
-
path.relative(top, snapAbs),
|
|
1248
|
-
t.test,
|
|
1249
|
-
]);
|
|
1250
|
-
diffs.push({ task: s.task, test: t.test, diff: diff.stdout });
|
|
1251
|
-
}
|
|
1252
|
-
else
|
|
1253
|
-
row.changed = "no";
|
|
1938
|
+
row.red = red.state;
|
|
1939
|
+
noRed(`the tests step left it as \`${red.state}\`, not red`);
|
|
1254
1940
|
}
|
|
1255
1941
|
if (!fs.existsSync(path.join(top, t.test))) {
|
|
1256
1942
|
row.green = "file missing";
|
|
@@ -1271,37 +1957,37 @@ export const testsCommand = {
|
|
|
1271
1957
|
}
|
|
1272
1958
|
gaps.push(...deleteGaps(top, repoRel(top, ctx.cwd, plan), parsed.tasks.filter((t) => group === undefined || t.group === group), parsed.tasks));
|
|
1273
1959
|
for (const e of parsed.errors)
|
|
1274
|
-
gaps.push({
|
|
1275
|
-
|
|
1276
|
-
kind: "test-first-unread",
|
|
1277
|
-
detail: `\`${plan}:${e.line}\` ${e.message} — tests there were not audited; the plan's blocks need fixing`,
|
|
1278
|
-
});
|
|
1279
|
-
const testOrderOnly = gaps.length > 0 && gaps.every((g) => TEST_ORDER_GAPS.has(g.kind));
|
|
1960
|
+
gaps.push(gapOf(null, "test-first-unread", `\`${plan}:${e.line}\` ${e.message} — tests there were not audited; the plan's blocks need fixing`));
|
|
1961
|
+
const verdict = verdictOf(gaps);
|
|
1280
1962
|
writeAuditReport(log, {
|
|
1281
1963
|
enforced: true,
|
|
1282
1964
|
plan,
|
|
1965
|
+
verdict,
|
|
1283
1966
|
rows,
|
|
1284
1967
|
gaps,
|
|
1285
|
-
testOrderOnly,
|
|
1286
1968
|
exceptions,
|
|
1287
|
-
|
|
1288
|
-
|
|
1969
|
+
landed,
|
|
1970
|
+
unproven,
|
|
1971
|
+
updated,
|
|
1289
1972
|
outputs,
|
|
1290
1973
|
});
|
|
1291
1974
|
ctx.emit({
|
|
1292
1975
|
enforced: true,
|
|
1293
1976
|
plan,
|
|
1977
|
+
verdict,
|
|
1294
1978
|
gaps,
|
|
1295
|
-
|
|
1296
|
-
|
|
1979
|
+
alreadyLanded: landed,
|
|
1980
|
+
unproven,
|
|
1981
|
+
updated,
|
|
1297
1982
|
log: logArg,
|
|
1298
1983
|
});
|
|
1299
|
-
ctx.log(`test-first audit: ${gaps.length} gap(s) — report in ${logArg}`);
|
|
1984
|
+
ctx.log(`test-first audit: ${gaps.length} gap(s), ${unproven.length} unproven — report in ${logArg}`);
|
|
1300
1985
|
return gaps.length === 0 ? EXIT.OK : EXIT.NO;
|
|
1301
1986
|
},
|
|
1302
1987
|
},
|
|
1303
1988
|
"plan-check": {
|
|
1304
|
-
summary: `Check a plan's group and task blocks: the block format, DONE WHEN and TEST FIRST on every task, DEPENDS ON order, the caps from package.json \`bkit.workflows.tasks\` (defaults: ${capsText(CONFIG_DEFAULTS.tasks)}), a Reuse Inventory NEW row per CREATE file, and that each READ / UPDATE / DELETE file and symbol exists; exit 0 no problems, 1 refused.`,
|
|
1989
|
+
summary: `Check a plan's group and task blocks: the block format, DONE WHEN and TEST FIRST on every task, DEPENDS ON order, the caps from package.json \`bkit.workflows.tasks\` (defaults: ${capsText(CONFIG_DEFAULTS.tasks)}), a Reuse Inventory NEW row per CREATE file, a Supersession Inventory row per DELETE, a \`Task\` on every Derived Test Matrix row and a row per test-bearing task, and that each READ / UPDATE / DELETE file and symbol exists; exit 0 no problems, 1 refused.`,
|
|
1990
|
+
use: "by-hand",
|
|
1305
1991
|
flags: [
|
|
1306
1992
|
{
|
|
1307
1993
|
name: "plan",
|
|
@@ -1325,8 +2011,9 @@ export const testsCommand = {
|
|
|
1325
2011
|
plan: "the plan file checked (as given)",
|
|
1326
2012
|
groups: "[{group, title, tasks, tests, kb}]: each group's tasks, planned TEST FIRST tests and size",
|
|
1327
2013
|
tasks: "[{task, group, tests, files}]: each task's planned tests and distinct test files",
|
|
1328
|
-
problems: "[{line, kind, message}]: format | stray | field | depends | cap | inventory | markers — each refuses the plan",
|
|
2014
|
+
problems: "[{line, kind, message}]: format | stray | field | depends | cap | inventory | matrix | supersession | markers — each refuses the plan",
|
|
1329
2015
|
existence: "[{line, task, message}]: READ / UPDATE / DELETE files or symbols not found; refuses only with --existence=refuse",
|
|
2016
|
+
existing: "[{line, task, test, name}]: planned tests of open tasks whose name a test in their file already carries; reported only, never refuses",
|
|
1330
2017
|
log: "where the report was written (as given)",
|
|
1331
2018
|
},
|
|
1332
2019
|
run: (ctx) => {
|
|
@@ -1344,6 +2031,7 @@ export const testsCommand = {
|
|
|
1344
2031
|
const caps = planCaps(ctx.cwd);
|
|
1345
2032
|
const { groups, sizes, problems } = checkPlanText(text, caps);
|
|
1346
2033
|
const existence = planExistence(text, toplevel(ctx.cwd));
|
|
2034
|
+
const existing = existingTestNames(text, toplevel(ctx.cwd));
|
|
1347
2035
|
const refused = problems.length > 0 || (existenceArg === "refuse" && existence.length > 0);
|
|
1348
2036
|
const out = [
|
|
1349
2037
|
`Problems: ${problems.length}`,
|
|
@@ -1380,10 +2068,14 @@ export const testsCommand = {
|
|
|
1380
2068
|
out.push("None.");
|
|
1381
2069
|
for (const e of existence)
|
|
1382
2070
|
out.push(`- \`${plan}:${e.line}\` — ${e.message}`);
|
|
2071
|
+
if (existing.length > 0)
|
|
2072
|
+
out.push("", "## Planned test names already in use (reported only)", "");
|
|
2073
|
+
for (const e of existing)
|
|
2074
|
+
out.push(`- \`${plan}:${e.line}\` TASK ${e.task}: \`${e.test}\` already holds a test named "${e.name}" — name the update to it in TEST FIRST, or give the new test a new name`);
|
|
1383
2075
|
const log = path.resolve(ctx.cwd, logArg);
|
|
1384
2076
|
fs.mkdirSync(path.dirname(log), { recursive: true });
|
|
1385
2077
|
fs.writeFileSync(log, `${out.join("\n")}\n`);
|
|
1386
|
-
ctx.emit({ plan, groups, tasks: sizes, problems, existence, log: logArg });
|
|
2078
|
+
ctx.emit({ plan, groups, tasks: sizes, problems, existence, existing, log: logArg });
|
|
1387
2079
|
ctx.log(`plan check: ${problems.length} problem(s), ${existence.length} existence finding(s)${refused ? " — refused" : ""}; report in ${logArg}`);
|
|
1388
2080
|
return refused ? EXIT.NO : EXIT.OK;
|
|
1389
2081
|
},
|