autonomous-coding-toolkit 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +22 -0
- package/.claude-plugin/plugin.json +13 -0
- package/LICENSE +21 -0
- package/Makefile +21 -0
- package/README.md +140 -0
- package/SECURITY.md +28 -0
- package/agents/bash-expert.md +113 -0
- package/agents/dependency-auditor.md +138 -0
- package/agents/integration-tester.md +120 -0
- package/agents/lesson-scanner.md +149 -0
- package/agents/python-expert.md +179 -0
- package/agents/service-monitor.md +141 -0
- package/agents/shell-expert.md +147 -0
- package/benchmarks/runner.sh +147 -0
- package/benchmarks/tasks/01-rest-endpoint/rubric.sh +29 -0
- package/benchmarks/tasks/01-rest-endpoint/task.md +17 -0
- package/benchmarks/tasks/02-refactor-module/task.md +8 -0
- package/benchmarks/tasks/03-fix-integration-bug/task.md +8 -0
- package/benchmarks/tasks/04-add-test-coverage/task.md +8 -0
- package/benchmarks/tasks/05-multi-file-feature/task.md +8 -0
- package/bin/act.js +238 -0
- package/commands/autocode.md +6 -0
- package/commands/cancel-ralph.md +18 -0
- package/commands/code-factory.md +53 -0
- package/commands/create-prd.md +55 -0
- package/commands/ralph-loop.md +18 -0
- package/commands/run-plan.md +117 -0
- package/commands/submit-lesson.md +122 -0
- package/docs/ARCHITECTURE.md +630 -0
- package/docs/CONTRIBUTING.md +125 -0
- package/docs/lessons/0001-bare-exception-swallowing.md +34 -0
- package/docs/lessons/0002-async-def-without-await.md +28 -0
- package/docs/lessons/0003-create-task-without-callback.md +28 -0
- package/docs/lessons/0004-hardcoded-test-counts.md +28 -0
- package/docs/lessons/0005-sqlite-without-closing.md +33 -0
- package/docs/lessons/0006-venv-pip-path.md +27 -0
- package/docs/lessons/0007-runner-state-self-rejection.md +35 -0
- package/docs/lessons/0008-quality-gate-blind-spot.md +33 -0
- package/docs/lessons/0009-parser-overcount-empty-batches.md +36 -0
- package/docs/lessons/0010-local-outside-function-bash.md +33 -0
- package/docs/lessons/0011-batch-tests-for-unimplemented-code.md +36 -0
- package/docs/lessons/0012-api-markdown-unescaped-chars.md +33 -0
- package/docs/lessons/0013-export-prefix-env-parsing.md +33 -0
- package/docs/lessons/0014-decorator-registry-import-side-effect.md +43 -0
- package/docs/lessons/0015-frontend-backend-schema-drift.md +43 -0
- package/docs/lessons/0016-event-driven-cold-start-seeding.md +44 -0
- package/docs/lessons/0017-copy-paste-logic-diverges.md +43 -0
- package/docs/lessons/0018-layer-passes-pipeline-broken.md +45 -0
- package/docs/lessons/0019-systemd-envfile-ignores-export.md +41 -0
- package/docs/lessons/0020-persist-state-incrementally.md +44 -0
- package/docs/lessons/0021-dual-axis-testing.md +48 -0
- package/docs/lessons/0022-jsx-factory-shadowing.md +43 -0
- package/docs/lessons/0023-static-analysis-spiral.md +51 -0
- package/docs/lessons/0024-shared-pipeline-implementation.md +55 -0
- package/docs/lessons/0025-defense-in-depth-all-entry-points.md +65 -0
- package/docs/lessons/0026-linter-no-rules-false-enforcement.md +54 -0
- package/docs/lessons/0027-jsx-silent-prop-drop.md +64 -0
- package/docs/lessons/0028-no-infrastructure-in-client-code.md +49 -0
- package/docs/lessons/0029-never-write-secrets-to-files.md +61 -0
- package/docs/lessons/0030-cache-merge-not-replace.md +62 -0
- package/docs/lessons/0031-verify-units-at-boundaries.md +66 -0
- package/docs/lessons/0032-module-lifecycle-subscribe-unsubscribe.md +89 -0
- package/docs/lessons/0033-async-iteration-mutable-snapshot.md +72 -0
- package/docs/lessons/0034-caller-missing-await-silent-discard.md +65 -0
- package/docs/lessons/0035-duplicate-registration-silent-overwrite.md +85 -0
- package/docs/lessons/0036-websocket-dirty-disconnect.md +33 -0
- package/docs/lessons/0037-parallel-agents-worktree-corruption.md +31 -0
- package/docs/lessons/0038-subscribe-no-stored-ref.md +36 -0
- package/docs/lessons/0039-fallback-or-default-hides-bugs.md +34 -0
- package/docs/lessons/0040-event-firehose-filter-first.md +36 -0
- package/docs/lessons/0041-ambiguous-base-dir-path-nesting.md +32 -0
- package/docs/lessons/0042-spec-compliance-insufficient.md +36 -0
- package/docs/lessons/0043-exact-count-extensible-collections.md +32 -0
- package/docs/lessons/0044-relative-file-deps-worktree.md +39 -0
- package/docs/lessons/0045-iterative-design-improvement.md +33 -0
- package/docs/lessons/0046-plan-assertion-math-bugs.md +38 -0
- package/docs/lessons/0047-pytest-single-threaded-default.md +37 -0
- package/docs/lessons/0048-integration-wiring-batch.md +40 -0
- package/docs/lessons/0049-ab-verification.md +41 -0
- package/docs/lessons/0050-editing-sourced-files-during-execution.md +33 -0
- package/docs/lessons/0051-infrastructure-fixes-cant-self-heal.md +30 -0
- package/docs/lessons/0052-uncommitted-changes-poison-quality-gates.md +31 -0
- package/docs/lessons/0053-jq-compact-flag-inconsistency.md +31 -0
- package/docs/lessons/0054-parser-matches-inside-code-blocks.md +30 -0
- package/docs/lessons/0055-agents-compensate-for-garbled-prompts.md +31 -0
- package/docs/lessons/0056-grep-count-exit-code-on-zero.md +42 -0
- package/docs/lessons/0057-new-artifacts-break-git-clean-gates.md +42 -0
- package/docs/lessons/0058-dead-config-keys-never-consumed.md +49 -0
- package/docs/lessons/0059-contract-test-shared-structures.md +53 -0
- package/docs/lessons/0060-set-e-silent-death-in-runners.md +53 -0
- package/docs/lessons/0061-context-injection-dirty-state.md +50 -0
- package/docs/lessons/0062-sibling-bug-neighborhood-scan.md +29 -0
- package/docs/lessons/0063-one-flag-two-lifetimes.md +31 -0
- package/docs/lessons/0064-test-passes-wrong-reason.md +31 -0
- package/docs/lessons/0065-pipefail-grep-count-double-output.md +39 -0
- package/docs/lessons/0066-local-keyword-outside-function.md +37 -0
- package/docs/lessons/0067-stdin-hang-non-interactive-shell.md +36 -0
- package/docs/lessons/0068-agent-builds-wrong-thing-correctly.md +31 -0
- package/docs/lessons/0069-plan-quality-dominates-execution.md +30 -0
- package/docs/lessons/0070-spec-echo-back-prevents-drift.md +31 -0
- package/docs/lessons/0071-positive-instructions-outperform-negative.md +30 -0
- package/docs/lessons/0072-lost-in-the-middle-context-placement.md +30 -0
- package/docs/lessons/0073-unscoped-lessons-cause-false-positives.md +30 -0
- package/docs/lessons/0074-stale-context-injection-wrong-batch.md +32 -0
- package/docs/lessons/0075-research-artifacts-must-persist.md +32 -0
- package/docs/lessons/0076-wrong-decomposition-contaminates-downstream.md +30 -0
- package/docs/lessons/0077-cherry-pick-merges-need-manual-resolution.md +30 -0
- package/docs/lessons/0078-static-review-without-live-test.md +30 -0
- package/docs/lessons/0079-integration-wiring-batch-required.md +32 -0
- package/docs/lessons/FRAMEWORK.md +161 -0
- package/docs/lessons/SUMMARY.md +201 -0
- package/docs/lessons/TEMPLATE.md +85 -0
- package/docs/plans/2026-02-21-code-factory-v2-design.md +204 -0
- package/docs/plans/2026-02-21-code-factory-v2-implementation-plan.md +2189 -0
- package/docs/plans/2026-02-21-code-factory-v2-phase4-design.md +537 -0
- package/docs/plans/2026-02-21-code-factory-v2-phase4-implementation-plan.md +2012 -0
- package/docs/plans/2026-02-21-hardening-pass-design.md +108 -0
- package/docs/plans/2026-02-21-hardening-pass-plan.md +1378 -0
- package/docs/plans/2026-02-21-mab-research-report.md +406 -0
- package/docs/plans/2026-02-21-marketplace-restructure-design.md +240 -0
- package/docs/plans/2026-02-21-marketplace-restructure-plan.md +832 -0
- package/docs/plans/2026-02-21-phase4-completion-plan.md +697 -0
- package/docs/plans/2026-02-21-validator-suite-design.md +148 -0
- package/docs/plans/2026-02-21-validator-suite-plan.md +540 -0
- package/docs/plans/2026-02-22-mab-research-round2.md +556 -0
- package/docs/plans/2026-02-22-mab-run-design.md +462 -0
- package/docs/plans/2026-02-22-mab-run-plan.md +2046 -0
- package/docs/plans/2026-02-22-operations-design-methodology-research.md +681 -0
- package/docs/plans/2026-02-22-research-agent-failure-taxonomy.md +532 -0
- package/docs/plans/2026-02-22-research-code-guideline-policies.md +886 -0
- package/docs/plans/2026-02-22-research-codebase-audit-refactoring.md +908 -0
- package/docs/plans/2026-02-22-research-coding-standards-documentation.md +541 -0
- package/docs/plans/2026-02-22-research-competitive-landscape.md +687 -0
- package/docs/plans/2026-02-22-research-comprehensive-testing.md +1076 -0
- package/docs/plans/2026-02-22-research-context-utilization.md +459 -0
- package/docs/plans/2026-02-22-research-cost-quality-tradeoff.md +548 -0
- package/docs/plans/2026-02-22-research-lesson-transferability.md +508 -0
- package/docs/plans/2026-02-22-research-multi-agent-coordination.md +312 -0
- package/docs/plans/2026-02-22-research-phase-integration.md +602 -0
- package/docs/plans/2026-02-22-research-plan-quality.md +428 -0
- package/docs/plans/2026-02-22-research-prompt-engineering.md +558 -0
- package/docs/plans/2026-02-22-research-unconventional-perspectives.md +528 -0
- package/docs/plans/2026-02-22-research-user-adoption.md +638 -0
- package/docs/plans/2026-02-22-research-verification-effectiveness.md +433 -0
- package/docs/plans/2026-02-23-agent-suite-design.md +299 -0
- package/docs/plans/2026-02-23-agent-suite-plan.md +578 -0
- package/docs/plans/2026-02-23-phase3-cost-infrastructure-design.md +148 -0
- package/docs/plans/2026-02-23-phase3-cost-infrastructure-plan.md +1062 -0
- package/docs/plans/2026-02-23-research-bash-expert-agent.md +543 -0
- package/docs/plans/2026-02-23-research-dependency-auditor-agent.md +564 -0
- package/docs/plans/2026-02-23-research-improving-existing-agents.md +503 -0
- package/docs/plans/2026-02-23-research-integration-tester-agent.md +454 -0
- package/docs/plans/2026-02-23-research-python-expert-agent.md +429 -0
- package/docs/plans/2026-02-23-research-service-monitor-agent.md +425 -0
- package/docs/plans/2026-02-23-research-shell-expert-agent.md +533 -0
- package/docs/plans/2026-02-23-roadmap-to-completion.md +530 -0
- package/docs/plans/2026-02-24-headless-module-split-design.md +98 -0
- package/docs/plans/2026-02-24-headless-module-split.md +443 -0
- package/docs/plans/2026-02-24-lesson-scope-metadata-design.md +228 -0
- package/docs/plans/2026-02-24-lesson-scope-metadata-plan.md +968 -0
- package/docs/plans/2026-02-24-npm-packaging-design.md +841 -0
- package/docs/plans/2026-02-24-npm-packaging-plan.md +1965 -0
- package/docs/plans/audit-findings.md +186 -0
- package/docs/telegram-notification-format.md +98 -0
- package/examples/example-plan.md +51 -0
- package/examples/example-prd.json +72 -0
- package/examples/example-roadmap.md +33 -0
- package/examples/quickstart-plan.md +63 -0
- package/hooks/hooks.json +26 -0
- package/hooks/setup-symlinks.sh +48 -0
- package/hooks/stop-hook.sh +135 -0
- package/package.json +47 -0
- package/policies/bash.md +71 -0
- package/policies/python.md +71 -0
- package/policies/testing.md +61 -0
- package/policies/universal.md +60 -0
- package/scripts/analyze-report.sh +97 -0
- package/scripts/architecture-map.sh +145 -0
- package/scripts/auto-compound.sh +273 -0
- package/scripts/batch-audit.sh +42 -0
- package/scripts/batch-test.sh +101 -0
- package/scripts/entropy-audit.sh +221 -0
- package/scripts/failure-digest.sh +51 -0
- package/scripts/generate-ast-rules.sh +96 -0
- package/scripts/init.sh +112 -0
- package/scripts/lesson-check.sh +428 -0
- package/scripts/lib/common.sh +61 -0
- package/scripts/lib/cost-tracking.sh +153 -0
- package/scripts/lib/ollama.sh +60 -0
- package/scripts/lib/progress-writer.sh +128 -0
- package/scripts/lib/run-plan-context.sh +215 -0
- package/scripts/lib/run-plan-echo-back.sh +231 -0
- package/scripts/lib/run-plan-headless.sh +396 -0
- package/scripts/lib/run-plan-notify.sh +57 -0
- package/scripts/lib/run-plan-parser.sh +81 -0
- package/scripts/lib/run-plan-prompt.sh +215 -0
- package/scripts/lib/run-plan-quality-gate.sh +132 -0
- package/scripts/lib/run-plan-routing.sh +315 -0
- package/scripts/lib/run-plan-sampling.sh +170 -0
- package/scripts/lib/run-plan-scoring.sh +146 -0
- package/scripts/lib/run-plan-state.sh +142 -0
- package/scripts/lib/run-plan-team.sh +199 -0
- package/scripts/lib/telegram.sh +54 -0
- package/scripts/lib/thompson-sampling.sh +176 -0
- package/scripts/license-check.sh +74 -0
- package/scripts/mab-run.sh +575 -0
- package/scripts/module-size-check.sh +146 -0
- package/scripts/patterns/async-no-await.yml +5 -0
- package/scripts/patterns/bare-except.yml +6 -0
- package/scripts/patterns/empty-catch.yml +6 -0
- package/scripts/patterns/hardcoded-localhost.yml +9 -0
- package/scripts/patterns/retry-loop-no-backoff.yml +12 -0
- package/scripts/pipeline-status.sh +197 -0
- package/scripts/policy-check.sh +226 -0
- package/scripts/prior-art-search.sh +133 -0
- package/scripts/promote-mab-lessons.sh +126 -0
- package/scripts/prompts/agent-a-superpowers.md +29 -0
- package/scripts/prompts/agent-b-ralph.md +29 -0
- package/scripts/prompts/judge-agent.md +61 -0
- package/scripts/prompts/planner-agent.md +44 -0
- package/scripts/pull-community-lessons.sh +90 -0
- package/scripts/quality-gate.sh +266 -0
- package/scripts/research-gate.sh +90 -0
- package/scripts/run-plan.sh +329 -0
- package/scripts/scope-infer.sh +159 -0
- package/scripts/setup-ralph-loop.sh +155 -0
- package/scripts/telemetry.sh +230 -0
- package/scripts/tests/run-all-tests.sh +52 -0
- package/scripts/tests/test-act-cli.sh +46 -0
- package/scripts/tests/test-agents-md.sh +87 -0
- package/scripts/tests/test-analyze-report.sh +114 -0
- package/scripts/tests/test-architecture-map.sh +89 -0
- package/scripts/tests/test-auto-compound.sh +169 -0
- package/scripts/tests/test-batch-test.sh +65 -0
- package/scripts/tests/test-benchmark-runner.sh +25 -0
- package/scripts/tests/test-common.sh +168 -0
- package/scripts/tests/test-cost-tracking.sh +158 -0
- package/scripts/tests/test-echo-back.sh +180 -0
- package/scripts/tests/test-entropy-audit.sh +146 -0
- package/scripts/tests/test-failure-digest.sh +66 -0
- package/scripts/tests/test-generate-ast-rules.sh +145 -0
- package/scripts/tests/test-helpers.sh +82 -0
- package/scripts/tests/test-init.sh +47 -0
- package/scripts/tests/test-lesson-check.sh +278 -0
- package/scripts/tests/test-lesson-local.sh +55 -0
- package/scripts/tests/test-license-check.sh +109 -0
- package/scripts/tests/test-mab-run.sh +182 -0
- package/scripts/tests/test-ollama-lib.sh +49 -0
- package/scripts/tests/test-ollama.sh +60 -0
- package/scripts/tests/test-pipeline-status.sh +198 -0
- package/scripts/tests/test-policy-check.sh +124 -0
- package/scripts/tests/test-prior-art-search.sh +96 -0
- package/scripts/tests/test-progress-writer.sh +140 -0
- package/scripts/tests/test-promote-mab-lessons.sh +110 -0
- package/scripts/tests/test-pull-community-lessons.sh +149 -0
- package/scripts/tests/test-quality-gate.sh +241 -0
- package/scripts/tests/test-research-gate.sh +132 -0
- package/scripts/tests/test-run-plan-cli.sh +86 -0
- package/scripts/tests/test-run-plan-context.sh +305 -0
- package/scripts/tests/test-run-plan-e2e.sh +153 -0
- package/scripts/tests/test-run-plan-headless.sh +424 -0
- package/scripts/tests/test-run-plan-notify.sh +124 -0
- package/scripts/tests/test-run-plan-parser.sh +217 -0
- package/scripts/tests/test-run-plan-prompt.sh +254 -0
- package/scripts/tests/test-run-plan-quality-gate.sh +222 -0
- package/scripts/tests/test-run-plan-routing.sh +178 -0
- package/scripts/tests/test-run-plan-scoring.sh +148 -0
- package/scripts/tests/test-run-plan-state.sh +261 -0
- package/scripts/tests/test-run-plan-team.sh +157 -0
- package/scripts/tests/test-scope-infer.sh +150 -0
- package/scripts/tests/test-setup-ralph-loop.sh +63 -0
- package/scripts/tests/test-telegram-env.sh +38 -0
- package/scripts/tests/test-telegram.sh +121 -0
- package/scripts/tests/test-telemetry.sh +46 -0
- package/scripts/tests/test-thompson-sampling.sh +139 -0
- package/scripts/tests/test-validate-all.sh +60 -0
- package/scripts/tests/test-validate-commands.sh +89 -0
- package/scripts/tests/test-validate-hooks.sh +98 -0
- package/scripts/tests/test-validate-lessons.sh +150 -0
- package/scripts/tests/test-validate-plan-quality.sh +235 -0
- package/scripts/tests/test-validate-plans.sh +187 -0
- package/scripts/tests/test-validate-plugin.sh +106 -0
- package/scripts/tests/test-validate-prd.sh +184 -0
- package/scripts/tests/test-validate-skills.sh +134 -0
- package/scripts/validate-all.sh +57 -0
- package/scripts/validate-commands.sh +67 -0
- package/scripts/validate-hooks.sh +89 -0
- package/scripts/validate-lessons.sh +98 -0
- package/scripts/validate-plan-quality.sh +369 -0
- package/scripts/validate-plans.sh +120 -0
- package/scripts/validate-plugin.sh +86 -0
- package/scripts/validate-policies.sh +42 -0
- package/scripts/validate-prd.sh +118 -0
- package/scripts/validate-skills.sh +96 -0
- package/skills/autocode/SKILL.md +285 -0
- package/skills/autocode/ab-verification.md +51 -0
- package/skills/autocode/code-quality-standards.md +37 -0
- package/skills/autocode/competitive-mode.md +364 -0
- package/skills/brainstorming/SKILL.md +97 -0
- package/skills/capture-lesson/SKILL.md +187 -0
- package/skills/check-lessons/SKILL.md +116 -0
- package/skills/dispatching-parallel-agents/SKILL.md +110 -0
- package/skills/executing-plans/SKILL.md +85 -0
- package/skills/finishing-a-development-branch/SKILL.md +201 -0
- package/skills/receiving-code-review/SKILL.md +72 -0
- package/skills/requesting-code-review/SKILL.md +59 -0
- package/skills/requesting-code-review/code-reviewer.md +82 -0
- package/skills/research/SKILL.md +145 -0
- package/skills/roadmap/SKILL.md +115 -0
- package/skills/subagent-driven-development/SKILL.md +98 -0
- package/skills/subagent-driven-development/code-quality-reviewer-prompt.md +18 -0
- package/skills/subagent-driven-development/implementer-prompt.md +73 -0
- package/skills/subagent-driven-development/spec-reviewer-prompt.md +57 -0
- package/skills/systematic-debugging/SKILL.md +134 -0
- package/skills/systematic-debugging/condition-based-waiting.md +64 -0
- package/skills/systematic-debugging/defense-in-depth.md +32 -0
- package/skills/systematic-debugging/root-cause-tracing.md +55 -0
- package/skills/test-driven-development/SKILL.md +167 -0
- package/skills/using-git-worktrees/SKILL.md +219 -0
- package/skills/using-superpowers/SKILL.md +54 -0
- package/skills/verification-before-completion/SKILL.md +140 -0
- package/skills/verify/SKILL.md +82 -0
- package/skills/writing-plans/SKILL.md +128 -0
- package/skills/writing-skills/SKILL.md +93 -0
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Test plan parser functions
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
|
|
5
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
6
|
+
source "$SCRIPT_DIR/../lib/run-plan-parser.sh"
|
|
7
|
+
|
|
8
|
+
FAILURES=0
|
|
9
|
+
TESTS=0
|
|
10
|
+
|
|
11
|
+
assert_eq() {
|
|
12
|
+
local desc="$1" expected="$2" actual="$3"
|
|
13
|
+
TESTS=$((TESTS + 1))
|
|
14
|
+
if [[ "$expected" != "$actual" ]]; then
|
|
15
|
+
echo "FAIL: $desc"
|
|
16
|
+
echo " expected: $expected"
|
|
17
|
+
echo " actual: $actual"
|
|
18
|
+
FAILURES=$((FAILURES + 1))
|
|
19
|
+
else
|
|
20
|
+
echo "PASS: $desc"
|
|
21
|
+
fi
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
assert_contains() {
|
|
25
|
+
local desc="$1" needle="$2" haystack="$3"
|
|
26
|
+
TESTS=$((TESTS + 1))
|
|
27
|
+
if [[ "$haystack" != *"$needle"* ]]; then
|
|
28
|
+
echo "FAIL: $desc"
|
|
29
|
+
echo " expected to contain: $needle"
|
|
30
|
+
echo " in: ${haystack:0:200}..."
|
|
31
|
+
FAILURES=$((FAILURES + 1))
|
|
32
|
+
else
|
|
33
|
+
echo "PASS: $desc"
|
|
34
|
+
fi
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
# --- Create test fixture ---
|
|
38
|
+
WORK=$(mktemp -d)
|
|
39
|
+
trap 'rm -rf "$WORK"' EXIT
|
|
40
|
+
FIXTURE="$WORK/fixture.md"
|
|
41
|
+
cat > "$FIXTURE" << 'EOF'
|
|
42
|
+
# Feature X Implementation Plan
|
|
43
|
+
|
|
44
|
+
**Goal:** Build feature X
|
|
45
|
+
|
|
46
|
+
**Tech Stack:** Python, pytest
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## Batch 1: Foundation (Tasks 1-2)
|
|
51
|
+
|
|
52
|
+
### Task 1: Create Data Model
|
|
53
|
+
|
|
54
|
+
**Files:**
|
|
55
|
+
- Create: `src/models.py`
|
|
56
|
+
- Test: `tests/test_models.py`
|
|
57
|
+
|
|
58
|
+
**Step 1: Write the failing test**
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
def test_model():
|
|
62
|
+
m = Model("test")
|
|
63
|
+
assert m.name == "test"
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
**Step 2: Implement**
|
|
67
|
+
|
|
68
|
+
Create the Model class.
|
|
69
|
+
|
|
70
|
+
### Task 2: Add Validation
|
|
71
|
+
|
|
72
|
+
**Files:**
|
|
73
|
+
- Modify: `src/models.py`
|
|
74
|
+
|
|
75
|
+
Add validation to Model.
|
|
76
|
+
|
|
77
|
+
## Batch 2: Integration (Tasks 3-4)
|
|
78
|
+
|
|
79
|
+
### Task 3: Wire Together
|
|
80
|
+
|
|
81
|
+
Wire the models into the API.
|
|
82
|
+
|
|
83
|
+
### Task 4: End-to-End Test
|
|
84
|
+
|
|
85
|
+
Write integration test.
|
|
86
|
+
|
|
87
|
+
## Batch 3: Dashboard ⚠ CRITICAL
|
|
88
|
+
|
|
89
|
+
### Task 5: UI Components
|
|
90
|
+
|
|
91
|
+
Build the dashboard.
|
|
92
|
+
EOF
|
|
93
|
+
|
|
94
|
+
# --- Test: count_batches ---
|
|
95
|
+
count=$(count_batches "$FIXTURE")
|
|
96
|
+
assert_eq "count_batches returns 3" "3" "$count"
|
|
97
|
+
|
|
98
|
+
# --- Test: get_batch_title ---
|
|
99
|
+
title=$(get_batch_title "$FIXTURE" 1)
|
|
100
|
+
assert_eq "batch 1 title" "Foundation (Tasks 1-2)" "$title"
|
|
101
|
+
|
|
102
|
+
title=$(get_batch_title "$FIXTURE" 2)
|
|
103
|
+
assert_eq "batch 2 title" "Integration (Tasks 3-4)" "$title"
|
|
104
|
+
|
|
105
|
+
title=$(get_batch_title "$FIXTURE" 3)
|
|
106
|
+
assert_eq "batch 3 title" "Dashboard ⚠ CRITICAL" "$title"
|
|
107
|
+
|
|
108
|
+
# --- Test: get_batch_text ---
|
|
109
|
+
text=$(get_batch_text "$FIXTURE" 1)
|
|
110
|
+
assert_contains "batch 1 has Task 1" "Task 1: Create Data Model" "$text"
|
|
111
|
+
assert_contains "batch 1 has Task 2" "Task 2: Add Validation" "$text"
|
|
112
|
+
assert_contains "batch 1 has code" "def test_model" "$text"
|
|
113
|
+
|
|
114
|
+
text2=$(get_batch_text "$FIXTURE" 2)
|
|
115
|
+
assert_contains "batch 2 has Task 3" "Task 3: Wire Together" "$text2"
|
|
116
|
+
assert_contains "batch 2 has Task 4" "Task 4: End-to-End Test" "$text2"
|
|
117
|
+
|
|
118
|
+
# Batch 2 should NOT contain batch 1 content
|
|
119
|
+
TESTS=$((TESTS + 1))
|
|
120
|
+
if [[ "$text2" == *"Create Data Model"* ]]; then
|
|
121
|
+
echo "FAIL: batch 2 text should not contain batch 1 content"
|
|
122
|
+
FAILURES=$((FAILURES + 1))
|
|
123
|
+
else
|
|
124
|
+
echo "PASS: batch 2 text does not leak batch 1"
|
|
125
|
+
fi
|
|
126
|
+
|
|
127
|
+
# Batch 1 should NOT contain batch 2 content
|
|
128
|
+
TESTS=$((TESTS + 1))
|
|
129
|
+
if [[ "$text" == *"Wire Together"* ]]; then
|
|
130
|
+
echo "FAIL: batch 1 text should not contain batch 2 content"
|
|
131
|
+
FAILURES=$((FAILURES + 1))
|
|
132
|
+
else
|
|
133
|
+
echo "PASS: batch 1 text does not leak batch 2"
|
|
134
|
+
fi
|
|
135
|
+
|
|
136
|
+
# --- Test: get_batch_task_count ---
|
|
137
|
+
tc=$(get_batch_task_count "$FIXTURE" 1)
|
|
138
|
+
assert_eq "batch 1 has 2 tasks" "2" "$tc"
|
|
139
|
+
|
|
140
|
+
tc2=$(get_batch_task_count "$FIXTURE" 2)
|
|
141
|
+
assert_eq "batch 2 has 2 tasks" "2" "$tc2"
|
|
142
|
+
|
|
143
|
+
tc3=$(get_batch_task_count "$FIXTURE" 3)
|
|
144
|
+
assert_eq "batch 3 has 1 task" "1" "$tc3"
|
|
145
|
+
|
|
146
|
+
# --- Test: is_critical_batch ---
|
|
147
|
+
TESTS=$((TESTS + 1))
|
|
148
|
+
if is_critical_batch "$FIXTURE" 3; then
|
|
149
|
+
echo "PASS: batch 3 is critical"
|
|
150
|
+
else
|
|
151
|
+
echo "FAIL: batch 3 should be critical"
|
|
152
|
+
FAILURES=$((FAILURES + 1))
|
|
153
|
+
fi
|
|
154
|
+
|
|
155
|
+
TESTS=$((TESTS + 1))
|
|
156
|
+
if is_critical_batch "$FIXTURE" 1; then
|
|
157
|
+
echo "FAIL: batch 1 should not be critical"
|
|
158
|
+
FAILURES=$((FAILURES + 1))
|
|
159
|
+
else
|
|
160
|
+
echo "PASS: batch 1 is not critical"
|
|
161
|
+
fi
|
|
162
|
+
|
|
163
|
+
TESTS=$((TESTS + 1))
|
|
164
|
+
if is_critical_batch "$FIXTURE" 2; then
|
|
165
|
+
echo "FAIL: batch 2 should not be critical"
|
|
166
|
+
FAILURES=$((FAILURES + 1))
|
|
167
|
+
else
|
|
168
|
+
echo "PASS: batch 2 is not critical"
|
|
169
|
+
fi
|
|
170
|
+
|
|
171
|
+
# --- Test: nonexistent batch ---
|
|
172
|
+
text_empty=$(get_batch_text "$FIXTURE" 99)
|
|
173
|
+
assert_eq "nonexistent batch returns empty" "" "$text_empty"
|
|
174
|
+
|
|
175
|
+
title_empty=$(get_batch_title "$FIXTURE" 99)
|
|
176
|
+
assert_eq "nonexistent batch title returns empty" "" "$title_empty"
|
|
177
|
+
|
|
178
|
+
tc_empty=$(get_batch_task_count "$FIXTURE" 99)
|
|
179
|
+
assert_eq "nonexistent batch task count returns 0" "0" "$tc_empty"
|
|
180
|
+
|
|
181
|
+
# === get_batch_context_refs tests ===
|
|
182
|
+
|
|
183
|
+
# Create a plan with context_refs
|
|
184
|
+
cat > "$WORK/refs-plan.md" << 'PLAN'
|
|
185
|
+
## Batch 1: Setup
|
|
186
|
+
|
|
187
|
+
### Task 1: Create base
|
|
188
|
+
Content here.
|
|
189
|
+
|
|
190
|
+
## Batch 2: Build on base
|
|
191
|
+
context_refs: src/auth.py, tests/test_auth.py
|
|
192
|
+
|
|
193
|
+
### Task 2: Extend
|
|
194
|
+
Uses auth module from batch 1.
|
|
195
|
+
PLAN
|
|
196
|
+
|
|
197
|
+
# Batch 1 has no refs
|
|
198
|
+
val=$(get_batch_context_refs "$WORK/refs-plan.md" 1)
|
|
199
|
+
assert_eq "get_batch_context_refs: batch 1 has no refs" "" "$val"
|
|
200
|
+
|
|
201
|
+
# Batch 2 has refs
|
|
202
|
+
val=$(get_batch_context_refs "$WORK/refs-plan.md" 2)
|
|
203
|
+
echo "$val" | grep -q "src/auth.py" && echo "PASS: batch 2 refs include src/auth.py" && TESTS=$((TESTS + 1)) || {
|
|
204
|
+
echo "FAIL: batch 2 refs missing src/auth.py"; TESTS=$((TESTS + 1)); FAILURES=$((FAILURES + 1))
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
echo "$val" | grep -q "tests/test_auth.py" && echo "PASS: batch 2 refs include tests/test_auth.py" && TESTS=$((TESTS + 1)) || {
|
|
208
|
+
echo "FAIL: batch 2 refs missing tests/test_auth.py"; TESTS=$((TESTS + 1)); FAILURES=$((FAILURES + 1))
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
echo ""
|
|
212
|
+
echo "Results: $((TESTS - FAILURES))/$TESTS passed"
|
|
213
|
+
if [[ $FAILURES -gt 0 ]]; then
|
|
214
|
+
echo "FAILURES: $FAILURES"
|
|
215
|
+
exit 1
|
|
216
|
+
fi
|
|
217
|
+
echo "ALL PASSED"
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Test prompt builder functions
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
|
|
5
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
6
|
+
source "$SCRIPT_DIR/../lib/run-plan-parser.sh"
|
|
7
|
+
source "$SCRIPT_DIR/../lib/run-plan-prompt.sh"
|
|
8
|
+
|
|
9
|
+
FAILURES=0
|
|
10
|
+
TESTS=0
|
|
11
|
+
|
|
12
|
+
assert_contains() {
|
|
13
|
+
local desc="$1" needle="$2" haystack="$3"
|
|
14
|
+
TESTS=$((TESTS + 1))
|
|
15
|
+
if [[ "$haystack" != *"$needle"* ]]; then
|
|
16
|
+
echo "FAIL: $desc"
|
|
17
|
+
echo " expected to contain: $needle"
|
|
18
|
+
echo " in: ${haystack:0:300}..."
|
|
19
|
+
FAILURES=$((FAILURES + 1))
|
|
20
|
+
else
|
|
21
|
+
echo "PASS: $desc"
|
|
22
|
+
fi
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
assert_not_contains() {
|
|
26
|
+
local desc="$1" needle="$2" haystack="$3"
|
|
27
|
+
TESTS=$((TESTS + 1))
|
|
28
|
+
if [[ "$haystack" == *"$needle"* ]]; then
|
|
29
|
+
echo "FAIL: $desc"
|
|
30
|
+
echo " expected NOT to contain: $needle"
|
|
31
|
+
echo " in: ${haystack:0:300}..."
|
|
32
|
+
FAILURES=$((FAILURES + 1))
|
|
33
|
+
else
|
|
34
|
+
echo "PASS: $desc"
|
|
35
|
+
fi
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
assert_eq() {
|
|
39
|
+
local desc="$1" expected="$2" actual="$3"
|
|
40
|
+
TESTS=$((TESTS + 1))
|
|
41
|
+
if [[ "$actual" != "$expected" ]]; then
|
|
42
|
+
echo "FAIL: $desc"
|
|
43
|
+
echo " expected: $expected"
|
|
44
|
+
echo " actual: $actual"
|
|
45
|
+
FAILURES=$((FAILURES + 1))
|
|
46
|
+
else
|
|
47
|
+
echo "PASS: $desc"
|
|
48
|
+
fi
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
# --- Setup: fixture plan + temp git worktree ---
|
|
52
|
+
TMPDIR_ROOT=$(mktemp -d)
|
|
53
|
+
trap 'rm -rf "$TMPDIR_ROOT"' EXIT
|
|
54
|
+
|
|
55
|
+
FIXTURE="$TMPDIR_ROOT/plan.md"
|
|
56
|
+
cat > "$FIXTURE" << 'EOF'
|
|
57
|
+
# Feature X Implementation Plan
|
|
58
|
+
|
|
59
|
+
**Goal:** Build feature X
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## Batch 1: Foundation (Tasks 1-2)
|
|
64
|
+
|
|
65
|
+
### Task 1: Create Data Model
|
|
66
|
+
|
|
67
|
+
**Files:**
|
|
68
|
+
- Create: `src/models.py`
|
|
69
|
+
- Test: `tests/test_models.py`
|
|
70
|
+
|
|
71
|
+
**Step 1: Write the failing test**
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
def test_model():
|
|
75
|
+
m = Model("test")
|
|
76
|
+
assert m.name == "test"
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
**Step 2: Implement**
|
|
80
|
+
|
|
81
|
+
Create the Model class.
|
|
82
|
+
|
|
83
|
+
### Task 2: Add Validation
|
|
84
|
+
|
|
85
|
+
**Files:**
|
|
86
|
+
- Modify: `src/models.py`
|
|
87
|
+
|
|
88
|
+
Add validation to Model.
|
|
89
|
+
|
|
90
|
+
## Batch 2: Integration (Tasks 3-4)
|
|
91
|
+
|
|
92
|
+
### Task 3: Wire Together
|
|
93
|
+
|
|
94
|
+
Wire the models into the API.
|
|
95
|
+
|
|
96
|
+
### Task 4: End-to-End Test
|
|
97
|
+
|
|
98
|
+
Write integration test.
|
|
99
|
+
EOF
|
|
100
|
+
|
|
101
|
+
# Create a temp git repo so git branch works
|
|
102
|
+
WORKTREE="$TMPDIR_ROOT/worktree"
|
|
103
|
+
mkdir -p "$WORKTREE"
|
|
104
|
+
git -C "$WORKTREE" init -b test-branch --quiet
|
|
105
|
+
git -C "$WORKTREE" config user.email "test@test.com"
|
|
106
|
+
git -C "$WORKTREE" config user.name "Test"
|
|
107
|
+
touch "$WORKTREE/.gitkeep"
|
|
108
|
+
git -C "$WORKTREE" add .gitkeep
|
|
109
|
+
git -C "$WORKTREE" commit -m "init" --quiet
|
|
110
|
+
|
|
111
|
+
# --- Test: build_batch_prompt for batch 1 ---
|
|
112
|
+
prompt=$(build_batch_prompt "$FIXTURE" 1 "$WORKTREE" "/usr/bin/python3" "scripts/quality-gate.sh --project-root ." 0)
|
|
113
|
+
|
|
114
|
+
assert_contains "has batch number" "Batch 1" "$prompt"
|
|
115
|
+
assert_contains "has batch title" "Foundation (Tasks 1-2)" "$prompt"
|
|
116
|
+
assert_contains "has plan file reference" "plan.md" "$prompt"
|
|
117
|
+
assert_contains "has worktree path" "$WORKTREE" "$prompt"
|
|
118
|
+
assert_contains "has python path" "/usr/bin/python3" "$prompt"
|
|
119
|
+
assert_contains "has branch name" "test-branch" "$prompt"
|
|
120
|
+
assert_contains "has task text - Task 1" "Task 1: Create Data Model" "$prompt"
|
|
121
|
+
assert_contains "has task text - Task 2" "Task 2: Add Validation" "$prompt"
|
|
122
|
+
assert_contains "has TDD instruction" "TDD" "$prompt"
|
|
123
|
+
assert_contains "has quality gate command" "scripts/quality-gate.sh --project-root ." "$prompt"
|
|
124
|
+
assert_contains "has previous test count" "0+" "$prompt"
|
|
125
|
+
assert_contains "has progress.txt instruction" "progress.txt" "$prompt"
|
|
126
|
+
|
|
127
|
+
# --- Test: build_batch_prompt for batch 2 ---
|
|
128
|
+
prompt2=$(build_batch_prompt "$FIXTURE" 2 "$WORKTREE" "/opt/python3.12" "make test" 15)
|
|
129
|
+
|
|
130
|
+
assert_contains "batch 2 has batch number" "Batch 2" "$prompt2"
|
|
131
|
+
assert_contains "batch 2 has batch title" "Integration (Tasks 3-4)" "$prompt2"
|
|
132
|
+
assert_contains "batch 2 has task text - Task 3" "Task 3: Wire Together" "$prompt2"
|
|
133
|
+
assert_contains "batch 2 has task text - Task 4" "Task 4: End-to-End Test" "$prompt2"
|
|
134
|
+
assert_contains "batch 2 has different python" "/opt/python3.12" "$prompt2"
|
|
135
|
+
assert_contains "batch 2 has different quality gate" "make test" "$prompt2"
|
|
136
|
+
assert_contains "batch 2 has prev test count" "15+" "$prompt2"
|
|
137
|
+
|
|
138
|
+
# --- Test: batch 2 does NOT contain batch 1 tasks ---
|
|
139
|
+
TESTS=$((TESTS + 1))
|
|
140
|
+
if [[ "$prompt2" == *"Create Data Model"* ]]; then
|
|
141
|
+
echo "FAIL: batch 2 prompt should not contain batch 1 tasks"
|
|
142
|
+
FAILURES=$((FAILURES + 1))
|
|
143
|
+
else
|
|
144
|
+
echo "PASS: batch 2 prompt does not leak batch 1 tasks"
|
|
145
|
+
fi
|
|
146
|
+
|
|
147
|
+
# =============================================================================
|
|
148
|
+
# build_stable_prefix tests
|
|
149
|
+
# =============================================================================
|
|
150
|
+
|
|
151
|
+
stable=$(build_stable_prefix "$FIXTURE" "$WORKTREE" "/usr/bin/python3" "scripts/quality-gate.sh")
|
|
152
|
+
|
|
153
|
+
assert_contains "stable prefix has worktree path" "$WORKTREE" "$stable"
|
|
154
|
+
assert_contains "stable prefix has python path" "/usr/bin/python3" "$stable"
|
|
155
|
+
assert_contains "stable prefix has branch name" "test-branch" "$stable"
|
|
156
|
+
assert_contains "stable prefix has TDD rule" "TDD" "$stable"
|
|
157
|
+
assert_contains "stable prefix has quality gate" "scripts/quality-gate.sh" "$stable"
|
|
158
|
+
assert_contains "stable prefix has progress.txt rule" "progress.txt" "$stable"
|
|
159
|
+
|
|
160
|
+
# #48: stable prefix must NOT contain prev_test_count — that belongs in variable suffix
|
|
161
|
+
assert_not_contains "stable prefix does NOT contain test count line" "tests must pass" "$stable"
|
|
162
|
+
|
|
163
|
+
# --- Test: build_stable_prefix with bad worktree emits warning but returns 'unknown' ---
|
|
164
|
+
TESTS=$((TESTS + 1))
|
|
165
|
+
bad_worktree_output=$(build_stable_prefix "$FIXTURE" "/nonexistent/path/xyz" "python3" "gate.sh" 2>&1)
|
|
166
|
+
if [[ "$bad_worktree_output" == *"WARNING"* && "$bad_worktree_output" == *"unknown"* ]]; then
|
|
167
|
+
echo "PASS: build_stable_prefix warns on missing worktree and uses 'unknown' branch"
|
|
168
|
+
else
|
|
169
|
+
echo "FAIL: build_stable_prefix should warn on missing worktree"
|
|
170
|
+
echo " output: $bad_worktree_output"
|
|
171
|
+
FAILURES=$((FAILURES + 1))
|
|
172
|
+
fi
|
|
173
|
+
|
|
174
|
+
# =============================================================================
|
|
175
|
+
# build_variable_suffix tests
|
|
176
|
+
# =============================================================================
|
|
177
|
+
|
|
178
|
+
suffix=$(build_variable_suffix "$FIXTURE" 1 "$WORKTREE" 7)
|
|
179
|
+
|
|
180
|
+
assert_contains "variable suffix has batch number" "Batch 1" "$suffix"
|
|
181
|
+
assert_contains "variable suffix has batch title" "Foundation" "$suffix"
|
|
182
|
+
assert_contains "variable suffix has task text" "Task 1: Create Data Model" "$suffix"
|
|
183
|
+
# #48: test count is in the variable suffix, not the stable prefix
|
|
184
|
+
assert_contains "variable suffix has test count" "7+" "$suffix"
|
|
185
|
+
|
|
186
|
+
# Different test count gives different suffix (confirms test count varies per batch)
|
|
187
|
+
suffix_b2=$(build_variable_suffix "$FIXTURE" 2 "$WORKTREE" 20)
|
|
188
|
+
assert_contains "variable suffix b2 has test count 20+" "20+" "$suffix_b2"
|
|
189
|
+
|
|
190
|
+
# =============================================================================
|
|
191
|
+
# Cross-batch context tests
|
|
192
|
+
# =============================================================================
|
|
193
|
+
|
|
194
|
+
# --- Setup: add progress.txt and a commit to the worktree ---
|
|
195
|
+
echo "Batch 1: Implemented auth module" > "$WORKTREE/progress.txt"
|
|
196
|
+
echo "code" > "$WORKTREE/code.py"
|
|
197
|
+
git -C "$WORKTREE" add code.py progress.txt
|
|
198
|
+
git -C "$WORKTREE" commit -q -m "feat: add auth"
|
|
199
|
+
|
|
200
|
+
# --- Test: prompt includes recent commits ---
|
|
201
|
+
prompt3=$(build_batch_prompt "$FIXTURE" 2 "$WORKTREE" "python3" "scripts/quality-gate.sh" 42)
|
|
202
|
+
assert_contains "cross-batch: has Recent commits" "Recent commits" "$prompt3"
|
|
203
|
+
|
|
204
|
+
# --- Test: prompt includes progress.txt content ---
|
|
205
|
+
assert_contains "cross-batch: has Previous progress" "Previous progress" "$prompt3"
|
|
206
|
+
assert_contains "cross-batch: has progress content" "Implemented auth module" "$prompt3"
|
|
207
|
+
|
|
208
|
+
# --- Test: prompt includes commit message ---
|
|
209
|
+
assert_contains "cross-batch: has commit in log" "feat: add auth" "$prompt3"
|
|
210
|
+
|
|
211
|
+
# =============================================================================
|
|
212
|
+
# #47: corrupted state file — jq failure emits warning, prev_gate stays empty
|
|
213
|
+
# =============================================================================
|
|
214
|
+
|
|
215
|
+
echo "NOT VALID JSON {{{" > "$WORKTREE/.run-plan-state.json"
|
|
216
|
+
TESTS=$((TESTS + 1))
|
|
217
|
+
jq_warn_output=$(build_variable_suffix "$FIXTURE" 2 "$WORKTREE" 0 2>&1)
|
|
218
|
+
if [[ "$jq_warn_output" == *"WARNING"* && "$jq_warn_output" == *"corrupted"* ]]; then
|
|
219
|
+
echo "PASS: build_variable_suffix warns on corrupted state file"
|
|
220
|
+
else
|
|
221
|
+
echo "FAIL: build_variable_suffix should warn on corrupted state file"
|
|
222
|
+
echo " output: ${jq_warn_output:0:200}"
|
|
223
|
+
FAILURES=$((FAILURES + 1))
|
|
224
|
+
fi
|
|
225
|
+
# Clean up corrupted state file
|
|
226
|
+
rm -f "$WORKTREE/.run-plan-state.json"
|
|
227
|
+
|
|
228
|
+
# =============================================================================
|
|
229
|
+
# #50: unreadable progress.txt — error is NOT silently swallowed (no 2>/dev/null || true)
|
|
230
|
+
# The fix removes the error suppression so stderr shows the permission denial.
|
|
231
|
+
# We verify by running tail directly under the same condition — the error must be visible.
|
|
232
|
+
# (Command substitution $() cannot propagate exit codes from within sourced functions
|
|
233
|
+
# reliably across bash versions, so we test the absence of suppression via the source.)
|
|
234
|
+
# =============================================================================
|
|
235
|
+
|
|
236
|
+
TESTS=$((TESTS + 1))
|
|
237
|
+
# Check that progress_tail assignment in build_variable_suffix does NOT use || true
|
|
238
|
+
# by inspecting the source code — the fix is the removal of the error suppression.
|
|
239
|
+
PROMPT_SRC="$SCRIPT_DIR/../lib/run-plan-prompt.sh"
|
|
240
|
+
if grep -q 'tail.*progress\.txt.*|| true' "$PROMPT_SRC" 2>/dev/null || \
|
|
241
|
+
grep -q 'tail.*progress\.txt.*2>/dev/null.*|| true' "$PROMPT_SRC" 2>/dev/null; then
|
|
242
|
+
echo "FAIL: build_variable_suffix still has suppressed tail error (|| true present)"
|
|
243
|
+
FAILURES=$((FAILURES + 1))
|
|
244
|
+
else
|
|
245
|
+
echo "PASS: build_variable_suffix does not suppress tail errors on progress.txt"
|
|
246
|
+
fi
|
|
247
|
+
|
|
248
|
+
echo ""
|
|
249
|
+
echo "Results: $((TESTS - FAILURES))/$TESTS passed"
|
|
250
|
+
if [[ $FAILURES -gt 0 ]]; then
|
|
251
|
+
echo "FAILURES: $FAILURES"
|
|
252
|
+
exit 1
|
|
253
|
+
fi
|
|
254
|
+
echo "ALL PASSED"
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Test quality gate runner helper functions
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
|
|
5
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
6
|
+
source "$SCRIPT_DIR/../lib/run-plan-quality-gate.sh"
|
|
7
|
+
|
|
8
|
+
FAILURES=0
|
|
9
|
+
TESTS=0
|
|
10
|
+
|
|
11
|
+
assert_eq() {
|
|
12
|
+
local desc="$1" expected="$2" actual="$3"
|
|
13
|
+
TESTS=$((TESTS + 1))
|
|
14
|
+
if [[ "$expected" != "$actual" ]]; then
|
|
15
|
+
echo "FAIL: $desc"
|
|
16
|
+
echo " expected: $expected"
|
|
17
|
+
echo " actual: $actual"
|
|
18
|
+
FAILURES=$((FAILURES + 1))
|
|
19
|
+
else
|
|
20
|
+
echo "PASS: $desc"
|
|
21
|
+
fi
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
assert_exit() {
|
|
25
|
+
local desc="$1" expected_exit="$2"
|
|
26
|
+
shift 2
|
|
27
|
+
local actual_exit=0
|
|
28
|
+
"$@" || actual_exit=$?
|
|
29
|
+
TESTS=$((TESTS + 1))
|
|
30
|
+
if [[ "$expected_exit" != "$actual_exit" ]]; then
|
|
31
|
+
echo "FAIL: $desc"
|
|
32
|
+
echo " expected exit: $expected_exit"
|
|
33
|
+
echo " actual exit: $actual_exit"
|
|
34
|
+
FAILURES=$((FAILURES + 1))
|
|
35
|
+
else
|
|
36
|
+
echo "PASS: $desc"
|
|
37
|
+
fi
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
# --- Temp dir for git repo simulation ---
|
|
41
|
+
WORK=$(mktemp -d)
|
|
42
|
+
trap 'rm -rf "$WORK"' EXIT
|
|
43
|
+
|
|
44
|
+
# =============================================================================
|
|
45
|
+
# extract_test_count tests
|
|
46
|
+
# =============================================================================
|
|
47
|
+
|
|
48
|
+
# --- Test: extract passed count with skipped ---
|
|
49
|
+
val=$(extract_test_count "1953 passed, 15 skipped in 45.2s")
|
|
50
|
+
assert_eq "extract_test_count: passed with skipped" "1953" "$val"
|
|
51
|
+
|
|
52
|
+
# --- Test: extract passed count without skipped ---
|
|
53
|
+
val=$(extract_test_count "42 passed in 2.1s")
|
|
54
|
+
assert_eq "extract_test_count: passed without skipped" "42" "$val"
|
|
55
|
+
|
|
56
|
+
# --- Test: extract from no tests ran ---
|
|
57
|
+
val=$(extract_test_count "ERROR: no tests ran")
|
|
58
|
+
assert_eq "extract_test_count: no tests ran" "-1" "$val"
|
|
59
|
+
|
|
60
|
+
# --- Test: extract from empty string ---
|
|
61
|
+
val=$(extract_test_count "")
|
|
62
|
+
assert_eq "extract_test_count: empty string" "-1" "$val"
|
|
63
|
+
|
|
64
|
+
# --- Test: extract from multi-line pytest output ---
|
|
65
|
+
output="============================= test session starts ==============================
|
|
66
|
+
collected 87 items
|
|
67
|
+
|
|
68
|
+
tests/test_foo.py ........ [ 9%]
|
|
69
|
+
tests/test_bar.py ........ [ 18%]
|
|
70
|
+
|
|
71
|
+
============================== 87 passed in 12.34s ==============================="
|
|
72
|
+
val=$(extract_test_count "$output")
|
|
73
|
+
assert_eq "extract_test_count: full pytest output" "87" "$val"
|
|
74
|
+
|
|
75
|
+
# --- Test: extract with failures in output ---
|
|
76
|
+
output="3 failed, 85 passed, 2 skipped in 30.1s"
|
|
77
|
+
val=$(extract_test_count "$output")
|
|
78
|
+
assert_eq "extract_test_count: with failures" "85" "$val"
|
|
79
|
+
|
|
80
|
+
# --- Test: extract from jest output ---
|
|
81
|
+
output="Tests: 3 failed, 45 passed, 48 total"
|
|
82
|
+
val=$(extract_test_count "$output")
|
|
83
|
+
assert_eq "extract_test_count: jest output" "45" "$val"
|
|
84
|
+
|
|
85
|
+
# --- Test: extract from jest all-pass output ---
|
|
86
|
+
output="Tests: 12 passed, 12 total"
|
|
87
|
+
val=$(extract_test_count "$output")
|
|
88
|
+
assert_eq "extract_test_count: jest all-pass" "12" "$val"
|
|
89
|
+
|
|
90
|
+
# --- Test: extract from go test output ---
|
|
91
|
+
output="ok github.com/foo/bar 0.123s
|
|
92
|
+
ok github.com/foo/baz 0.456s
|
|
93
|
+
FAIL github.com/foo/qux 0.789s"
|
|
94
|
+
val=$(extract_test_count "$output")
|
|
95
|
+
assert_eq "extract_test_count: go test (2 ok of 3)" "2" "$val"
|
|
96
|
+
|
|
97
|
+
# --- Test: unrecognized format returns -1 ---
|
|
98
|
+
val=$(extract_test_count "Some random build output with no test results")
|
|
99
|
+
assert_eq "extract_test_count: unrecognized format" "-1" "$val"
|
|
100
|
+
|
|
101
|
+
# =============================================================================
|
|
102
|
+
# check_test_count_regression tests
|
|
103
|
+
# =============================================================================
|
|
104
|
+
|
|
105
|
+
# --- Test: no regression (increase) ---
|
|
106
|
+
assert_exit "check_test_count_regression: 200 >= 150 passes" 0 \
|
|
107
|
+
check_test_count_regression 200 150
|
|
108
|
+
|
|
109
|
+
# --- Test: no regression (equal) ---
|
|
110
|
+
assert_exit "check_test_count_regression: 150 >= 150 passes" 0 \
|
|
111
|
+
check_test_count_regression 150 150
|
|
112
|
+
|
|
113
|
+
# --- Test: regression detected ---
|
|
114
|
+
assert_exit "check_test_count_regression: 100 < 150 fails" 1 \
|
|
115
|
+
check_test_count_regression 100 150
|
|
116
|
+
|
|
117
|
+
# --- Test: no regression from zero baseline ---
|
|
118
|
+
assert_exit "check_test_count_regression: 50 >= 0 passes" 0 \
|
|
119
|
+
check_test_count_regression 50 0
|
|
120
|
+
|
|
121
|
+
# =============================================================================
|
|
122
|
+
# check_git_clean tests
|
|
123
|
+
# =============================================================================
|
|
124
|
+
|
|
125
|
+
# --- Setup: create a temp git repo ---
|
|
126
|
+
git -C "$WORK" init -q
|
|
127
|
+
git -C "$WORK" config user.email "test@test.com"
|
|
128
|
+
git -C "$WORK" config user.name "Test"
|
|
129
|
+
echo "initial" > "$WORK/file.txt"
|
|
130
|
+
git -C "$WORK" add file.txt
|
|
131
|
+
git -C "$WORK" commit -q -m "initial"
|
|
132
|
+
|
|
133
|
+
# --- Test: clean repo passes ---
|
|
134
|
+
assert_exit "check_git_clean: clean repo passes" 0 \
|
|
135
|
+
check_git_clean "$WORK"
|
|
136
|
+
|
|
137
|
+
# --- Test: dirty repo (untracked file) fails ---
|
|
138
|
+
echo "dirty" > "$WORK/untracked.txt"
|
|
139
|
+
assert_exit "check_git_clean: untracked file fails" 1 \
|
|
140
|
+
check_git_clean "$WORK"
|
|
141
|
+
|
|
142
|
+
# --- Test: dirty repo (modified file) fails ---
|
|
143
|
+
rm "$WORK/untracked.txt"
|
|
144
|
+
echo "modified" >> "$WORK/file.txt"
|
|
145
|
+
assert_exit "check_git_clean: modified file fails" 1 \
|
|
146
|
+
check_git_clean "$WORK"
|
|
147
|
+
|
|
148
|
+
# --- Test: -1 skips regression check ---
|
|
149
|
+
assert_exit "check_test_count_regression: -1 new skips check" 0 \
|
|
150
|
+
check_test_count_regression -1 150
|
|
151
|
+
|
|
152
|
+
assert_exit "check_test_count_regression: -1 previous skips check" 0 \
|
|
153
|
+
check_test_count_regression 50 -1
|
|
154
|
+
|
|
155
|
+
# Clean up for any subsequent tests
|
|
156
|
+
git -C "$WORK" checkout -- file.txt
|
|
157
|
+
|
|
158
|
+
# =============================================================================
|
|
159
|
+
# Security: no eval in quality gate runner (#3)
|
|
160
|
+
# =============================================================================
|
|
161
|
+
|
|
162
|
+
TESTS=$((TESTS + 1))
|
|
163
|
+
QG_FILE="$SCRIPT_DIR/../lib/run-plan-quality-gate.sh"
|
|
164
|
+
if grep -q 'eval.*quality_gate_cmd\|eval.*\$quality_gate' "$QG_FILE"; then
|
|
165
|
+
echo "FAIL: run-plan-quality-gate.sh uses eval on quality_gate_cmd (command injection risk, bug #3)"
|
|
166
|
+
FAILURES=$((FAILURES + 1))
|
|
167
|
+
else
|
|
168
|
+
echo "PASS: run-plan-quality-gate.sh does not use eval on quality_gate_cmd"
|
|
169
|
+
fi
|
|
170
|
+
|
|
171
|
+
TESTS=$((TESTS + 1))
|
|
172
|
+
if grep -q 'bash -c.*quality_gate_cmd\|bash -c.*\$quality_gate' "$QG_FILE"; then
|
|
173
|
+
echo "PASS: run-plan-quality-gate.sh uses bash -c instead of eval"
|
|
174
|
+
else
|
|
175
|
+
echo "FAIL: run-plan-quality-gate.sh should use bash -c instead of eval (bug #3)"
|
|
176
|
+
FAILURES=$((FAILURES + 1))
|
|
177
|
+
fi
|
|
178
|
+
|
|
179
|
+
# =============================================================================
|
|
180
|
+
# Bug #3 BEHAVIORAL: bash -c limits injection scope vs eval
|
|
181
|
+
# =============================================================================
|
|
182
|
+
# eval "$cmd" would expand variables/globs in the current shell context.
|
|
183
|
+
# bash -c "$cmd" runs in a fresh subshell — variable references from the
|
|
184
|
+
# parent shell don't leak into the command execution.
|
|
185
|
+
|
|
186
|
+
# Create a gate command that tries to reference a parent shell variable
|
|
187
|
+
INJECT_WORK=$(mktemp -d)
|
|
188
|
+
git -C "$INJECT_WORK" init -q
|
|
189
|
+
git -C "$INJECT_WORK" config user.email "test@test.com"
|
|
190
|
+
git -C "$INJECT_WORK" config user.name "Test"
|
|
191
|
+
echo "init" > "$INJECT_WORK/file.txt"
|
|
192
|
+
git -C "$INJECT_WORK" add file.txt
|
|
193
|
+
git -C "$INJECT_WORK" commit -q -m "init"
|
|
194
|
+
|
|
195
|
+
# Set a variable in the current shell that bash -c should NOT see
|
|
196
|
+
_GATE_SECRET="LEAKED"
|
|
197
|
+
|
|
198
|
+
# A gate command that tries to echo the secret — with bash -c it gets empty string
|
|
199
|
+
gate_output=$(cd "$INJECT_WORK" && bash -c 'echo "secret=${_GATE_SECRET:-none}"' 2>&1) || true
|
|
200
|
+
|
|
201
|
+
TESTS=$((TESTS + 1))
|
|
202
|
+
if [[ "$gate_output" == *"secret=none"* ]]; then
|
|
203
|
+
echo "PASS: bash -c does not leak parent shell variables (injection limited)"
|
|
204
|
+
else
|
|
205
|
+
echo "FAIL: bash -c should not see parent shell variable _GATE_SECRET"
|
|
206
|
+
echo " output: $gate_output"
|
|
207
|
+
FAILURES=$((FAILURES + 1))
|
|
208
|
+
fi
|
|
209
|
+
|
|
210
|
+
rm -rf "$INJECT_WORK"
|
|
211
|
+
|
|
212
|
+
# =============================================================================
|
|
213
|
+
# Summary
|
|
214
|
+
# =============================================================================
|
|
215
|
+
|
|
216
|
+
echo ""
|
|
217
|
+
echo "Results: $((TESTS - FAILURES))/$TESTS passed"
|
|
218
|
+
if [[ $FAILURES -gt 0 ]]; then
|
|
219
|
+
echo "FAILURES: $FAILURES"
|
|
220
|
+
exit 1
|
|
221
|
+
fi
|
|
222
|
+
echo "ALL PASSED"
|