@mmerterden/multi-agent-pipeline 12.7.0 → 12.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +126 -0
- package/install/_common.mjs +48 -0
- package/install/_dev-only-files.mjs +125 -5
- package/install/claude.mjs +14 -8
- package/install/copilot.mjs +5 -8
- package/package.json +17 -2
- package/pipeline/commands/multi-agent/analysis/SKILL.md +1 -1
- package/pipeline/lib/credential-store.sh +20 -0
- package/pipeline/multi-agent-refs/_account-picker.md +1 -1
- package/pipeline/multi-agent-refs/_dev-context.md +1 -1
- package/pipeline/multi-agent-refs/_repo-picker.md +1 -1
- package/pipeline/multi-agent-refs/phases/operations.md +28 -0
- package/pipeline/multi-agent-refs/phases/phase-0-init.md +1 -0
- package/pipeline/multi-agent-refs/phases/phase-2-planning.md +1 -1
- package/pipeline/multi-agent-refs/phases/phase-3-dev.md +1 -2
- package/pipeline/multi-agent-refs/phases/phase-4-review.md +49 -4
- package/pipeline/schemas/prefs.schema.json +6 -0
- package/pipeline/scripts/_smoke-root.sh +61 -0
- package/pipeline/scripts/audit-log.sh +25 -0
- package/pipeline/scripts/smoke-cross-cli-behavior.sh +21 -7
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/expected/phase-1-analysis.json +0 -25
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/expected/phase-2-plan.json +0 -30
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/expected/phase-4-review.json +0 -20
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/expected/phase-4-triage.json +0 -15
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/task.json +0 -12
- package/pipeline/eval/golden-tasks/02-android-feature-compose/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/02-android-feature-compose/expected/phase-2-plan.json +0 -43
- package/pipeline/eval/golden-tasks/02-android-feature-compose/expected/phase-4-review.json +0 -35
- package/pipeline/eval/golden-tasks/02-android-feature-compose/expected/phase-4-triage.json +0 -35
- package/pipeline/eval/golden-tasks/02-android-feature-compose/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/02-android-feature-compose/task.json +0 -12
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/expected/phase-2-plan.json +0 -42
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/expected/phase-4-review.json +0 -20
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/expected/phase-4-triage.json +0 -15
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/task.json +0 -12
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/expected/phase-2-plan.json +0 -40
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/expected/phase-4-review.json +0 -20
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/expected/phase-4-triage.json +0 -15
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/task.json +0 -12
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/expected/phase-2-plan.json +0 -42
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/expected/phase-4-review.json +0 -28
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/expected/phase-4-triage.json +0 -27
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/task.json +0 -12
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/expected/phase-2-plan.json +0 -41
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/expected/phase-4-review.json +0 -12
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/expected/phase-4-triage.json +0 -6
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/task.json +0 -12
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/expected/phase-2-plan.json +0 -42
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/expected/phase-4-review.json +0 -28
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/expected/phase-4-triage.json +0 -27
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/task.json +0 -12
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-1-analysis.json +0 -25
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-2-plan.json +0 -31
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-4-review.json +0 -12
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-4-triage.json +0 -18
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/task.json +0 -12
- package/pipeline/eval/golden-tasks/README.md +0 -65
- package/pipeline/eval/intent-cases.json +0 -40
- package/pipeline/eval/run-metrics-fixture.json +0 -93
- package/pipeline/eval/triage/01-empty-findings/expected.json +0 -6
- package/pipeline/eval/triage/01-empty-findings/input.json +0 -5
- package/pipeline/eval/triage/01-empty-findings/notes.md +0 -7
- package/pipeline/eval/triage/02-real-blocker/expected.json +0 -15
- package/pipeline/eval/triage/02-real-blocker/input.json +0 -14
- package/pipeline/eval/triage/02-real-blocker/notes.md +0 -7
- package/pipeline/eval/triage/03-out-of-scope-defer/expected.json +0 -18
- package/pipeline/eval/triage/03-out-of-scope-defer/input.json +0 -14
- package/pipeline/eval/triage/03-out-of-scope-defer/notes.md +0 -10
- package/pipeline/eval/triage/04-false-positive-reject/expected.json +0 -18
- package/pipeline/eval/triage/04-false-positive-reject/input.json +0 -14
- package/pipeline/eval/triage/04-false-positive-reject/notes.md +0 -10
- package/pipeline/eval/triage/05-mixed-classification/expected.json +0 -43
- package/pipeline/eval/triage/05-mixed-classification/input.json +0 -38
- package/pipeline/eval/triage/05-mixed-classification/notes.md +0 -17
- package/pipeline/eval/triage/06-severity-mismatch/expected.json +0 -15
- package/pipeline/eval/triage/06-severity-mismatch/input.json +0 -14
- package/pipeline/eval/triage/06-severity-mismatch/notes.md +0 -9
- package/pipeline/eval/triage/07-duplicate-reviewers/expected.json +0 -27
- package/pipeline/eval/triage/07-duplicate-reviewers/input.json +0 -22
- package/pipeline/eval/triage/07-duplicate-reviewers/notes.md +0 -9
- package/pipeline/eval/triage/08-style-misclassified/expected.json +0 -18
- package/pipeline/eval/triage/08-style-misclassified/input.json +0 -14
- package/pipeline/eval/triage/08-style-misclassified/notes.md +0 -9
- package/pipeline/eval/triage/09-cascading-finding/expected.json +0 -23
- package/pipeline/eval/triage/09-cascading-finding/input.json +0 -22
- package/pipeline/eval/triage/09-cascading-finding/notes.md +0 -9
- package/pipeline/eval/triage/10-deferred-crossref/expected.json +0 -18
- package/pipeline/eval/triage/10-deferred-crossref/input.json +0 -14
- package/pipeline/eval/triage/10-deferred-crossref/notes.md +0 -9
- package/pipeline/eval/triage/11-vercel-token-leak-blocker/expected.json +0 -27
- package/pipeline/eval/triage/11-vercel-token-leak-blocker/input.json +0 -22
- package/pipeline/eval/triage/11-vercel-token-leak-blocker/notes.md +0 -14
- package/pipeline/eval/triage/README.md +0 -54
- package/pipeline/scripts/benchmark-phase-0.sh +0 -128
- package/pipeline/scripts/check-md-links.mjs +0 -88
- package/pipeline/scripts/eval-golden-tasks-live.mjs +0 -302
- package/pipeline/scripts/eval-golden-tasks.mjs +0 -224
- package/pipeline/scripts/eval-intent.mjs +0 -107
- package/pipeline/scripts/eval-mine-corpus.mjs +0 -211
- package/pipeline/scripts/eval-triage.mjs +0 -171
- package/pipeline/scripts/fixtures/diff-risk-android.diff +0 -40
- package/pipeline/scripts/fixtures/diff-risk-ios.diff +0 -48
- package/pipeline/scripts/fixtures/diff-risk-test-removal.diff +0 -40
- package/pipeline/scripts/fixtures/install-layout.tsv +0 -19
- package/pipeline/scripts/fixtures/pack-expected-count.txt +0 -1
- package/pipeline/scripts/fixtures/test-gap-node.diff +0 -30
- package/pipeline/scripts/fixtures/test-gap-python.diff +0 -32
- package/pipeline/scripts/lint-mcp-refs.mjs +0 -218
- package/pipeline/scripts/lint-skills.mjs +0 -154
- package/pipeline/scripts/run-smokes.mjs +0 -130
- package/pipeline/scripts/scorecard.mjs +0 -258
- package/pipeline/scripts/smoke-add-detail.sh +0 -137
- package/pipeline/scripts/smoke-agent-guard.sh +0 -74
- package/pipeline/scripts/smoke-agent-log-cost.sh +0 -262
- package/pipeline/scripts/smoke-agent-model-routing.sh +0 -87
- package/pipeline/scripts/smoke-ask-choice.sh +0 -42
- package/pipeline/scripts/smoke-autopilot-circuit-breaker.sh +0 -36
- package/pipeline/scripts/smoke-bitbucket-contract.sh +0 -255
- package/pipeline/scripts/smoke-changelog-version.sh +0 -47
- package/pipeline/scripts/smoke-channels-approval-gate.sh +0 -60
- package/pipeline/scripts/smoke-channels-flow.sh +0 -130
- package/pipeline/scripts/smoke-ci-workflows.sh +0 -88
- package/pipeline/scripts/smoke-clarify.sh +0 -148
- package/pipeline/scripts/smoke-command-inventory.sh +0 -81
- package/pipeline/scripts/smoke-commands-skills-parity.sh +0 -87
- package/pipeline/scripts/smoke-community-gates.sh +0 -75
- package/pipeline/scripts/smoke-compliance-skills.sh +0 -119
- package/pipeline/scripts/smoke-config-hygiene.sh +0 -58
- package/pipeline/scripts/smoke-context-budget.sh +0 -72
- package/pipeline/scripts/smoke-cost-budget.sh +0 -70
- package/pipeline/scripts/smoke-cost-summary.sh +0 -139
- package/pipeline/scripts/smoke-cross-phase-cohesion.sh +0 -128
- package/pipeline/scripts/smoke-description-tr.sh +0 -82
- package/pipeline/scripts/smoke-dev-critic.sh +0 -144
- package/pipeline/scripts/smoke-diff-explain.sh +0 -147
- package/pipeline/scripts/smoke-diff-risk.sh +0 -190
- package/pipeline/scripts/smoke-dynamic-skill-loading.sh +0 -160
- package/pipeline/scripts/smoke-eval-live.sh +0 -136
- package/pipeline/scripts/smoke-evidence-gate.sh +0 -93
- package/pipeline/scripts/smoke-extract-conventions.sh +0 -163
- package/pipeline/scripts/smoke-fetchers-offline.sh +0 -448
- package/pipeline/scripts/smoke-figma-dispatch.sh +0 -112
- package/pipeline/scripts/smoke-gate-hooks.sh +0 -74
- package/pipeline/scripts/smoke-gc-tmp.sh +0 -130
- package/pipeline/scripts/smoke-gc-worktrees.sh +0 -125
- package/pipeline/scripts/smoke-generate-issue.sh +0 -120
- package/pipeline/scripts/smoke-handoff-contract.sh +0 -92
- package/pipeline/scripts/smoke-identity-isolation.sh +0 -70
- package/pipeline/scripts/smoke-install-layout.sh +0 -248
- package/pipeline/scripts/smoke-intent-guard.sh +0 -86
- package/pipeline/scripts/smoke-issue-comment-template.sh +0 -86
- package/pipeline/scripts/smoke-issue-jira-triad.sh +0 -120
- package/pipeline/scripts/smoke-keychain.sh +0 -158
- package/pipeline/scripts/smoke-language-axis.sh +0 -109
- package/pipeline/scripts/smoke-learning-curve.sh +0 -61
- package/pipeline/scripts/smoke-learnings-ledger.sh +0 -86
- package/pipeline/scripts/smoke-lib-scripts.sh +0 -448
- package/pipeline/scripts/smoke-mcp-gate.sh +0 -68
- package/pipeline/scripts/smoke-md-links.sh +0 -8
- package/pipeline/scripts/smoke-md2confluence.sh +0 -126
- package/pipeline/scripts/smoke-metrics-cache-ratio.sh +0 -72
- package/pipeline/scripts/smoke-migrate-state.sh +0 -102
- package/pipeline/scripts/smoke-mode-dispatch-drift.sh +0 -161
- package/pipeline/scripts/smoke-model-fallback.sh +0 -89
- package/pipeline/scripts/smoke-multi-repo-integration.sh +0 -116
- package/pipeline/scripts/smoke-multi-repo-worktree.sh +0 -61
- package/pipeline/scripts/smoke-no-mcp-in-dev-phases.sh +0 -194
- package/pipeline/scripts/smoke-no-token-prompt.sh +0 -85
- package/pipeline/scripts/smoke-own-punctuation.sh +0 -103
- package/pipeline/scripts/smoke-pack-contents.sh +0 -140
- package/pipeline/scripts/smoke-pat-audit.sh +0 -128
- package/pipeline/scripts/smoke-per-repo-memory.sh +0 -156
- package/pipeline/scripts/smoke-phase-0-multi-repo.sh +0 -170
- package/pipeline/scripts/smoke-phase-6-multi.sh +0 -79
- package/pipeline/scripts/smoke-phase-banner.sh +0 -101
- package/pipeline/scripts/smoke-phase-tracker.sh +0 -324
- package/pipeline/scripts/smoke-phase0-bridge-contract.sh +0 -241
- package/pipeline/scripts/smoke-phase4-gates.sh +0 -45
- package/pipeline/scripts/smoke-phase4-triage.sh +0 -229
- package/pipeline/scripts/smoke-plan-approval-gate.sh +0 -71
- package/pipeline/scripts/smoke-plan-safety.sh +0 -139
- package/pipeline/scripts/smoke-plan-todos.sh +0 -196
- package/pipeline/scripts/smoke-pr-review-actions.sh +0 -152
- package/pipeline/scripts/smoke-pre-commit.sh +0 -170
- package/pipeline/scripts/smoke-pref-migration.sh +0 -226
- package/pipeline/scripts/smoke-prefs-language.sh +0 -134
- package/pipeline/scripts/smoke-progress-contract.sh +0 -127
- package/pipeline/scripts/smoke-prune-logs.sh +0 -137
- package/pipeline/scripts/smoke-purge.sh +0 -138
- package/pipeline/scripts/smoke-push-retry.sh +0 -75
- package/pipeline/scripts/smoke-repo-map.sh +0 -300
- package/pipeline/scripts/smoke-review-readiness.sh +0 -92
- package/pipeline/scripts/smoke-review-watch.sh +0 -146
- package/pipeline/scripts/smoke-routines.sh +0 -84
- package/pipeline/scripts/smoke-run-aggregator.sh +0 -216
- package/pipeline/scripts/smoke-run-metrics.sh +0 -50
- package/pipeline/scripts/smoke-search.sh +0 -187
- package/pipeline/scripts/smoke-shadow-git.sh +0 -224
- package/pipeline/scripts/smoke-skill-authoring.sh +0 -137
- package/pipeline/scripts/smoke-skill-language.sh +0 -83
- package/pipeline/scripts/smoke-skill-manifest.sh +0 -138
- package/pipeline/scripts/smoke-skill-scan.sh +0 -198
- package/pipeline/scripts/smoke-source-parity.sh +0 -85
- package/pipeline/scripts/smoke-subagent-validators.sh +0 -108
- package/pipeline/scripts/smoke-sync-parity.sh +0 -92
- package/pipeline/scripts/smoke-tasklist-ordering.sh +0 -112
- package/pipeline/scripts/smoke-telemetry.sh +0 -147
- package/pipeline/scripts/smoke-test-gap.sh +0 -183
- package/pipeline/scripts/smoke-token-budget.sh +0 -67
- package/pipeline/scripts/smoke-token-preflight.sh +0 -82
- package/pipeline/scripts/smoke-tracker-contract.sh +0 -191
- package/pipeline/scripts/smoke-tracker-tokens-invocation.sh +0 -73
- package/pipeline/scripts/smoke-triage-memory.sh +0 -174
- package/pipeline/scripts/smoke-update-check.sh +0 -135
- package/pipeline/scripts/smoke-url-enrichment.sh +0 -70
- package/pipeline/scripts/smoke-validate-analysis-doc.sh +0 -161
- package/pipeline/scripts/smoke-validator-contradiction.sh +0 -67
- package/pipeline/scripts/smoke-validator-gates.sh +0 -164
- package/pipeline/scripts/smoke-vercel-deploy-redact.sh +0 -129
- package/pipeline/scripts/smoke-verify-by-test.sh +0 -148
- package/pipeline/scripts/smoke-wiki-integration.sh +0 -122
- package/pipeline/scripts/smoke-work-summary.sh +0 -163
- package/pipeline/scripts/smoke-workflow-audit.sh +0 -101
- package/pipeline/scripts/smoke-worktree-path-convention.sh +0 -86
- package/pipeline/scripts/smoke-wrapper-preservation.sh +0 -68
- package/pipeline/scripts/smoke-write-state.sh +0 -159
- package/pipeline/scripts/sync-parity-check.sh +0 -135
- package/pipeline/scripts/test-gap-rules/android.json +0 -25
- package/pipeline/scripts/test-gap-rules/ios.json +0 -34
- package/pipeline/scripts/test-gap-rules/node.json +0 -29
- package/pipeline/scripts/test-gap-rules/python.json +0 -25
- package/pipeline/scripts/validate-schemas.mjs +0 -88
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# smoke-cost-budget.sh - exercise cost-budget-check.mjs decision logic with
|
|
3
|
-
# fixture tracker files: disabled no-op, ok, warn threshold, hard ceiling,
|
|
4
|
-
# missing tracker, and prefs-driven config.
|
|
5
|
-
|
|
6
|
-
set -uo pipefail
|
|
7
|
-
|
|
8
|
-
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
9
|
-
CHECK="$REPO_ROOT/pipeline/scripts/cost-budget-check.mjs"
|
|
10
|
-
|
|
11
|
-
PASS=0
|
|
12
|
-
FAIL=0
|
|
13
|
-
pass() { PASS=$((PASS + 1)); echo " ✓ $1"; }
|
|
14
|
-
fail() { FAIL=$((FAIL + 1)); echo " ✗ $1"; }
|
|
15
|
-
|
|
16
|
-
cleanup() { [ -n "${TMP:-}" ] && rm -rf "$TMP"; }
|
|
17
|
-
trap cleanup EXIT
|
|
18
|
-
TMP=$(mktemp -d)
|
|
19
|
-
|
|
20
|
-
# opus rate in cost-table.json: $15/Mtok in, $75/Mtok out.
|
|
21
|
-
# 100k in + 100k out = 0.1*5 + 0.1*25 = $0.50 + $2.50 = $3.00.
|
|
22
|
-
mk_tracker() { # <in> <out> -> path
|
|
23
|
-
local f="$TMP/tracker-$1-$2.json"
|
|
24
|
-
printf '{"phases":{"1":{"name":"Analysis","tokens_in":%s,"tokens_out":%s}}}' "$1" "$2" > "$f"
|
|
25
|
-
echo "$f"
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
run() { node "$CHECK" "$@" >/dev/null 2>&1; echo $?; }
|
|
29
|
-
|
|
30
|
-
echo "→ 1. Disabled (no --enabled, no prefs) is a silent no-op (exit 0)"
|
|
31
|
-
T=$(mk_tracker 100000 100000)
|
|
32
|
-
[ "$(run --tracker "$T" --max-usd 5)" = "0" ] && pass "disabled exits 0" || fail "disabled should exit 0"
|
|
33
|
-
|
|
34
|
-
echo "→ 2. Under budget -> OK (exit 0)"
|
|
35
|
-
T=$(mk_tracker 1000 1000) # ~ $0.03 of $5
|
|
36
|
-
[ "$(run --tracker "$T" --enabled --max-usd 5 --pricing-model opus)" = "0" ] && pass "under budget exits 0" || fail "under budget should exit 0"
|
|
37
|
-
|
|
38
|
-
echo "→ 3. Past warn threshold -> WARN (exit 10)"
|
|
39
|
-
# $3.00 spend, max $4, warn 70% ($2.80) -> warn
|
|
40
|
-
T=$(mk_tracker 100000 100000)
|
|
41
|
-
[ "$(run --tracker "$T" --enabled --max-usd 4 --warn-pct 70 --pricing-model opus)" = "10" ] && pass "warn threshold exits 10" || fail "warn threshold should exit 10"
|
|
42
|
-
|
|
43
|
-
echo "→ 4. Over ceiling -> EXCEEDED (exit 11)"
|
|
44
|
-
# $3.00 spend, max $2 -> exceeded
|
|
45
|
-
[ "$(run --tracker "$T" --enabled --max-usd 2 --pricing-model opus)" = "11" ] && pass "over ceiling exits 11" || fail "over ceiling should exit 11"
|
|
46
|
-
|
|
47
|
-
echo "→ 5. Cheaper pricing model lowers the estimate"
|
|
48
|
-
# same tokens at haiku ($1/$5): 0.1*1 + 0.1*5 = $0.60 -> under $5
|
|
49
|
-
[ "$(run --tracker "$T" --enabled --max-usd 5 --pricing-model haiku)" = "0" ] && pass "haiku rate stays under budget" || fail "haiku rate should be under budget"
|
|
50
|
-
|
|
51
|
-
echo "→ 6. Missing tracker is non-fatal (exit 0)"
|
|
52
|
-
[ "$(run --tracker "$TMP/does-not-exist.json" --enabled --max-usd 5)" = "2" ] && pass "explicit missing tracker errors (exit 2)" || fail "explicit missing tracker should exit 2"
|
|
53
|
-
[ "$(run --task-id 99999 --enabled --max-usd 5)" = "0" ] && pass "unresolvable task-id tracker exits 0 (nothing to gate)" || fail "unresolvable task-id should exit 0"
|
|
54
|
-
|
|
55
|
-
echo "→ 7. prefs-driven config (enabled + maxUsd from prefs)"
|
|
56
|
-
cat > "$TMP/prefs.json" <<'EOF'
|
|
57
|
-
{"global":{"costBudget":{"enabled":true,"maxUsd":2.0,"warnPct":80,"pricingModel":"opus"}}}
|
|
58
|
-
EOF
|
|
59
|
-
[ "$(run --tracker "$T" --prefs "$TMP/prefs.json")" = "11" ] && pass "prefs enable + low ceiling exits 11" || fail "prefs-driven exceeded should exit 11"
|
|
60
|
-
|
|
61
|
-
echo "→ 8. prefs disabled overrides to no-op unless --enabled forces it"
|
|
62
|
-
cat > "$TMP/prefs-off.json" <<'EOF'
|
|
63
|
-
{"global":{"costBudget":{"enabled":false,"maxUsd":1.0}}}
|
|
64
|
-
EOF
|
|
65
|
-
[ "$(run --tracker "$T" --prefs "$TMP/prefs-off.json")" = "0" ] && pass "prefs disabled exits 0" || fail "prefs disabled should exit 0"
|
|
66
|
-
[ "$(run --tracker "$T" --prefs "$TMP/prefs-off.json" --enabled --max-usd 1)" = "11" ] && pass "--enabled overrides prefs disabled" || fail "--enabled should override prefs disabled"
|
|
67
|
-
|
|
68
|
-
echo ""
|
|
69
|
-
echo "══ cost-budget smoke: $PASS passed, $FAIL failed ══"
|
|
70
|
-
[ "$FAIL" -eq 0 ]
|
|
@@ -1,139 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# smoke-cost-summary.sh - v6.1.J
|
|
3
|
-
#
|
|
4
|
-
# Verifies the cost-summary pipeline:
|
|
5
|
-
# 1. cost-table.json parses and has opus/sonnet/haiku entries
|
|
6
|
-
# 2. render-cost-summary.sh exits 2 when no tracker data available
|
|
7
|
-
# 3. render-cost-summary.sh emits a valid markdown table from tracker fixture
|
|
8
|
-
# 4. OTel-spans fallback path renders when tracker JSON is missing
|
|
9
|
-
# 5. Unknown model gracefully renders USD as - with footnote
|
|
10
|
-
# 6. Schema (prefs.schema.json) exposes costSummary toggle
|
|
11
|
-
# 7. channels.md documents Cost Summary content option
|
|
12
|
-
#
|
|
13
|
-
# Exit 0 = all pass, 1 = any failure.
|
|
14
|
-
|
|
15
|
-
set -euo pipefail
|
|
16
|
-
|
|
17
|
-
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
18
|
-
RENDERER="$ROOT/pipeline/scripts/render-cost-summary.sh"
|
|
19
|
-
TABLE="$ROOT/pipeline/scripts/cost-table.json"
|
|
20
|
-
SCHEMA="$ROOT/pipeline/schemas/prefs.schema.json"
|
|
21
|
-
CHANNELS_DOC="$ROOT/pipeline/commands/multi-agent/channels/SKILL.md"
|
|
22
|
-
|
|
23
|
-
pass=0
|
|
24
|
-
fail=0
|
|
25
|
-
failures=()
|
|
26
|
-
record_pass() { pass=$((pass + 1)); printf ' \033[0;32mPASS\033[0m %s\n' "$1"; }
|
|
27
|
-
record_fail() { fail=$((fail + 1)); failures+=("$1"); printf ' \033[0;31mFAIL\033[0m %s\n' "$1"; }
|
|
28
|
-
|
|
29
|
-
printf '→ smoke-cost-summary (v6.1.J): PR-body Cost Summary renderer contract\n'
|
|
30
|
-
|
|
31
|
-
# --- 1: cost-table.json shape ---
|
|
32
|
-
if jq -e '.prices.opus.inPerMtok and .prices.sonnet.inPerMtok and .prices.haiku.inPerMtok' "$TABLE" >/dev/null 2>&1; then
|
|
33
|
-
record_pass "cost-table.json has opus/sonnet/haiku prices"
|
|
34
|
-
else
|
|
35
|
-
record_fail "cost-table.json missing one of opus/sonnet/haiku"
|
|
36
|
-
fi
|
|
37
|
-
|
|
38
|
-
# --- 2: missing-data path returns exit 2 ---
|
|
39
|
-
tmp=$(mktemp -d)
|
|
40
|
-
pushd "$tmp" >/dev/null
|
|
41
|
-
set +e
|
|
42
|
-
bash "$RENDERER" GHOST-1 >/dev/null 2>&1
|
|
43
|
-
rc=$?
|
|
44
|
-
set -e
|
|
45
|
-
if [ "$rc" -eq 2 ]; then
|
|
46
|
-
record_pass "renderer exits 2 when no tracker data"
|
|
47
|
-
else
|
|
48
|
-
record_fail "renderer should exit 2 with no data (got $rc)"
|
|
49
|
-
fi
|
|
50
|
-
popd >/dev/null
|
|
51
|
-
rm -rf "$tmp"
|
|
52
|
-
|
|
53
|
-
# --- 3: tracker fixture renders markdown ---
|
|
54
|
-
tmp=$(mktemp -d)
|
|
55
|
-
pushd "$tmp" >/dev/null
|
|
56
|
-
mkdir -p .worktrees/DEMO-1
|
|
57
|
-
cat > .worktrees/DEMO-1/phase-tracker.json <<JSON
|
|
58
|
-
{"taskId":"DEMO-1","phases":{
|
|
59
|
-
"1":{"name":"1. Analysis","model":"opus","tokens_in":12480,"tokens_out":3210},
|
|
60
|
-
"3":{"name":"3. Dev","model":"sonnet","tokens_in":42310,"tokens_out":9880}
|
|
61
|
-
}}
|
|
62
|
-
JSON
|
|
63
|
-
out=$(bash "$RENDERER" DEMO-1)
|
|
64
|
-
popd >/dev/null
|
|
65
|
-
rm -rf "$tmp"
|
|
66
|
-
|
|
67
|
-
if grep -q '^### Cost Summary$' <<< "$out" && \
|
|
68
|
-
grep -q '1. Analysis' <<< "$out" && \
|
|
69
|
-
grep -q '3. Dev' <<< "$out" && \
|
|
70
|
-
grep -q '\*\*Total\*\*' <<< "$out"; then
|
|
71
|
-
record_pass "tracker fixture renders header/rows/total"
|
|
72
|
-
else
|
|
73
|
-
record_fail "tracker fixture output missing required rows"
|
|
74
|
-
fi
|
|
75
|
-
|
|
76
|
-
if grep -qE '\$[0-9]+\.[0-9]{2}' <<< "$out"; then
|
|
77
|
-
record_pass "USD formatted to 2 dp"
|
|
78
|
-
else
|
|
79
|
-
record_fail "USD not formatted to 2 dp"
|
|
80
|
-
fi
|
|
81
|
-
|
|
82
|
-
# --- 4: OTel spans fallback ---
|
|
83
|
-
tmp=$(mktemp -d)
|
|
84
|
-
pushd "$tmp" >/dev/null
|
|
85
|
-
mkdir -p .worktrees/OTEL-1
|
|
86
|
-
cat > /tmp/cost-spans.jsonl <<JSON
|
|
87
|
-
{"name":"phase.tokens.1","attributes":{"phase_id":"1","phase_name":"1. Analysis","model":"opus","tokens_in_delta":5000,"tokens_out_delta":1200}}
|
|
88
|
-
{"name":"phase.tokens.1","attributes":{"phase_id":"1","phase_name":"1. Analysis","model":"opus","tokens_in_delta":3000,"tokens_out_delta":800}}
|
|
89
|
-
JSON
|
|
90
|
-
out2=$(bash "$RENDERER" OTEL-1 --otel-spans /tmp/cost-spans.jsonl || true)
|
|
91
|
-
popd >/dev/null
|
|
92
|
-
rm -rf "$tmp" /tmp/cost-spans.jsonl
|
|
93
|
-
|
|
94
|
-
if grep -q '1. Analysis' <<< "$out2" && grep -q '8,000' <<< "$out2"; then
|
|
95
|
-
record_pass "OTel spans fallback aggregates phase tokens"
|
|
96
|
-
else
|
|
97
|
-
record_fail "OTel fallback did not aggregate tokens (expected 8,000 in, got: $(echo "$out2" | tr '\n' ' ' | cut -c1-200))"
|
|
98
|
-
fi
|
|
99
|
-
|
|
100
|
-
# --- 5: unknown model → - + footnote ---
|
|
101
|
-
tmp=$(mktemp -d)
|
|
102
|
-
pushd "$tmp" >/dev/null
|
|
103
|
-
mkdir -p .worktrees/UNK-1
|
|
104
|
-
cat > .worktrees/UNK-1/phase-tracker.json <<JSON
|
|
105
|
-
{"taskId":"UNK-1","phases":{
|
|
106
|
-
"1":{"name":"1. Analysis","model":"mystery-model","tokens_in":1000,"tokens_out":500}
|
|
107
|
-
}}
|
|
108
|
-
JSON
|
|
109
|
-
out3=$(bash "$RENDERER" UNK-1)
|
|
110
|
-
popd >/dev/null
|
|
111
|
-
rm -rf "$tmp"
|
|
112
|
-
|
|
113
|
-
if grep -q '| - ' <<< "$out3" && grep -q 'USD unavailable' <<< "$out3"; then
|
|
114
|
-
record_pass "unknown model renders dash + footnote"
|
|
115
|
-
else
|
|
116
|
-
record_fail "unknown model path missing dash or footnote"
|
|
117
|
-
fi
|
|
118
|
-
|
|
119
|
-
# --- 6: prefs schema exposes costSummary toggle ---
|
|
120
|
-
if jq -e '.properties.global.properties.reportContent.properties.costSummary' "$SCHEMA" >/dev/null 2>&1; then
|
|
121
|
-
record_pass "prefs schema exposes reportContent.costSummary"
|
|
122
|
-
else
|
|
123
|
-
record_fail "prefs schema missing reportContent.costSummary"
|
|
124
|
-
fi
|
|
125
|
-
|
|
126
|
-
# --- 7: channels.md documents the option ---
|
|
127
|
-
if grep -q '^### Cost Summary' "$CHANNELS_DOC" && grep -q 'Cost özeti' "$CHANNELS_DOC"; then
|
|
128
|
-
record_pass "channels.md documents Cost Summary option"
|
|
129
|
-
else
|
|
130
|
-
record_fail "channels.md missing Cost Summary docs"
|
|
131
|
-
fi
|
|
132
|
-
|
|
133
|
-
printf '\n══ cost-summary smoke: %d passed, %d failed ══\n' "$pass" "$fail"
|
|
134
|
-
if [ "$fail" -gt 0 ]; then
|
|
135
|
-
printf '\nFailures:\n'
|
|
136
|
-
for msg in "${failures[@]}"; do printf ' - %s\n' "$msg"; done
|
|
137
|
-
exit 1
|
|
138
|
-
fi
|
|
139
|
-
exit 0
|
|
@@ -1,128 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
#
|
|
3
|
-
# smoke-cross-phase-cohesion.sh - assert every Phase N → Phase N+1 hand-off
|
|
4
|
-
# in the multi-agent pipeline references the same schema on both sides.
|
|
5
|
-
#
|
|
6
|
-
# Why this exists (v8.0.0): existing schema-validation smoke proves each
|
|
7
|
-
# JSON schema is itself well-formed. It does NOT prove that Phase 1's
|
|
8
|
-
# reference doc says "I emit analysis-output.schema.json" while Phase 2's
|
|
9
|
-
# reference doc says "I read analysis-output.schema.json". A reference can
|
|
10
|
-
# rename or drop the schema without the validator catching it - phase docs
|
|
11
|
-
# would silently desynchronize from the runtime contract.
|
|
12
|
-
#
|
|
13
|
-
# The contract chain:
|
|
14
|
-
# Phase 1 (Analysis) → analysis-output.schema.json → Phase 2 (Planning)
|
|
15
|
-
# Phase 2 (Planning) → planning-output.schema.json → Phase 3 (Dev)
|
|
16
|
-
# Phase 4 (Review) → reviewer-output.schema.json → Phase 4 triage
|
|
17
|
-
# Phase 4 (Review) → triage-output.schema.json → Phase 6 (Commit)
|
|
18
|
-
# Phase 6 (Commit) → agent-state.json fields → Phase 7 (Report)
|
|
19
|
-
#
|
|
20
|
-
# Each schema must (a) exist on disk, (b) be referenced in the producer
|
|
21
|
-
# phase ref doc, (c) be referenced in the consumer phase ref doc.
|
|
22
|
-
#
|
|
23
|
-
# Exit codes: 0=pass, 1=cohesion break detected
|
|
24
|
-
set -euo pipefail
|
|
25
|
-
|
|
26
|
-
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
27
|
-
PHASE_DIR="$REPO_ROOT/pipeline/multi-agent-refs/phases"
|
|
28
|
-
SCHEMA_DIR="$REPO_ROOT/pipeline/schemas"
|
|
29
|
-
|
|
30
|
-
PASS=0
|
|
31
|
-
FAIL=0
|
|
32
|
-
pass() { echo " ✓ $*"; PASS=$((PASS + 1)); }
|
|
33
|
-
fail() { echo " ✗ $*"; FAIL=$((FAIL + 1)); }
|
|
34
|
-
|
|
35
|
-
assert_schema_exists() {
|
|
36
|
-
local schema="$1"
|
|
37
|
-
if [ -f "$SCHEMA_DIR/$schema" ]; then
|
|
38
|
-
pass "$schema exists in pipeline/schemas/"
|
|
39
|
-
else
|
|
40
|
-
fail "$schema MISSING - runtime contract broken"
|
|
41
|
-
fi
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
assert_phase_references() {
|
|
45
|
-
local phase_file="$1" schema="$2" role="$3"
|
|
46
|
-
local path="$PHASE_DIR/$phase_file"
|
|
47
|
-
if [ ! -f "$path" ]; then
|
|
48
|
-
fail "$phase_file not found at $path"
|
|
49
|
-
return
|
|
50
|
-
fi
|
|
51
|
-
if grep -Fq "$schema" "$path"; then
|
|
52
|
-
pass "$phase_file [$role] references $schema"
|
|
53
|
-
else
|
|
54
|
-
fail "$phase_file [$role] does NOT reference $schema - cohesion break"
|
|
55
|
-
fi
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
# -----------------------------------------------------------------------------
|
|
59
|
-
echo "→ 1. Pipeline schemas all exist on disk"
|
|
60
|
-
# -----------------------------------------------------------------------------
|
|
61
|
-
assert_schema_exists "analysis-output.schema.json"
|
|
62
|
-
assert_schema_exists "planning-output.schema.json"
|
|
63
|
-
assert_schema_exists "reviewer-output.schema.json"
|
|
64
|
-
assert_schema_exists "triage-output.schema.json"
|
|
65
|
-
assert_schema_exists "agent-state.schema.json"
|
|
66
|
-
assert_schema_exists "prefs.schema.json"
|
|
67
|
-
|
|
68
|
-
# -----------------------------------------------------------------------------
|
|
69
|
-
echo ""
|
|
70
|
-
echo "→ 2. Phase 1 → Phase 2 hand-off (analysis-output.schema.json)"
|
|
71
|
-
# -----------------------------------------------------------------------------
|
|
72
|
-
assert_phase_references "phase-1-analysis.md" "analysis-output.schema.json" "producer"
|
|
73
|
-
assert_phase_references "phase-2-planning.md" "analysis-output.schema.json" "consumer"
|
|
74
|
-
|
|
75
|
-
# -----------------------------------------------------------------------------
|
|
76
|
-
echo ""
|
|
77
|
-
echo "→ 3. Phase 2 → Phase 3 hand-off (planning-output.schema.json)"
|
|
78
|
-
# -----------------------------------------------------------------------------
|
|
79
|
-
assert_phase_references "phase-2-planning.md" "planning-output.schema.json" "producer"
|
|
80
|
-
assert_phase_references "phase-3-dev.md" "planning-output.schema.json" "consumer"
|
|
81
|
-
|
|
82
|
-
# -----------------------------------------------------------------------------
|
|
83
|
-
echo ""
|
|
84
|
-
echo "→ 4. Phase 4 review → triage hand-off (reviewer-output.schema.json)"
|
|
85
|
-
# -----------------------------------------------------------------------------
|
|
86
|
-
assert_phase_references "phase-4-review.md" "reviewer-output.schema.json" "producer"
|
|
87
|
-
# Triage lives inside phase-4-review.md too; same file is producer + consumer for the chain.
|
|
88
|
-
if grep -B 2 -A 60 "reviewer-output.schema.json" "$PHASE_DIR/phase-4-review.md" \
|
|
89
|
-
| grep -q "triage-output.schema.json"; then
|
|
90
|
-
pass "phase-4-review.md threads reviewer→triage in the same doc"
|
|
91
|
-
else
|
|
92
|
-
fail "phase-4-review.md missing triage hand-off block"
|
|
93
|
-
fi
|
|
94
|
-
|
|
95
|
-
# -----------------------------------------------------------------------------
|
|
96
|
-
echo ""
|
|
97
|
-
echo "→ 5. Phase 4 triage → Phase 6 commit hand-off (triage-output.schema.json)"
|
|
98
|
-
# -----------------------------------------------------------------------------
|
|
99
|
-
assert_phase_references "phase-4-review.md" "triage-output.schema.json" "producer"
|
|
100
|
-
# Phase 6 doesn't always re-state the schema name; instead it references the
|
|
101
|
-
# triage outcome bucket counts. Accept either form.
|
|
102
|
-
if grep -Eq "triage-output\.schema\.json|triage outcome|accepted findings" "$PHASE_DIR/phase-6-commit.md"; then
|
|
103
|
-
pass "phase-6-commit.md consumes triage output (schema or outcome reference)"
|
|
104
|
-
else
|
|
105
|
-
fail "phase-6-commit.md does NOT consume Phase 4 triage output"
|
|
106
|
-
fi
|
|
107
|
-
|
|
108
|
-
# -----------------------------------------------------------------------------
|
|
109
|
-
echo ""
|
|
110
|
-
echo "→ 6. Phase 6 → Phase 7 hand-off (agent-state.json mutations)"
|
|
111
|
-
# -----------------------------------------------------------------------------
|
|
112
|
-
assert_phase_references "phase-6-commit.md" "agent-state" "producer"
|
|
113
|
-
assert_phase_references "phase-7-report.md" "agent-state" "consumer"
|
|
114
|
-
|
|
115
|
-
# -----------------------------------------------------------------------------
|
|
116
|
-
echo ""
|
|
117
|
-
echo "→ 7. Validator runs the round-trip without errors"
|
|
118
|
-
# -----------------------------------------------------------------------------
|
|
119
|
-
if node "$REPO_ROOT/pipeline/scripts/validate-schemas.mjs" >/dev/null 2>&1; then
|
|
120
|
-
pass "validate-schemas.mjs reports all schemas valid"
|
|
121
|
-
else
|
|
122
|
-
fail "validate-schemas.mjs failed - schema integrity broken"
|
|
123
|
-
fi
|
|
124
|
-
|
|
125
|
-
# -----------------------------------------------------------------------------
|
|
126
|
-
echo ""
|
|
127
|
-
echo "══ smoke-cross-phase-cohesion: $PASS passed, $FAIL failed ══"
|
|
128
|
-
[ "$FAIL" -gt 0 ] && exit 1 || exit 0
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# smoke-description-tr.sh - localized picker-description contract.
|
|
3
|
-
#
|
|
4
|
-
# Verifies: every shipped command SKILL.md carries a description-tr line
|
|
5
|
-
# (coverage), repo description: lines stay English with no description-en
|
|
6
|
-
# sidecars committed (hygiene), and localize-commands.mjs apply/restore is
|
|
7
|
-
# an idempotent, byte-identical round trip (mechanics).
|
|
8
|
-
|
|
9
|
-
set -uo pipefail
|
|
10
|
-
|
|
11
|
-
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
12
|
-
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
13
|
-
CMD_DIR="$REPO_ROOT/pipeline/commands/multi-agent"
|
|
14
|
-
LOCALIZER="$REPO_ROOT/pipeline/scripts/localize-commands.mjs"
|
|
15
|
-
|
|
16
|
-
PASS=0
|
|
17
|
-
FAIL=0
|
|
18
|
-
record_pass() { PASS=$((PASS + 1)); printf ' \342\234\223 %s\n' "$1"; }
|
|
19
|
-
record_fail() { FAIL=$((FAIL + 1)); printf ' \342\234\227 %s\n' "$1"; }
|
|
20
|
-
|
|
21
|
-
printf '\342\206\222 smoke-description-tr: localized picker-description contract\n'
|
|
22
|
-
|
|
23
|
-
# 1. Coverage: every command SKILL.md (incl. the dispatcher) has description-tr
|
|
24
|
-
missing=""
|
|
25
|
-
count=0
|
|
26
|
-
for f in "$CMD_DIR/SKILL.md" "$CMD_DIR"/*/SKILL.md; do
|
|
27
|
-
[ -f "$f" ] || continue
|
|
28
|
-
count=$((count + 1))
|
|
29
|
-
grep -q '^description-tr:' "$f" || missing="$missing ${f#"$CMD_DIR/"}"
|
|
30
|
-
done
|
|
31
|
-
if [ -z "$missing" ]; then
|
|
32
|
-
record_pass "all $count command SKILL.md files carry description-tr"
|
|
33
|
-
else
|
|
34
|
-
record_fail "missing description-tr in:$missing"
|
|
35
|
-
fi
|
|
36
|
-
|
|
37
|
-
# 2. Hygiene: repo descriptions stay English, no sidecars committed
|
|
38
|
-
if grep -rn '^description:.*[çğıöşüÇĞİÖŞÜ]' "$CMD_DIR" >/dev/null 2>&1; then
|
|
39
|
-
record_fail "repo description: lines must stay English (Turkish characters found)"
|
|
40
|
-
else
|
|
41
|
-
record_pass "repo description: lines are English"
|
|
42
|
-
fi
|
|
43
|
-
if grep -rln '^description-en:' "$CMD_DIR" >/dev/null 2>&1; then
|
|
44
|
-
record_fail "description-en sidecar committed to the repo (installed-tree artifact)"
|
|
45
|
-
else
|
|
46
|
-
record_pass "no description-en sidecar in the repo"
|
|
47
|
-
fi
|
|
48
|
-
|
|
49
|
-
# 3. Translation length within the agent-skills description budget
|
|
50
|
-
long=$(awk '/^description-tr:/ && length($0) > 1044 { print FILENAME }' "$CMD_DIR"/*/SKILL.md "$CMD_DIR/SKILL.md" 2>/dev/null | head -1)
|
|
51
|
-
if [ -z "$long" ]; then
|
|
52
|
-
record_pass "every description-tr is within the 1024-char budget"
|
|
53
|
-
else
|
|
54
|
-
record_fail "description-tr exceeds budget in $long"
|
|
55
|
-
fi
|
|
56
|
-
|
|
57
|
-
# 4. Round trip: apply is idempotent, restore is byte-identical
|
|
58
|
-
tmpdir=$(mktemp -d)
|
|
59
|
-
trap 'rm -rf "$tmpdir"' EXIT
|
|
60
|
-
cp -R "$CMD_DIR" "$tmpdir/cmds"
|
|
61
|
-
node "$LOCALIZER" apply --dir "$tmpdir/cmds" >/dev/null 2>&1
|
|
62
|
-
if grep -q '^description:.*[çğıöşüÇĞİÖŞÜ]' "$tmpdir/cmds/dev/SKILL.md" \
|
|
63
|
-
&& grep -q '^description-en:' "$tmpdir/cmds/dev/SKILL.md"; then
|
|
64
|
-
record_pass "apply swaps description and writes the description-en sidecar"
|
|
65
|
-
else
|
|
66
|
-
record_fail "apply did not localize the sample file"
|
|
67
|
-
fi
|
|
68
|
-
second=$(node "$LOCALIZER" apply --dir "$tmpdir/cmds" 2>/dev/null)
|
|
69
|
-
if printf '%s' "$second" | grep -q ' 0 file(s) changed'; then
|
|
70
|
-
record_pass "second apply is a no-op (idempotent)"
|
|
71
|
-
else
|
|
72
|
-
record_fail "second apply changed files again: $second"
|
|
73
|
-
fi
|
|
74
|
-
node "$LOCALIZER" restore --dir "$tmpdir/cmds" >/dev/null 2>&1
|
|
75
|
-
if diff -r "$CMD_DIR" "$tmpdir/cmds" >/dev/null 2>&1; then
|
|
76
|
-
record_pass "restore returns a byte-identical tree"
|
|
77
|
-
else
|
|
78
|
-
record_fail "restore left differences vs the original tree"
|
|
79
|
-
fi
|
|
80
|
-
|
|
81
|
-
printf '\n\342\225\220\342\225\220 smoke-description-tr: %d passed, %d failed \342\225\220\342\225\220\n' "$PASS" "$FAIL"
|
|
82
|
-
[ "$FAIL" -eq 0 ]
|
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# smoke-dev-critic.sh
|
|
3
|
-
#
|
|
4
|
-
# Verifies the Phase 3.5 evaluator-optimizer contract:
|
|
5
|
-
# 1. pipeline/agents/dev-critic.md exists with required frontmatter
|
|
6
|
-
# 2. phase-3-dev.md documents Step 3.5 with loop cap + gates + escalate rule
|
|
7
|
-
# 3. prefs.schema.json exposes global.devCritic.{enabled,maxIterations,model,minDiffLoc}
|
|
8
|
-
# 4. dev-critic-output.schema.json exists and is valid JSON Schema
|
|
9
|
-
# 5. Severity enum + gate enum match the agent doc
|
|
10
|
-
# 6. Loop cap default is 2 (matches "max 2" rule documented in the agent)
|
|
11
|
-
# 7. Off-by-default (no surprise cost on existing users)
|
|
12
|
-
#
|
|
13
|
-
# Exit 0 = all pass, 1 = any failure.
|
|
14
|
-
|
|
15
|
-
set -euo pipefail
|
|
16
|
-
|
|
17
|
-
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
18
|
-
AGENT="$ROOT/pipeline/agents/dev-critic.md"
|
|
19
|
-
PHASE_DOC="$ROOT/pipeline/multi-agent-refs/phases/phase-3-dev.md"
|
|
20
|
-
PREFS_SCHEMA="$ROOT/pipeline/schemas/prefs.schema.json"
|
|
21
|
-
CRITIC_SCHEMA="$ROOT/pipeline/schemas/dev-critic-output.schema.json"
|
|
22
|
-
|
|
23
|
-
pass=0
|
|
24
|
-
fail=0
|
|
25
|
-
failures=()
|
|
26
|
-
record_pass() { pass=$((pass + 1)); printf ' \033[0;32mPASS\033[0m %s\n' "$1"; }
|
|
27
|
-
record_fail() { fail=$((fail + 1)); failures+=("$1"); printf ' \033[0;31mFAIL\033[0m %s\n' "$1"; }
|
|
28
|
-
|
|
29
|
-
printf '→ smoke-dev-critic: Phase 3.5 evaluator-optimizer contract\n'
|
|
30
|
-
|
|
31
|
-
# 1. Agent file exists with required frontmatter
|
|
32
|
-
if [ ! -f "$AGENT" ]; then
|
|
33
|
-
record_fail "pipeline/agents/dev-critic.md missing"
|
|
34
|
-
else
|
|
35
|
-
# Extract frontmatter (between first two --- lines)
|
|
36
|
-
fm=$(awk '/^---$/{n++; if(n==2) exit; if(n==1) next} n==1{print}' "$AGENT")
|
|
37
|
-
for key in description model preferredModel; do
|
|
38
|
-
if grep -qE "^${key}:" <<<"$fm"; then
|
|
39
|
-
record_pass "agent frontmatter has '${key}'"
|
|
40
|
-
else
|
|
41
|
-
record_fail "agent frontmatter missing '${key}'"
|
|
42
|
-
fi
|
|
43
|
-
done
|
|
44
|
-
# Model must be sonnet (the documented default)
|
|
45
|
-
if grep -qE "^model: sonnet" <<<"$fm"; then
|
|
46
|
-
record_pass "agent default model is sonnet"
|
|
47
|
-
else
|
|
48
|
-
record_fail "agent model default should be sonnet (was: $(grep '^model:' <<<"$fm" || echo missing))"
|
|
49
|
-
fi
|
|
50
|
-
fi
|
|
51
|
-
|
|
52
|
-
# 2. Phase doc mentions Step 3.5 with the loop contract
|
|
53
|
-
if [ ! -f "$PHASE_DOC" ]; then
|
|
54
|
-
record_fail "phase-3-dev.md missing"
|
|
55
|
-
else
|
|
56
|
-
if grep -qF "Step 3.5" "$PHASE_DOC" && grep -qF "Dev Critic" "$PHASE_DOC"; then
|
|
57
|
-
record_pass "phase-3-dev.md documents Step 3.5 - Dev Critic"
|
|
58
|
-
else
|
|
59
|
-
record_fail "phase-3-dev.md missing Step 3.5 section"
|
|
60
|
-
fi
|
|
61
|
-
if grep -qE "max(imum)? (\*\*)?2(\*\*)? iterations?" "$PHASE_DOC"; then
|
|
62
|
-
record_pass "phase doc documents max 2 iterations"
|
|
63
|
-
else
|
|
64
|
-
record_fail "phase doc must state 'max 2 iterations' rule"
|
|
65
|
-
fi
|
|
66
|
-
if grep -qF "escalate" "$PHASE_DOC"; then
|
|
67
|
-
record_pass "phase doc documents escalate path"
|
|
68
|
-
else
|
|
69
|
-
record_fail "phase doc must document escalate behavior"
|
|
70
|
-
fi
|
|
71
|
-
if grep -qF "dev-critic.md" "$PHASE_DOC"; then
|
|
72
|
-
record_pass "phase doc cites agents/dev-critic.md"
|
|
73
|
-
else
|
|
74
|
-
record_fail "phase doc must reference agents/dev-critic.md"
|
|
75
|
-
fi
|
|
76
|
-
fi
|
|
77
|
-
|
|
78
|
-
# 3. Prefs schema exposes devCritic toggle
|
|
79
|
-
if jq -e '.properties.global.properties.devCritic.properties.enabled' "$PREFS_SCHEMA" >/dev/null 2>&1; then
|
|
80
|
-
record_pass "prefs schema exposes devCritic.enabled"
|
|
81
|
-
else
|
|
82
|
-
record_fail "prefs schema missing devCritic.enabled"
|
|
83
|
-
fi
|
|
84
|
-
for prop in maxIterations model minDiffLoc; do
|
|
85
|
-
if jq -e ".properties.global.properties.devCritic.properties.${prop}" "$PREFS_SCHEMA" >/dev/null 2>&1; then
|
|
86
|
-
record_pass "prefs schema exposes devCritic.${prop}"
|
|
87
|
-
else
|
|
88
|
-
record_fail "prefs schema missing devCritic.${prop}"
|
|
89
|
-
fi
|
|
90
|
-
done
|
|
91
|
-
|
|
92
|
-
# 4. dev-critic-output schema is valid JSON
|
|
93
|
-
if [ ! -f "$CRITIC_SCHEMA" ]; then
|
|
94
|
-
record_fail "dev-critic-output.schema.json missing"
|
|
95
|
-
elif ! jq empty "$CRITIC_SCHEMA" 2>/dev/null; then
|
|
96
|
-
record_fail "dev-critic-output.schema.json is not valid JSON"
|
|
97
|
-
else
|
|
98
|
-
record_pass "dev-critic-output.schema.json parses"
|
|
99
|
-
fi
|
|
100
|
-
|
|
101
|
-
# 5. Severity enum matches doc (blocking/important/suggestion)
|
|
102
|
-
if jq -e '.properties.findings.items.properties.severity.enum
|
|
103
|
-
| (index("blocking") != null
|
|
104
|
-
and index("important") != null
|
|
105
|
-
and index("suggestion") != null)' "$CRITIC_SCHEMA" >/dev/null 2>&1; then
|
|
106
|
-
record_pass "schema severity enum includes blocking/important/suggestion"
|
|
107
|
-
else
|
|
108
|
-
record_fail "schema severity enum incomplete"
|
|
109
|
-
fi
|
|
110
|
-
|
|
111
|
-
# Gate keys (build/lint/test/secrets) match doc
|
|
112
|
-
for gate in build lint test secrets; do
|
|
113
|
-
if jq -e ".properties.gates.properties.${gate}" "$CRITIC_SCHEMA" >/dev/null 2>&1; then
|
|
114
|
-
record_pass "schema gates expose '${gate}'"
|
|
115
|
-
else
|
|
116
|
-
record_fail "schema gates missing '${gate}'"
|
|
117
|
-
fi
|
|
118
|
-
done
|
|
119
|
-
|
|
120
|
-
# 6. Loop cap default = 2
|
|
121
|
-
default_iter=$(jq -r '.properties.global.properties.devCritic.properties.maxIterations.default // empty' "$PREFS_SCHEMA")
|
|
122
|
-
if [ "$default_iter" = "2" ]; then
|
|
123
|
-
record_pass "devCritic.maxIterations defaults to 2"
|
|
124
|
-
else
|
|
125
|
-
record_fail "devCritic.maxIterations default should be 2 (was: ${default_iter:-missing})"
|
|
126
|
-
fi
|
|
127
|
-
|
|
128
|
-
# 7. Off by default - preserves existing-user baseline
|
|
129
|
-
# Note: jq's `// empty` treats boolean false as falsy, so use `has("default")` + raw value.
|
|
130
|
-
if jq -e '.properties.global.properties.devCritic.properties.enabled
|
|
131
|
-
| has("default") and .default == false' "$PREFS_SCHEMA" >/dev/null 2>&1; then
|
|
132
|
-
record_pass "devCritic.enabled defaults to false (opt-in)"
|
|
133
|
-
else
|
|
134
|
-
default_enabled=$(jq -c '.properties.global.properties.devCritic.properties.enabled.default' "$PREFS_SCHEMA")
|
|
135
|
-
record_fail "devCritic.enabled should default to false (was: ${default_enabled})"
|
|
136
|
-
fi
|
|
137
|
-
|
|
138
|
-
printf '\n══ dev-critic smoke: %d passed, %d failed ══\n' "$pass" "$fail"
|
|
139
|
-
if [ "$fail" -gt 0 ]; then
|
|
140
|
-
printf '\nFailures:\n'
|
|
141
|
-
for msg in "${failures[@]}"; do printf ' - %s\n' "$msg"; done
|
|
142
|
-
exit 1
|
|
143
|
-
fi
|
|
144
|
-
exit 0
|