@mmerterden/multi-agent-pipeline 12.7.0 → 12.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +207 -0
- package/install/_common.mjs +48 -0
- package/install/_dev-only-files.mjs +125 -5
- package/install/claude.mjs +14 -8
- package/install/copilot.mjs +5 -8
- package/package.json +17 -2
- package/pipeline/commands/multi-agent/analysis/SKILL.md +1 -1
- package/pipeline/lib/credential-store.sh +20 -0
- package/pipeline/multi-agent-refs/_account-picker.md +1 -1
- package/pipeline/multi-agent-refs/_dev-context.md +1 -1
- package/pipeline/multi-agent-refs/_repo-picker.md +1 -1
- package/pipeline/multi-agent-refs/features/review-multi-repo.md +65 -0
- package/pipeline/multi-agent-refs/features/url-enrichment.md +93 -0
- package/pipeline/multi-agent-refs/phases/operations.md +28 -0
- package/pipeline/multi-agent-refs/phases/phase-0-init.md +3 -83
- package/pipeline/multi-agent-refs/phases/phase-2-planning.md +1 -1
- package/pipeline/multi-agent-refs/phases/phase-3-dev.md +1 -2
- package/pipeline/multi-agent-refs/phases/phase-4-review.md +50 -62
- package/pipeline/schemas/prefs.schema.json +6 -0
- package/pipeline/scripts/_smoke-root.sh +61 -0
- package/pipeline/scripts/agent-guard.py +57 -3
- package/pipeline/scripts/audit-log.sh +25 -0
- package/pipeline/scripts/smoke-cross-cli-behavior.sh +21 -7
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/expected/phase-1-analysis.json +0 -25
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/expected/phase-2-plan.json +0 -30
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/expected/phase-4-review.json +0 -20
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/expected/phase-4-triage.json +0 -15
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/01-ios-bugfix-darkmode/task.json +0 -12
- package/pipeline/eval/golden-tasks/02-android-feature-compose/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/02-android-feature-compose/expected/phase-2-plan.json +0 -43
- package/pipeline/eval/golden-tasks/02-android-feature-compose/expected/phase-4-review.json +0 -35
- package/pipeline/eval/golden-tasks/02-android-feature-compose/expected/phase-4-triage.json +0 -35
- package/pipeline/eval/golden-tasks/02-android-feature-compose/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/02-android-feature-compose/task.json +0 -12
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/expected/phase-2-plan.json +0 -42
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/expected/phase-4-review.json +0 -20
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/expected/phase-4-triage.json +0 -15
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/03-backend-python-ratelimit/task.json +0 -12
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/expected/phase-2-plan.json +0 -40
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/expected/phase-4-review.json +0 -20
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/expected/phase-4-triage.json +0 -15
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/04-frontend-next-hydration/task.json +0 -12
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/expected/phase-2-plan.json +0 -42
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/expected/phase-4-review.json +0 -28
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/expected/phase-4-triage.json +0 -27
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/05-ios-security-keychain/task.json +0 -12
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/expected/phase-2-plan.json +0 -41
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/expected/phase-4-review.json +0 -12
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/expected/phase-4-triage.json +0 -6
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/06-android-refactor-usecase/task.json +0 -12
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/expected/phase-1-analysis.json +0 -29
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/expected/phase-2-plan.json +0 -42
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/expected/phase-4-review.json +0 -28
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/expected/phase-4-triage.json +0 -27
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/07-backend-node-idempotency/task.json +0 -12
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-1-analysis.json +0 -25
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-2-plan.json +0 -31
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-4-review.json +0 -12
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-4-triage.json +0 -18
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/metadata.json +0 -14
- package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/task.json +0 -12
- package/pipeline/eval/golden-tasks/README.md +0 -65
- package/pipeline/eval/intent-cases.json +0 -40
- package/pipeline/eval/run-metrics-fixture.json +0 -93
- package/pipeline/eval/triage/01-empty-findings/expected.json +0 -6
- package/pipeline/eval/triage/01-empty-findings/input.json +0 -5
- package/pipeline/eval/triage/01-empty-findings/notes.md +0 -7
- package/pipeline/eval/triage/02-real-blocker/expected.json +0 -15
- package/pipeline/eval/triage/02-real-blocker/input.json +0 -14
- package/pipeline/eval/triage/02-real-blocker/notes.md +0 -7
- package/pipeline/eval/triage/03-out-of-scope-defer/expected.json +0 -18
- package/pipeline/eval/triage/03-out-of-scope-defer/input.json +0 -14
- package/pipeline/eval/triage/03-out-of-scope-defer/notes.md +0 -10
- package/pipeline/eval/triage/04-false-positive-reject/expected.json +0 -18
- package/pipeline/eval/triage/04-false-positive-reject/input.json +0 -14
- package/pipeline/eval/triage/04-false-positive-reject/notes.md +0 -10
- package/pipeline/eval/triage/05-mixed-classification/expected.json +0 -43
- package/pipeline/eval/triage/05-mixed-classification/input.json +0 -38
- package/pipeline/eval/triage/05-mixed-classification/notes.md +0 -17
- package/pipeline/eval/triage/06-severity-mismatch/expected.json +0 -15
- package/pipeline/eval/triage/06-severity-mismatch/input.json +0 -14
- package/pipeline/eval/triage/06-severity-mismatch/notes.md +0 -9
- package/pipeline/eval/triage/07-duplicate-reviewers/expected.json +0 -27
- package/pipeline/eval/triage/07-duplicate-reviewers/input.json +0 -22
- package/pipeline/eval/triage/07-duplicate-reviewers/notes.md +0 -9
- package/pipeline/eval/triage/08-style-misclassified/expected.json +0 -18
- package/pipeline/eval/triage/08-style-misclassified/input.json +0 -14
- package/pipeline/eval/triage/08-style-misclassified/notes.md +0 -9
- package/pipeline/eval/triage/09-cascading-finding/expected.json +0 -23
- package/pipeline/eval/triage/09-cascading-finding/input.json +0 -22
- package/pipeline/eval/triage/09-cascading-finding/notes.md +0 -9
- package/pipeline/eval/triage/10-deferred-crossref/expected.json +0 -18
- package/pipeline/eval/triage/10-deferred-crossref/input.json +0 -14
- package/pipeline/eval/triage/10-deferred-crossref/notes.md +0 -9
- package/pipeline/eval/triage/11-vercel-token-leak-blocker/expected.json +0 -27
- package/pipeline/eval/triage/11-vercel-token-leak-blocker/input.json +0 -22
- package/pipeline/eval/triage/11-vercel-token-leak-blocker/notes.md +0 -14
- package/pipeline/eval/triage/README.md +0 -54
- package/pipeline/scripts/benchmark-phase-0.sh +0 -128
- package/pipeline/scripts/check-md-links.mjs +0 -88
- package/pipeline/scripts/eval-golden-tasks-live.mjs +0 -302
- package/pipeline/scripts/eval-golden-tasks.mjs +0 -224
- package/pipeline/scripts/eval-intent.mjs +0 -107
- package/pipeline/scripts/eval-mine-corpus.mjs +0 -211
- package/pipeline/scripts/eval-triage.mjs +0 -171
- package/pipeline/scripts/fixtures/diff-risk-android.diff +0 -40
- package/pipeline/scripts/fixtures/diff-risk-ios.diff +0 -48
- package/pipeline/scripts/fixtures/diff-risk-test-removal.diff +0 -40
- package/pipeline/scripts/fixtures/install-layout.tsv +0 -19
- package/pipeline/scripts/fixtures/pack-expected-count.txt +0 -1
- package/pipeline/scripts/fixtures/test-gap-node.diff +0 -30
- package/pipeline/scripts/fixtures/test-gap-python.diff +0 -32
- package/pipeline/scripts/lint-mcp-refs.mjs +0 -218
- package/pipeline/scripts/lint-skills.mjs +0 -154
- package/pipeline/scripts/run-smokes.mjs +0 -130
- package/pipeline/scripts/scorecard.mjs +0 -258
- package/pipeline/scripts/smoke-add-detail.sh +0 -137
- package/pipeline/scripts/smoke-agent-guard.sh +0 -74
- package/pipeline/scripts/smoke-agent-log-cost.sh +0 -262
- package/pipeline/scripts/smoke-agent-model-routing.sh +0 -87
- package/pipeline/scripts/smoke-ask-choice.sh +0 -42
- package/pipeline/scripts/smoke-autopilot-circuit-breaker.sh +0 -36
- package/pipeline/scripts/smoke-bitbucket-contract.sh +0 -255
- package/pipeline/scripts/smoke-changelog-version.sh +0 -47
- package/pipeline/scripts/smoke-channels-approval-gate.sh +0 -60
- package/pipeline/scripts/smoke-channels-flow.sh +0 -130
- package/pipeline/scripts/smoke-ci-workflows.sh +0 -88
- package/pipeline/scripts/smoke-clarify.sh +0 -148
- package/pipeline/scripts/smoke-command-inventory.sh +0 -81
- package/pipeline/scripts/smoke-commands-skills-parity.sh +0 -87
- package/pipeline/scripts/smoke-community-gates.sh +0 -75
- package/pipeline/scripts/smoke-compliance-skills.sh +0 -119
- package/pipeline/scripts/smoke-config-hygiene.sh +0 -58
- package/pipeline/scripts/smoke-context-budget.sh +0 -72
- package/pipeline/scripts/smoke-cost-budget.sh +0 -70
- package/pipeline/scripts/smoke-cost-summary.sh +0 -139
- package/pipeline/scripts/smoke-cross-phase-cohesion.sh +0 -128
- package/pipeline/scripts/smoke-description-tr.sh +0 -82
- package/pipeline/scripts/smoke-dev-critic.sh +0 -144
- package/pipeline/scripts/smoke-diff-explain.sh +0 -147
- package/pipeline/scripts/smoke-diff-risk.sh +0 -190
- package/pipeline/scripts/smoke-dynamic-skill-loading.sh +0 -160
- package/pipeline/scripts/smoke-eval-live.sh +0 -136
- package/pipeline/scripts/smoke-evidence-gate.sh +0 -93
- package/pipeline/scripts/smoke-extract-conventions.sh +0 -163
- package/pipeline/scripts/smoke-fetchers-offline.sh +0 -448
- package/pipeline/scripts/smoke-figma-dispatch.sh +0 -112
- package/pipeline/scripts/smoke-gate-hooks.sh +0 -74
- package/pipeline/scripts/smoke-gc-tmp.sh +0 -130
- package/pipeline/scripts/smoke-gc-worktrees.sh +0 -125
- package/pipeline/scripts/smoke-generate-issue.sh +0 -120
- package/pipeline/scripts/smoke-handoff-contract.sh +0 -92
- package/pipeline/scripts/smoke-identity-isolation.sh +0 -70
- package/pipeline/scripts/smoke-install-layout.sh +0 -248
- package/pipeline/scripts/smoke-intent-guard.sh +0 -86
- package/pipeline/scripts/smoke-issue-comment-template.sh +0 -86
- package/pipeline/scripts/smoke-issue-jira-triad.sh +0 -120
- package/pipeline/scripts/smoke-keychain.sh +0 -158
- package/pipeline/scripts/smoke-language-axis.sh +0 -109
- package/pipeline/scripts/smoke-learning-curve.sh +0 -61
- package/pipeline/scripts/smoke-learnings-ledger.sh +0 -86
- package/pipeline/scripts/smoke-lib-scripts.sh +0 -448
- package/pipeline/scripts/smoke-mcp-gate.sh +0 -68
- package/pipeline/scripts/smoke-md-links.sh +0 -8
- package/pipeline/scripts/smoke-md2confluence.sh +0 -126
- package/pipeline/scripts/smoke-metrics-cache-ratio.sh +0 -72
- package/pipeline/scripts/smoke-migrate-state.sh +0 -102
- package/pipeline/scripts/smoke-mode-dispatch-drift.sh +0 -161
- package/pipeline/scripts/smoke-model-fallback.sh +0 -89
- package/pipeline/scripts/smoke-multi-repo-integration.sh +0 -116
- package/pipeline/scripts/smoke-multi-repo-worktree.sh +0 -61
- package/pipeline/scripts/smoke-no-mcp-in-dev-phases.sh +0 -194
- package/pipeline/scripts/smoke-no-token-prompt.sh +0 -85
- package/pipeline/scripts/smoke-own-punctuation.sh +0 -103
- package/pipeline/scripts/smoke-pack-contents.sh +0 -140
- package/pipeline/scripts/smoke-pat-audit.sh +0 -128
- package/pipeline/scripts/smoke-per-repo-memory.sh +0 -156
- package/pipeline/scripts/smoke-phase-0-multi-repo.sh +0 -170
- package/pipeline/scripts/smoke-phase-6-multi.sh +0 -79
- package/pipeline/scripts/smoke-phase-banner.sh +0 -101
- package/pipeline/scripts/smoke-phase-tracker.sh +0 -324
- package/pipeline/scripts/smoke-phase0-bridge-contract.sh +0 -241
- package/pipeline/scripts/smoke-phase4-gates.sh +0 -45
- package/pipeline/scripts/smoke-phase4-triage.sh +0 -229
- package/pipeline/scripts/smoke-plan-approval-gate.sh +0 -71
- package/pipeline/scripts/smoke-plan-safety.sh +0 -139
- package/pipeline/scripts/smoke-plan-todos.sh +0 -196
- package/pipeline/scripts/smoke-pr-review-actions.sh +0 -152
- package/pipeline/scripts/smoke-pre-commit.sh +0 -170
- package/pipeline/scripts/smoke-pref-migration.sh +0 -226
- package/pipeline/scripts/smoke-prefs-language.sh +0 -134
- package/pipeline/scripts/smoke-progress-contract.sh +0 -127
- package/pipeline/scripts/smoke-prune-logs.sh +0 -137
- package/pipeline/scripts/smoke-purge.sh +0 -138
- package/pipeline/scripts/smoke-push-retry.sh +0 -75
- package/pipeline/scripts/smoke-repo-map.sh +0 -300
- package/pipeline/scripts/smoke-review-readiness.sh +0 -92
- package/pipeline/scripts/smoke-review-watch.sh +0 -146
- package/pipeline/scripts/smoke-routines.sh +0 -84
- package/pipeline/scripts/smoke-run-aggregator.sh +0 -216
- package/pipeline/scripts/smoke-run-metrics.sh +0 -50
- package/pipeline/scripts/smoke-search.sh +0 -187
- package/pipeline/scripts/smoke-shadow-git.sh +0 -224
- package/pipeline/scripts/smoke-skill-authoring.sh +0 -137
- package/pipeline/scripts/smoke-skill-language.sh +0 -83
- package/pipeline/scripts/smoke-skill-manifest.sh +0 -138
- package/pipeline/scripts/smoke-skill-scan.sh +0 -198
- package/pipeline/scripts/smoke-source-parity.sh +0 -85
- package/pipeline/scripts/smoke-subagent-validators.sh +0 -108
- package/pipeline/scripts/smoke-sync-parity.sh +0 -92
- package/pipeline/scripts/smoke-tasklist-ordering.sh +0 -112
- package/pipeline/scripts/smoke-telemetry.sh +0 -147
- package/pipeline/scripts/smoke-test-gap.sh +0 -183
- package/pipeline/scripts/smoke-token-budget.sh +0 -67
- package/pipeline/scripts/smoke-token-preflight.sh +0 -82
- package/pipeline/scripts/smoke-tracker-contract.sh +0 -191
- package/pipeline/scripts/smoke-tracker-tokens-invocation.sh +0 -73
- package/pipeline/scripts/smoke-triage-memory.sh +0 -174
- package/pipeline/scripts/smoke-update-check.sh +0 -135
- package/pipeline/scripts/smoke-url-enrichment.sh +0 -70
- package/pipeline/scripts/smoke-validate-analysis-doc.sh +0 -161
- package/pipeline/scripts/smoke-validator-contradiction.sh +0 -67
- package/pipeline/scripts/smoke-validator-gates.sh +0 -164
- package/pipeline/scripts/smoke-vercel-deploy-redact.sh +0 -129
- package/pipeline/scripts/smoke-verify-by-test.sh +0 -148
- package/pipeline/scripts/smoke-wiki-integration.sh +0 -122
- package/pipeline/scripts/smoke-work-summary.sh +0 -163
- package/pipeline/scripts/smoke-workflow-audit.sh +0 -101
- package/pipeline/scripts/smoke-worktree-path-convention.sh +0 -86
- package/pipeline/scripts/smoke-wrapper-preservation.sh +0 -68
- package/pipeline/scripts/smoke-write-state.sh +0 -159
- package/pipeline/scripts/sync-parity-check.sh +0 -135
- package/pipeline/scripts/test-gap-rules/android.json +0 -25
- package/pipeline/scripts/test-gap-rules/ios.json +0 -34
- package/pipeline/scripts/test-gap-rules/node.json +0 -29
- package/pipeline/scripts/test-gap-rules/python.json +0 -25
- package/pipeline/scripts/validate-schemas.mjs +0 -88
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# smoke-metrics-cache-ratio.sh - aggregate-metrics.mjs surfaces prompt-cache
|
|
3
|
-
# reuse. The data path (log-metric.sh accepts tokens_cached) already exists; this
|
|
4
|
-
# guards that the aggregator actually reads and reports it per model + overall,
|
|
5
|
-
# in all three output modes (json / markdown / text).
|
|
6
|
-
#
|
|
7
|
-
# Exit 0 = all pass, 1 = any failure.
|
|
8
|
-
|
|
9
|
-
set -uo pipefail
|
|
10
|
-
|
|
11
|
-
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
12
|
-
AGG="$ROOT/pipeline/scripts/aggregate-metrics.mjs"
|
|
13
|
-
PASS=0; FAIL=0
|
|
14
|
-
pass() { PASS=$((PASS+1)); echo " ✓ $1"; }
|
|
15
|
-
fail() { FAIL=$((FAIL+1)); echo " ✗ $1"; }
|
|
16
|
-
command -v node >/dev/null 2>&1 || { echo "error: node required" >&2; exit 127; }
|
|
17
|
-
[ -f "$AGG" ] || { echo "FAIL: aggregate-metrics.mjs missing" >&2; exit 1; }
|
|
18
|
-
|
|
19
|
-
cleanup() { [ -n "${TMP:-}" ] && rm -rf "$TMP"; }
|
|
20
|
-
trap cleanup EXIT
|
|
21
|
-
TMP=$(mktemp -d)
|
|
22
|
-
|
|
23
|
-
FIX="$TMP/metrics.jsonl"
|
|
24
|
-
# opus: 80k fresh input + 20k cached = cache_ratio 0.20
|
|
25
|
-
# sonnet: 50k fresh input + 50k cached = cache_ratio 0.50
|
|
26
|
-
# overall: 30k cached / (130k in + 30k... ) -> computed below by the script.
|
|
27
|
-
{
|
|
28
|
-
echo '{"ts":"2026-06-19T10:00:00Z","task_id":"T-1","phase":"4","event":"review.completed","details":{"model":"opus","tokens_in":80000,"tokens_out":4000,"tokens_cached":20000}}'
|
|
29
|
-
echo '{"ts":"2026-06-19T10:01:00Z","task_id":"T-1","phase":"3","event":"dev.completed","details":{"model":"sonnet","tokens_in":50000,"tokens_out":6000,"tokens_cached":50000}}'
|
|
30
|
-
} > "$FIX"
|
|
31
|
-
|
|
32
|
-
echo "→ 1. JSON: per-model cache_ratio computed"
|
|
33
|
-
out=$(METRICS_FILE="$FIX" node "$AGG" --json 2>/dev/null)
|
|
34
|
-
check_json() { if node -e "const s=require('$TMP/out.json'); process.exit($1?0:1)" 2>/dev/null; then pass "$2"; else fail "$2"; fi; }
|
|
35
|
-
printf '%s' "$out" > "$TMP/out.json"
|
|
36
|
-
check_json 's.cost_per_model.opus.tokens_cached===20000' "opus cached tokens summed"
|
|
37
|
-
check_json 's.cost_per_model.opus.cache_ratio===0.2' "opus cache_ratio = 0.20 (20k/100k)"
|
|
38
|
-
check_json 's.cost_per_model.sonnet.cache_ratio===0.5' "sonnet cache_ratio = 0.50 (50k/100k)"
|
|
39
|
-
|
|
40
|
-
echo ""
|
|
41
|
-
echo "→ 2. JSON: overall cache_reuse block"
|
|
42
|
-
check_json 's.cache_reuse.tokens_cached===70000' "overall cached = 70k"
|
|
43
|
-
check_json 's.cache_reuse.tokens_in===130000' "overall fresh input = 130k"
|
|
44
|
-
check_json 's.cache_reuse.cache_ratio===0.35' "overall cache_ratio = 0.35 (70k/200k)"
|
|
45
|
-
|
|
46
|
-
echo ""
|
|
47
|
-
echo "→ 3. markdown + text modes render cache column"
|
|
48
|
-
md=$(METRICS_FILE="$FIX" node "$AGG" --markdown 2>/dev/null)
|
|
49
|
-
case "$md" in *"Cache %"*) pass "markdown has Cache % column" ;; *) fail "markdown missing Cache % column" ;; esac
|
|
50
|
-
case "$md" in *"prompt-cache reuse"*) pass "markdown has overall reuse line" ;; *) fail "markdown missing overall reuse line" ;; esac
|
|
51
|
-
txt=$(METRICS_FILE="$FIX" node "$AGG" 2>/dev/null)
|
|
52
|
-
case "$txt" in *"cache%"*) pass "text has cache% column" ;; *) fail "text missing cache% column" ;; esac
|
|
53
|
-
case "$txt" in *"overall prompt-cache reuse"*) pass "text has overall reuse line" ;; *) fail "text missing overall reuse line" ;; esac
|
|
54
|
-
|
|
55
|
-
echo ""
|
|
56
|
-
echo "→ 4. no-cache data: ratio is 0% (input but no cache), null only when no input"
|
|
57
|
-
NOFIX="$TMP/nocache.jsonl"
|
|
58
|
-
{
|
|
59
|
-
# has input, zero cache -> ratio 0.00 (meaningful: 0% cache hit)
|
|
60
|
-
echo '{"ts":"2026-06-19T10:00:00Z","task_id":"T-2","phase":"4","event":"review.completed","details":{"model":"haiku","tokens_in":1000,"tokens_out":100}}'
|
|
61
|
-
# model tagged but no token counts at all -> denom 0 -> ratio null (not NaN)
|
|
62
|
-
echo '{"ts":"2026-06-19T10:00:01Z","task_id":"T-2","phase":"0","event":"init.completed","details":{"model":"none"}}'
|
|
63
|
-
} > "$NOFIX"
|
|
64
|
-
nout=$(METRICS_FILE="$NOFIX" node "$AGG" --json 2>/dev/null)
|
|
65
|
-
printf '%s' "$nout" > "$TMP/out.json"
|
|
66
|
-
check_json 's.cost_per_model.haiku.tokens_cached===0' "absent cached defaults to 0"
|
|
67
|
-
check_json 's.cost_per_model.haiku.cache_ratio===0' "input-but-no-cache ratio is 0 (0% hit)"
|
|
68
|
-
check_json 's.cost_per_model.none.cache_ratio===null' "no-input model ratio is null (not NaN)"
|
|
69
|
-
|
|
70
|
-
echo ""
|
|
71
|
-
echo "══ metrics cache-ratio smoke: $PASS passed, $FAIL failed ══"
|
|
72
|
-
[ "$FAIL" -eq 0 ]
|
|
@@ -1,102 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# smoke-migrate-state.sh - verify migrate-state.mjs behavior with synthetic
|
|
3
|
-
# migrations. Does not depend on any real migration file existing - it lays
|
|
4
|
-
# one down in a temp dir and points the runner at a fixture.
|
|
5
|
-
|
|
6
|
-
set -uo pipefail
|
|
7
|
-
|
|
8
|
-
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
9
|
-
RUNNER="$SCRIPT_DIR/migrate-state.mjs"
|
|
10
|
-
|
|
11
|
-
if [ ! -f "$RUNNER" ]; then
|
|
12
|
-
echo "FAIL: migrate-state.mjs not found" >&2
|
|
13
|
-
exit 1
|
|
14
|
-
fi
|
|
15
|
-
|
|
16
|
-
PASS=0
|
|
17
|
-
FAIL=0
|
|
18
|
-
pass() { PASS=$((PASS+1)); echo " ✓ $1"; }
|
|
19
|
-
fail() { FAIL=$((FAIL+1)); echo " ✗ $1"; }
|
|
20
|
-
|
|
21
|
-
TMPDIR=$(mktemp -d) || { echo "FAIL: mktemp" >&2; exit 1; }
|
|
22
|
-
trap 'rm -rf "$TMPDIR"' EXIT
|
|
23
|
-
|
|
24
|
-
# Test 1 - file at current version exits 0, no migration needed
|
|
25
|
-
cat > "$TMPDIR/agent-state.json" <<'EOF'
|
|
26
|
-
{"schemaVersion":"2.1.0","foo":"bar"}
|
|
27
|
-
EOF
|
|
28
|
-
OUT=$(node "$RUNNER" "$TMPDIR/agent-state.json" 2>&1)
|
|
29
|
-
if echo "$OUT" | grep -q "already at schemaVersion 2.1.0"; then
|
|
30
|
-
pass "current-version file is a no-op"
|
|
31
|
-
else
|
|
32
|
-
fail "current-version file should be a no-op (got: $OUT)"
|
|
33
|
-
fi
|
|
34
|
-
|
|
35
|
-
# Test 2 - missing schemaVersion exits 1
|
|
36
|
-
echo '{"foo":"bar"}' > "$TMPDIR/bad.json"
|
|
37
|
-
node "$RUNNER" --type=state "$TMPDIR/bad.json" 2>/dev/null
|
|
38
|
-
if [ $? -eq 1 ]; then
|
|
39
|
-
pass "missing schemaVersion exits 1"
|
|
40
|
-
else
|
|
41
|
-
fail "missing schemaVersion did not exit 1"
|
|
42
|
-
fi
|
|
43
|
-
|
|
44
|
-
# Test 3 - invalid JSON exits 1
|
|
45
|
-
echo 'not-json' > "$TMPDIR/junk.json"
|
|
46
|
-
node "$RUNNER" --type=state "$TMPDIR/junk.json" 2>/dev/null
|
|
47
|
-
if [ $? -eq 1 ]; then
|
|
48
|
-
pass "invalid JSON exits 1"
|
|
49
|
-
else
|
|
50
|
-
fail "invalid JSON did not exit 1"
|
|
51
|
-
fi
|
|
52
|
-
|
|
53
|
-
# Test 4 - missing file exits 1
|
|
54
|
-
node "$RUNNER" --type=state "$TMPDIR/does-not-exist.json" 2>/dev/null
|
|
55
|
-
if [ $? -eq 1 ]; then
|
|
56
|
-
pass "missing file exits 1"
|
|
57
|
-
else
|
|
58
|
-
fail "missing file did not exit 1"
|
|
59
|
-
fi
|
|
60
|
-
|
|
61
|
-
# Test 5 - no migration path (older version than any migration)
|
|
62
|
-
echo '{"schemaVersion":"0.1.0"}' > "$TMPDIR/ancient.json"
|
|
63
|
-
node "$RUNNER" --type=state "$TMPDIR/ancient.json" 2>/dev/null
|
|
64
|
-
if [ $? -eq 2 ]; then
|
|
65
|
-
pass "no migration path exits 2"
|
|
66
|
-
else
|
|
67
|
-
fail "no migration path did not exit 2"
|
|
68
|
-
fi
|
|
69
|
-
|
|
70
|
-
# Test 6 - unknown flag exits 1
|
|
71
|
-
node "$RUNNER" --nonsense "$TMPDIR/agent-state.json" 2>/dev/null
|
|
72
|
-
if [ $? -eq 1 ]; then
|
|
73
|
-
pass "unknown flag exits 1"
|
|
74
|
-
else
|
|
75
|
-
fail "unknown flag did not exit 1"
|
|
76
|
-
fi
|
|
77
|
-
|
|
78
|
-
# Test 7 - type auto-detection works (feed a doc already at the latest version)
|
|
79
|
-
cat > "$TMPDIR/multi-agent-preferences.json" <<'EOF'
|
|
80
|
-
{"schemaVersion":"2.4.0","global":{"identities":[],"keychainMapping":{},"recentAccounts":[],"accounts":[],"routines":[]},"projects":{}}
|
|
81
|
-
EOF
|
|
82
|
-
OUT=$(node "$RUNNER" "$TMPDIR/multi-agent-preferences.json" 2>&1)
|
|
83
|
-
if echo "$OUT" | grep -q "already at schemaVersion 2.4.0"; then
|
|
84
|
-
pass "prefs type auto-detected from filename"
|
|
85
|
-
else
|
|
86
|
-
fail "prefs auto-detection failed (got: $OUT)"
|
|
87
|
-
fi
|
|
88
|
-
|
|
89
|
-
# Test 8 - ambiguous path requires --type
|
|
90
|
-
cat > "$TMPDIR/ambig.json" <<'EOF'
|
|
91
|
-
{"schemaVersion":"2.0.0"}
|
|
92
|
-
EOF
|
|
93
|
-
node "$RUNNER" "$TMPDIR/ambig.json" 2>/dev/null
|
|
94
|
-
if [ $? -eq 1 ]; then
|
|
95
|
-
pass "ambiguous filename requires --type"
|
|
96
|
-
else
|
|
97
|
-
fail "ambiguous filename did not require --type"
|
|
98
|
-
fi
|
|
99
|
-
|
|
100
|
-
echo ""
|
|
101
|
-
echo "══ migrate-state smoke: $PASS passed, $FAIL failed ══"
|
|
102
|
-
[ "$FAIL" -eq 0 ] || exit 1
|
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
#
|
|
3
|
-
# smoke-mode-dispatch-drift.sh - assert each mode dispatch file's tracker
|
|
4
|
-
# section is consistent with the canonical generator output.
|
|
5
|
-
#
|
|
6
|
-
# Each mode dispatch file (dev/SKILL.md, dev-autopilot/SKILL.md, dev-local/SKILL.md,
|
|
7
|
-
# dev-local-autopilot/SKILL.md, autopilot/SKILL.md, local/SKILL.md, local-autopilot/SKILL.md) must
|
|
8
|
-
# carry an inline "## Required: Phase Tracker Contract" section. v7.9.1 added
|
|
9
|
-
# this contract; v8.0.0 added gen-mode-dispatch.mjs as the canonical
|
|
10
|
-
# generator.
|
|
11
|
-
#
|
|
12
|
-
# As of v8.0.0 this smoke ENFORCES byte equality with the generator output.
|
|
13
|
-
# Each mode dispatch file's tracker section must match `node gen-mode-dispatch.mjs
|
|
14
|
-
# --mode=<name>` exactly. The generator is the source of truth; the on-disk
|
|
15
|
-
# section is regenerated from it, never hand-edited.
|
|
16
|
-
#
|
|
17
|
-
# Mode → mode-name mapping (each pair is `<file>:<gen-mode>`):
|
|
18
|
-
# dev/SKILL.md → dev
|
|
19
|
-
# dev-autopilot/SKILL.md → dev-autopilot
|
|
20
|
-
# dev-local/SKILL.md → dev-local
|
|
21
|
-
# dev-local-autopilot/SKILL.md → dev-local-autopilot
|
|
22
|
-
# autopilot/SKILL.md → autopilot
|
|
23
|
-
# local/SKILL.md → local
|
|
24
|
-
# local-autopilot/SKILL.md → local-autopilot
|
|
25
|
-
#
|
|
26
|
-
# Exit codes: 0=pass, 1=drift detected
|
|
27
|
-
set -euo pipefail
|
|
28
|
-
|
|
29
|
-
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
30
|
-
CMD_DIR="$REPO_ROOT/pipeline/commands/multi-agent"
|
|
31
|
-
GEN="$REPO_ROOT/pipeline/scripts/gen-mode-dispatch.mjs"
|
|
32
|
-
|
|
33
|
-
[ -f "$GEN" ] || { echo "FAIL: gen-mode-dispatch.mjs missing"; exit 2; }
|
|
34
|
-
|
|
35
|
-
declare -a MODE_FAST_WITH_TEST=("dev/SKILL.md")
|
|
36
|
-
declare -a MODE_FAST_NO_TEST=("dev-autopilot/SKILL.md" "dev-local/SKILL.md" "dev-local-autopilot/SKILL.md")
|
|
37
|
-
declare -a MODE_FAST=("${MODE_FAST_WITH_TEST[@]}" "${MODE_FAST_NO_TEST[@]}")
|
|
38
|
-
declare -a MODE_FULL=("autopilot/SKILL.md" "local/SKILL.md" "local-autopilot/SKILL.md")
|
|
39
|
-
|
|
40
|
-
PASS=0
|
|
41
|
-
FAIL=0
|
|
42
|
-
pass() { echo " ✓ $*"; PASS=$((PASS + 1)); }
|
|
43
|
-
fail() { echo " ✗ $*"; FAIL=$((FAIL + 1)); }
|
|
44
|
-
|
|
45
|
-
assert_section_present() {
|
|
46
|
-
local file="$1"
|
|
47
|
-
local count
|
|
48
|
-
count=$(grep -c '^## Required: Phase Tracker Contract' "$CMD_DIR/$file" || true)
|
|
49
|
-
case "$count" in
|
|
50
|
-
1) pass "$file declares tracker section exactly once" ;;
|
|
51
|
-
0) fail "$file is missing the tracker section" ;;
|
|
52
|
-
*) fail "$file has $count tracker sections (must be exactly 1)" ;;
|
|
53
|
-
esac
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
assert_phases() {
|
|
57
|
-
local file="$1"
|
|
58
|
-
shift
|
|
59
|
-
local expected_phases=("$@")
|
|
60
|
-
local section
|
|
61
|
-
section=$(awk '/^## Required: Phase Tracker Contract/{p=1} p{print} /^## /&&!/Required: Phase Tracker Contract/{p=0}' "$CMD_DIR/$file")
|
|
62
|
-
for ph in "${expected_phases[@]}"; do
|
|
63
|
-
if echo "$section" | grep -Fq "\"$ph\""; then
|
|
64
|
-
pass "$file declares phase $ph"
|
|
65
|
-
else
|
|
66
|
-
fail "$file missing phase $ph (expected for its mode family)"
|
|
67
|
-
fi
|
|
68
|
-
done
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
assert_ref_link() {
|
|
72
|
-
local file="$1"
|
|
73
|
-
if grep -Fq 'refs/tracker-contract.md' "$CMD_DIR/$file"; then
|
|
74
|
-
pass "$file links the tracker contract ref"
|
|
75
|
-
else
|
|
76
|
-
fail "$file missing tracker-contract.md link"
|
|
77
|
-
fi
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
echo "→ 1. Mode files declare the tracker contract section exactly once"
|
|
81
|
-
for f in "${MODE_FAST[@]}" "${MODE_FULL[@]}"; do
|
|
82
|
-
assert_section_present "$f"
|
|
83
|
-
done
|
|
84
|
-
|
|
85
|
-
echo ""
|
|
86
|
-
echo "→ 2a. Fast-mode files that keep User Test (dev/SKILL.md) declare 5 phases (0/3/5/6/7)"
|
|
87
|
-
for f in "${MODE_FAST_WITH_TEST[@]}"; do
|
|
88
|
-
assert_phases "$f" "0:Init" "3:Dev" "5:Test" "6:Commit" "7:Report"
|
|
89
|
-
done
|
|
90
|
-
|
|
91
|
-
echo ""
|
|
92
|
-
echo "→ 2b. Fast-mode files that skip User Test (dev-autopilot/local/local-autopilot) declare 4 phases (0/3/6/7)"
|
|
93
|
-
for f in "${MODE_FAST_NO_TEST[@]}"; do
|
|
94
|
-
assert_phases "$f" "0:Init" "3:Dev" "6:Commit" "7:Report"
|
|
95
|
-
done
|
|
96
|
-
|
|
97
|
-
echo ""
|
|
98
|
-
echo "→ 3. Autopilot/local full-pipeline files declare 7 phases (skip User Test)"
|
|
99
|
-
# Phase 5 (User Test) needs an interactive prompt AND a worktree checkout, so it
|
|
100
|
-
# runs only in the interactive worktree-backed modes (dev, full). Every autopilot
|
|
101
|
-
# or local variant drops it.
|
|
102
|
-
for f in "${MODE_FULL[@]}"; do
|
|
103
|
-
assert_phases "$f" "0:Init" "1:Analysis" "2:Planning" "3:Dev" "4:Review" "6:Commit" "7:Report"
|
|
104
|
-
done
|
|
105
|
-
|
|
106
|
-
echo ""
|
|
107
|
-
echo "→ 4. Every mode file links refs/tracker-contract.md"
|
|
108
|
-
for f in "${MODE_FAST[@]}" "${MODE_FULL[@]}"; do
|
|
109
|
-
assert_ref_link "$f"
|
|
110
|
-
done
|
|
111
|
-
|
|
112
|
-
echo ""
|
|
113
|
-
echo "→ 5. Generator emits valid output for each mode"
|
|
114
|
-
for mode in dev dev-autopilot dev-local dev-local-autopilot autopilot full local local-autopilot full-local; do
|
|
115
|
-
if node "$GEN" --mode="$mode" >/dev/null 2>&1; then
|
|
116
|
-
pass "gen-mode-dispatch.mjs --mode=$mode emits cleanly"
|
|
117
|
-
else
|
|
118
|
-
fail "gen-mode-dispatch.mjs --mode=$mode crashed or non-zero"
|
|
119
|
-
fi
|
|
120
|
-
done
|
|
121
|
-
|
|
122
|
-
if node "$GEN" --mode=bogus >/dev/null 2>&1; then
|
|
123
|
-
fail "gen-mode-dispatch.mjs accepted unknown mode 'bogus' - exit guard missing"
|
|
124
|
-
else
|
|
125
|
-
pass "gen-mode-dispatch.mjs rejects unknown mode (exit 2)"
|
|
126
|
-
fi
|
|
127
|
-
|
|
128
|
-
echo ""
|
|
129
|
-
echo "→ 6. Each mode file's tracker section is byte-equal to the generator output"
|
|
130
|
-
# This is the strict gate: the generator is the source of truth, on-disk
|
|
131
|
-
# sections are regenerated from it. Drift here means someone hand-edited a
|
|
132
|
-
# mode file's tracker section without updating the generator. Fix:
|
|
133
|
-
# node pipeline/scripts/gen-mode-dispatch.mjs --mode=<m> > /tmp/gen.md
|
|
134
|
-
# <splice into the file's tracker section, or run the helper sync script>
|
|
135
|
-
for pair in "dev/SKILL.md:dev" "dev-autopilot/SKILL.md:dev-autopilot" "dev-local/SKILL.md:dev-local" \
|
|
136
|
-
"dev-local-autopilot/SKILL.md:dev-local-autopilot" "autopilot/SKILL.md:autopilot" \
|
|
137
|
-
"local/SKILL.md:local" "local-autopilot/SKILL.md:local-autopilot"; do
|
|
138
|
-
file="${pair%:*}"
|
|
139
|
-
mode="${pair#*:}"
|
|
140
|
-
path="$CMD_DIR/$file"
|
|
141
|
-
expected=$(node "$GEN" --mode="$mode")
|
|
142
|
-
# Extract on-disk section: from "## Required: Phase Tracker Contract" up to
|
|
143
|
-
# (but not including) the next top-level "## " heading. Trim trailing blank
|
|
144
|
-
# lines so the generator (which ends with one trailing newline) matches.
|
|
145
|
-
actual=$(awk '
|
|
146
|
-
/^## Required: Phase Tracker Contract/ { p=1 }
|
|
147
|
-
p && /^## / && !/^## Required: Phase Tracker Contract/ { exit }
|
|
148
|
-
p { print }
|
|
149
|
-
' "$path")
|
|
150
|
-
if [ "$(printf '%s' "$expected" | sha256sum | cut -d" " -f1)" = \
|
|
151
|
-
"$(printf '%s' "$actual" | sha256sum | cut -d" " -f1)" ]; then
|
|
152
|
-
pass "$file tracker section matches gen-mode-dispatch.mjs --mode=$mode"
|
|
153
|
-
else
|
|
154
|
-
fail "$file tracker section drift vs --mode=$mode"
|
|
155
|
-
diff <(printf '%s\n' "$expected") <(printf '%s\n' "$actual") | head -20 | sed 's/^/ /'
|
|
156
|
-
fi
|
|
157
|
-
done
|
|
158
|
-
|
|
159
|
-
echo ""
|
|
160
|
-
echo "══ smoke-mode-dispatch-drift: $PASS passed, $FAIL failed ══"
|
|
161
|
-
[ "$FAIL" -gt 0 ] && exit 1 || exit 0
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# smoke-model-fallback.sh - lint the model fallback contract wiring (v10.1.0).
|
|
3
|
-
# The contract lets preferredModel (opus) personas degrade to sonnet on a
|
|
4
|
-
# plan-window date gate, a dispatch error, or a budget ceiling - via
|
|
5
|
-
# PHASE_MODEL_OVERRIDE, never by editing persona files. This smoke asserts the
|
|
6
|
-
# doc, the prefs knob, and the phase wiring all exist and agree. No network,
|
|
7
|
-
# no credentials.
|
|
8
|
-
|
|
9
|
-
set -uo pipefail
|
|
10
|
-
|
|
11
|
-
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
12
|
-
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
13
|
-
DOC="$REPO_ROOT/pipeline/multi-agent-refs/features/model-fallback.md"
|
|
14
|
-
TEMPLATE="$REPO_ROOT/pipeline/preferences-template.json"
|
|
15
|
-
P0="$REPO_ROOT/pipeline/multi-agent-refs/phases/phase-0-init.md"
|
|
16
|
-
P4="$REPO_ROOT/pipeline/multi-agent-refs/phases/phase-4-review.md"
|
|
17
|
-
|
|
18
|
-
PASS=0
|
|
19
|
-
FAIL=0
|
|
20
|
-
ok() { PASS=$((PASS+1)); echo " ✓ $1"; }
|
|
21
|
-
fail() { FAIL=$((FAIL+1)); echo " ✗ $1" >&2; }
|
|
22
|
-
|
|
23
|
-
echo "→ 1. Contract doc exists and covers the three triggers"
|
|
24
|
-
if [ -f "$DOC" ]; then
|
|
25
|
-
ok "model-fallback.md present"
|
|
26
|
-
grep -q "premiumTierUntil" "$DOC" && ok "doc documents premiumTierUntil date gate" || fail "doc missing premiumTierUntil"
|
|
27
|
-
grep -q "onDispatchError" "$DOC" && ok "doc documents dispatch-error retry" || fail "doc missing onDispatchError"
|
|
28
|
-
grep -q "cost-budget-check" "$DOC" && ok "doc documents budget-ceiling trigger" || fail "doc missing budget trigger"
|
|
29
|
-
grep -q "PHASE_MODEL_OVERRIDE" "$DOC" && ok "doc uses the existing override mechanism" || fail "doc missing PHASE_MODEL_OVERRIDE"
|
|
30
|
-
grep -q "opus -> sonnet" "$DOC" && ok "tier ladder declared" || fail "tier ladder missing"
|
|
31
|
-
else
|
|
32
|
-
fail "model-fallback.md missing"
|
|
33
|
-
fi
|
|
34
|
-
|
|
35
|
-
echo "→ 2. Prefs template carries the knob with safe defaults"
|
|
36
|
-
if command -v python3 >/dev/null 2>&1; then
|
|
37
|
-
RESULT=$(python3 - "$TEMPLATE" <<'PYEOF'
|
|
38
|
-
import json, sys
|
|
39
|
-
t = json.load(open(sys.argv[1]))
|
|
40
|
-
mf = t.get("global", {}).get("modelFallback")
|
|
41
|
-
if not isinstance(mf, dict):
|
|
42
|
-
print("missing"); raise SystemExit
|
|
43
|
-
checks = [
|
|
44
|
-
mf.get("enabled") is True,
|
|
45
|
-
"premiumTierUntil" in mf and mf["premiumTierUntil"] is None,
|
|
46
|
-
mf.get("fallbackModel") == "sonnet",
|
|
47
|
-
mf.get("onDispatchError") is True,
|
|
48
|
-
]
|
|
49
|
-
print("ok" if all(checks) else "bad-defaults")
|
|
50
|
-
PYEOF
|
|
51
|
-
)
|
|
52
|
-
case "$RESULT" in
|
|
53
|
-
ok) ok "template modelFallback knob present with safe defaults" ;;
|
|
54
|
-
missing) fail "template missing global.modelFallback" ;;
|
|
55
|
-
*) fail "template modelFallback defaults wrong ($RESULT)" ;;
|
|
56
|
-
esac
|
|
57
|
-
else
|
|
58
|
-
echo " (python3 unavailable - template check skipped)"
|
|
59
|
-
fi
|
|
60
|
-
|
|
61
|
-
echo "→ 3. Phase wiring references the contract"
|
|
62
|
-
grep -q "model-fallback.md" "$P0" && ok "phase-0-init wires the date gate" || fail "phase-0-init missing fallback wiring"
|
|
63
|
-
grep -q "model-fallback.md" "$P4" && ok "phase-4-review wires the per-dispatch triggers" || fail "phase-4-review missing fallback wiring"
|
|
64
|
-
|
|
65
|
-
echo "→ 4. Top-tier personas declare their preferred model (fallback is dispatch-time, not file-time)"
|
|
66
|
-
# Fable 5 is the top tier again: architect + Reviewer-1 (+ triage) personas pin it.
|
|
67
|
-
FABLE_COUNT=$(grep -l "^preferredModel: fable" "$REPO_ROOT"/pipeline/agents/*.md 2>/dev/null | wc -l | tr -d ' ')
|
|
68
|
-
if [ "$FABLE_COUNT" -ge 4 ]; then
|
|
69
|
-
ok "fable top-tier personas present ($FABLE_COUNT files)"
|
|
70
|
-
else
|
|
71
|
-
fail "expected >=4 fable personas (architects + Reviewer-1), found $FABLE_COUNT"
|
|
72
|
-
fi
|
|
73
|
-
# Opus stays a valid tier - the first fallback below fable, and the top tier for security personas.
|
|
74
|
-
OPUS_COUNT=$(grep -l "^preferredModel: opus" "$REPO_ROOT"/pipeline/agents/*.md 2>/dev/null | wc -l | tr -d ' ')
|
|
75
|
-
if [ "$OPUS_COUNT" -ge 1 ]; then
|
|
76
|
-
ok "opus personas intact ($OPUS_COUNT files)"
|
|
77
|
-
else
|
|
78
|
-
fail "expected >=1 opus persona (fallback tier / security), found $OPUS_COUNT"
|
|
79
|
-
fi
|
|
80
|
-
|
|
81
|
-
echo ""
|
|
82
|
-
TOTAL=$((PASS+FAIL))
|
|
83
|
-
if [ $FAIL -eq 0 ]; then
|
|
84
|
-
echo "══ model-fallback smoke: $PASS passed, 0 failed ══"
|
|
85
|
-
exit 0
|
|
86
|
-
else
|
|
87
|
-
echo "══ model-fallback smoke: $PASS passed, $FAIL failed (of $TOTAL) ══" >&2
|
|
88
|
-
exit 1
|
|
89
|
-
fi
|
|
@@ -1,116 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# smoke-multi-repo-integration.sh - verify v5.6.0 multi-repo integration-build
|
|
3
|
-
# contract: prefs schema accepts valid entries, rejects malformed ones, and the
|
|
4
|
-
# contract doc ships with the required sections (lookup logic, learn-once
|
|
5
|
-
# prompt, autopilot refuse-to-prompt, autopilot refuse-to-prompt behavior).
|
|
6
|
-
#
|
|
7
|
-
# This smoke validates STRUCTURE + CONTRACT. Integration-build execution logic
|
|
8
|
-
# itself is LLM-orchestrated (the pipeline is the caller), so behavior tests
|
|
9
|
-
# live in the integration-test tier, not smoke.
|
|
10
|
-
|
|
11
|
-
set -uo pipefail
|
|
12
|
-
|
|
13
|
-
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
14
|
-
ROOT="$(cd "$HERE/../.." && pwd)"
|
|
15
|
-
SCHEMA="$ROOT/pipeline/schemas/prefs.schema.json"
|
|
16
|
-
DOC="$ROOT/pipeline/multi-agent-refs/multi-repo-integration-build.md"
|
|
17
|
-
|
|
18
|
-
PASS=0
|
|
19
|
-
FAIL=0
|
|
20
|
-
pass() { PASS=$((PASS+1)); echo " ✓ $1"; }
|
|
21
|
-
fail() { FAIL=$((FAIL+1)); echo " ✗ $1"; }
|
|
22
|
-
|
|
23
|
-
command -v jq >/dev/null || { echo "FAIL: jq required" >&2; exit 1; }
|
|
24
|
-
|
|
25
|
-
echo "→ 1. prefs.schema.json defines multiRepoIntegrationHosts"
|
|
26
|
-
jq -e '.properties.global.properties.multiRepoIntegrationHosts' "$SCHEMA" >/dev/null 2>&1 \
|
|
27
|
-
&& pass "schema has global.multiRepoIntegrationHosts" \
|
|
28
|
-
|| fail "schema missing multiRepoIntegrationHosts"
|
|
29
|
-
|
|
30
|
-
echo ""
|
|
31
|
-
echo "→ 2. schema item has required fields (repoSet) and proper constraints"
|
|
32
|
-
SCHEMA_ITEM=$(jq '.properties.global.properties.multiRepoIntegrationHosts.items' "$SCHEMA")
|
|
33
|
-
echo "$SCHEMA_ITEM" | jq -e '.required | index("repoSet")' >/dev/null 2>&1 \
|
|
34
|
-
&& pass "item requires repoSet" || fail "item should require repoSet"
|
|
35
|
-
echo "$SCHEMA_ITEM" | jq -e '.properties.repoSet.minItems == 2' >/dev/null 2>&1 \
|
|
36
|
-
&& pass "repoSet minItems=2" || fail "repoSet should enforce minItems=2"
|
|
37
|
-
echo "$SCHEMA_ITEM" | jq -e '.properties.platform.enum | index("ios") and index("android") and index("mixed")' >/dev/null 2>&1 \
|
|
38
|
-
&& pass "platform enum covers ios/android/mixed" || fail "platform enum incomplete"
|
|
39
|
-
echo "$SCHEMA_ITEM" | jq -e '.properties.noHost.type == "boolean"' >/dev/null 2>&1 \
|
|
40
|
-
&& pass "noHost flag supports skip-combo" || fail "noHost flag missing"
|
|
41
|
-
echo "$SCHEMA_ITEM" | jq -e '.properties.lastResult.enum | index("success") and index("failed") and index("skipped")' >/dev/null 2>&1 \
|
|
42
|
-
&& pass "lastResult enum covers all terminal states" || fail "lastResult enum incomplete"
|
|
43
|
-
|
|
44
|
-
echo ""
|
|
45
|
-
echo "→ 3. contract doc exists and ships with required sections"
|
|
46
|
-
[ -f "$DOC" ] && pass "multi-repo-integration-build.md exists" || { fail "doc missing"; echo "══ smoke: $PASS passed, $FAIL failed ══"; exit 1; }
|
|
47
|
-
grep -q "## Why this exists" "$DOC" && pass "doc has 'Why this exists'" || fail "missing 'Why this exists'"
|
|
48
|
-
grep -q "## When this rule fires" "$DOC" && pass "doc has 'When this rule fires'" || fail "missing trigger section"
|
|
49
|
-
grep -q "Learn-once prompt" "$DOC" && pass "doc has learn-once prompt spec" || fail "missing learn-once prompt"
|
|
50
|
-
grep -q "Autopilot behavior" "$DOC" && pass "doc has autopilot behavior" || fail "missing autopilot behavior"
|
|
51
|
-
grep -q "noHost" "$DOC" && pass "doc documents noHost entries" || fail "missing noHost handling"
|
|
52
|
-
grep -qE "phase-tracker\.sh sub 6 0" "$DOC" && pass "doc wires tracker sub-step" || fail "missing tracker wiring"
|
|
53
|
-
|
|
54
|
-
echo ""
|
|
55
|
-
echo "→ 4. phase-6-commit.md references the new Step 0"
|
|
56
|
-
PHASE6="$ROOT/pipeline/multi-agent-refs/phases/phase-6-commit.md"
|
|
57
|
-
grep -q "Step 0 - Multi-Repo Integration Build" "$PHASE6" \
|
|
58
|
-
&& pass "phase-6-commit has Step 0 entry" || fail "phase-6-commit missing Step 0"
|
|
59
|
-
grep -q "multi-repo-integration-build.md" "$PHASE6" \
|
|
60
|
-
&& pass "phase-6-commit links the contract doc" || fail "phase-6-commit missing contract link"
|
|
61
|
-
|
|
62
|
-
echo ""
|
|
63
|
-
echo "→ 5. generateCopilotInstructions includes the generic rule"
|
|
64
|
-
# v8.0.0: install.js is a thin shim; the instruction body lives in
|
|
65
|
-
# install/templates/copilot-instructions.md (Markdown source) and is read by
|
|
66
|
-
# install/_copilot-instructions.mjs. The grep target is the source-of-truth
|
|
67
|
-
# Markdown file.
|
|
68
|
-
INSTALL_TPL="$ROOT/install/templates/copilot-instructions.md"
|
|
69
|
-
[ -f "$INSTALL_TPL" ] || fail "install/templates/copilot-instructions.md missing - v8.0.0 generator template missing"
|
|
70
|
-
grep -q "Post-Development Integration Build (Multi-Repo)" "$INSTALL_TPL" \
|
|
71
|
-
&& pass "generator template includes generic rule heading" || fail "generator template missing generic rule"
|
|
72
|
-
grep -q "multiRepoIntegrationHosts" "$INSTALL_TPL" \
|
|
73
|
-
&& pass "generator template references registry" || fail "generator template missing registry reference"
|
|
74
|
-
|
|
75
|
-
echo ""
|
|
76
|
-
echo "→ 6. repoSet sorted-equality matching logic (spec check)"
|
|
77
|
-
# Synthesize two entries, confirm match logic shape described by the doc.
|
|
78
|
-
TMP=$(mktemp -d)
|
|
79
|
-
trap 'rm -rf "$TMP"' EXIT
|
|
80
|
-
cat > "$TMP/prefs.json" <<'EOF'
|
|
81
|
-
{
|
|
82
|
-
"schemaVersion": "2.1.0",
|
|
83
|
-
"global": {
|
|
84
|
-
"multiRepoIntegrationHosts": [
|
|
85
|
-
{ "repoSet": ["common-lib", "ui-components"], "hostPath": "/host/proj", "platform": "ios", "hostScheme": "UI", "count": 2, "lastResult": "success" },
|
|
86
|
-
{ "repoSet": ["foo", "bar"], "noHost": true }
|
|
87
|
-
]
|
|
88
|
-
},
|
|
89
|
-
"projects": {}
|
|
90
|
-
}
|
|
91
|
-
EOF
|
|
92
|
-
|
|
93
|
-
# Lookup: sort incoming repoSet and compare to each entry's sorted repoSet.
|
|
94
|
-
MATCH=$(jq -r --arg combo "common-lib,ui-components" \
|
|
95
|
-
'.global.multiRepoIntegrationHosts[]
|
|
96
|
-
| select((.repoSet | sort | join(",")) == $combo)
|
|
97
|
-
| .hostPath // "NOHOST"' "$TMP/prefs.json")
|
|
98
|
-
[ "$MATCH" = "/host/proj" ] && pass "sorted-equality lookup returns hostPath" || fail "lookup returned: $MATCH"
|
|
99
|
-
|
|
100
|
-
# Reversed input order still matches
|
|
101
|
-
MATCH=$(jq -r --arg combo "common-lib,ui-components" \
|
|
102
|
-
'.global.multiRepoIntegrationHosts[]
|
|
103
|
-
| select((.repoSet | sort | join(",")) == $combo)
|
|
104
|
-
| .hostPath // "NOHOST"' "$TMP/prefs.json")
|
|
105
|
-
[ "$MATCH" = "/host/proj" ] && pass "reversed-order input matches same host" || fail "reversed-order did not match"
|
|
106
|
-
|
|
107
|
-
# noHost combo returns skip-intent
|
|
108
|
-
NOHOST=$(jq -r --arg combo "bar,foo" \
|
|
109
|
-
'.global.multiRepoIntegrationHosts[]
|
|
110
|
-
| select((.repoSet | sort | join(",")) == $combo)
|
|
111
|
-
| .noHost // false' "$TMP/prefs.json")
|
|
112
|
-
[ "$NOHOST" = "true" ] && pass "noHost entry returned for marked combo" || fail "noHost lookup failed: $NOHOST"
|
|
113
|
-
|
|
114
|
-
echo ""
|
|
115
|
-
echo "══ multi-repo-integration smoke: $PASS passed, $FAIL failed ══"
|
|
116
|
-
[ "$FAIL" -eq 0 ] || exit 1
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# smoke-multi-repo-worktree.sh - asserts Phase 3 multi-repo dev contract is
|
|
3
|
-
# documented correctly. Doc-shape smoke (no actual git ops).
|
|
4
|
-
|
|
5
|
-
set -uo pipefail
|
|
6
|
-
|
|
7
|
-
PHASE3="$HOME/.claude/multi-agent-refs/phases/phase-3-dev.md"
|
|
8
|
-
|
|
9
|
-
PASS=0; FAIL=0
|
|
10
|
-
pass() { PASS=$((PASS+1)); echo " ✓ $1"; }
|
|
11
|
-
fail() { FAIL=$((FAIL+1)); echo " ✗ $1"; }
|
|
12
|
-
|
|
13
|
-
if [ ! -f "$PHASE3" ]; then
|
|
14
|
-
fail "phase-3-dev.md missing"; echo "══ aborted ══"; exit 2
|
|
15
|
-
fi
|
|
16
|
-
|
|
17
|
-
echo "→ 1. v2.1.0+ Multi-Repo Mode subsection present"
|
|
18
|
-
grep -q "v2.1.0+ Multi-Repo Mode" "$PHASE3" \
|
|
19
|
-
&& pass "Phase 3 has v2.1.0+ Multi-Repo Mode subsection" \
|
|
20
|
-
|| fail "Phase 3 missing v2.1.0+ Multi-Repo Mode subsection"
|
|
21
|
-
|
|
22
|
-
echo "→ 2. Todo tagging contract"
|
|
23
|
-
grep -q "\"repo\":" "$PHASE3" \
|
|
24
|
-
&& pass "Todo schema includes \"repo\" tag" \
|
|
25
|
-
|| fail "Todo schema missing \"repo\" tag"
|
|
26
|
-
grep -q "Per-todo worktree switch" "$PHASE3" \
|
|
27
|
-
&& pass "Per-todo worktree switch documented" \
|
|
28
|
-
|| fail "Per-todo worktree switch missing"
|
|
29
|
-
grep -q 'cd "\$WT_PATH"' "$PHASE3" \
|
|
30
|
-
&& pass "Documents cd \$WT_PATH for build/test" \
|
|
31
|
-
|| fail "cd \$WT_PATH missing from doc"
|
|
32
|
-
|
|
33
|
-
echo "→ 3. Single-ticket status update"
|
|
34
|
-
grep -q "exactly \*\*once per todo\*\*" "$PHASE3" \
|
|
35
|
-
&& pass "Status update fires once per todo (not per repo per todo)" \
|
|
36
|
-
|| fail "Single-ticket status-update rule missing"
|
|
37
|
-
|
|
38
|
-
echo "→ 4. Build serialization"
|
|
39
|
-
grep -q "BUILD_LOCK" "$PHASE3" \
|
|
40
|
-
&& pass "Multi-repo respects BUILD_LOCK" \
|
|
41
|
-
|| fail "BUILD_LOCK reference missing"
|
|
42
|
-
grep -q "dependsOn\|dependency order" "$PHASE3" \
|
|
43
|
-
&& pass "Cross-repo dependency order respected" \
|
|
44
|
-
|| fail "Cross-repo dependency order not documented"
|
|
45
|
-
|
|
46
|
-
echo "→ 5. Failure isolation"
|
|
47
|
-
grep -q "buildStatus" "$PHASE3" \
|
|
48
|
-
&& pass "Per-repo buildStatus persisted" \
|
|
49
|
-
|| fail "Per-repo buildStatus missing"
|
|
50
|
-
grep -q "do NOT proceed to Phase 4" "$PHASE3" \
|
|
51
|
-
&& pass "Pipeline blocks Phase 4 if any repo unbuilt" \
|
|
52
|
-
|| fail "Phase 4 block rule missing"
|
|
53
|
-
|
|
54
|
-
echo "→ 6. Telemetry per-repo"
|
|
55
|
-
grep -q 'log-metric.sh.*build.completed.*repo=' "$PHASE3" \
|
|
56
|
-
&& pass "Per-repo build.completed telemetry documented" \
|
|
57
|
-
|| fail "Per-repo telemetry missing"
|
|
58
|
-
|
|
59
|
-
echo ""
|
|
60
|
-
echo "══ multi-repo-worktree smoke: $PASS passed, $FAIL failed ══"
|
|
61
|
-
[ "$FAIL" -eq 0 ]
|