@runchr/gstack-antigravity 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (297) hide show
  1. package/.agents/rules/ETHOS.md +129 -0
  2. package/.agents/rules/global-gstack.md +117 -0
  3. package/.agents/rules/persona-gstack-autoplan.md +14 -0
  4. package/.agents/rules/persona-gstack-benchmark.md +14 -0
  5. package/.agents/rules/persona-gstack-browse.md +14 -0
  6. package/.agents/rules/persona-gstack-canary.md +14 -0
  7. package/.agents/rules/persona-gstack-careful.md +14 -0
  8. package/.agents/rules/persona-gstack-codex.md +14 -0
  9. package/.agents/rules/persona-gstack-cso.md +14 -0
  10. package/.agents/rules/persona-gstack-design-consultation.md +14 -0
  11. package/.agents/rules/persona-gstack-design-review.md +14 -0
  12. package/.agents/rules/persona-gstack-document-release.md +14 -0
  13. package/.agents/rules/persona-gstack-freeze.md +14 -0
  14. package/.agents/rules/persona-gstack-gstack-upgrade.md +14 -0
  15. package/.agents/rules/persona-gstack-guard.md +14 -0
  16. package/.agents/rules/persona-gstack-investigate.md +14 -0
  17. package/.agents/rules/persona-gstack-land-and-deploy.md +14 -0
  18. package/.agents/rules/persona-gstack-office-hours.md +14 -0
  19. package/.agents/rules/persona-gstack-plan-ceo-review.md +14 -0
  20. package/.agents/rules/persona-gstack-plan-design-review.md +14 -0
  21. package/.agents/rules/persona-gstack-plan-eng-review.md +14 -0
  22. package/.agents/rules/persona-gstack-qa-only.md +14 -0
  23. package/.agents/rules/persona-gstack-qa.md +14 -0
  24. package/.agents/rules/persona-gstack-retro.md +14 -0
  25. package/.agents/rules/persona-gstack-review.md +14 -0
  26. package/.agents/rules/persona-gstack-setup-browser-cookies.md +14 -0
  27. package/.agents/rules/persona-gstack-setup-deploy.md +14 -0
  28. package/.agents/rules/persona-gstack-ship.md +14 -0
  29. package/.agents/rules/persona-gstack-unfreeze.md +14 -0
  30. package/.agents/rules/persona-gstack.md +40 -0
  31. package/.agents/rules/recursive-identities.md +22 -0
  32. package/.agents/workflows/autoplan.md +30 -0
  33. package/.agents/workflows/benchmark.md +31 -0
  34. package/.agents/workflows/browse.md +26 -0
  35. package/.agents/workflows/canary.md +33 -0
  36. package/.agents/workflows/careful.md +22 -0
  37. package/.agents/workflows/codex.md +36 -0
  38. package/.agents/workflows/cso.md +29 -0
  39. package/.agents/workflows/design-consultation.md +28 -0
  40. package/.agents/workflows/design-review.md +28 -0
  41. package/.agents/workflows/document-release.md +32 -0
  42. package/.agents/workflows/freeze.md +17 -0
  43. package/.agents/workflows/gstack-upgrade.md +54 -0
  44. package/.agents/workflows/gstack.md +56 -0
  45. package/.agents/workflows/guard.md +18 -0
  46. package/.agents/workflows/investigate.md +37 -0
  47. package/.agents/workflows/land-and-deploy.md +35 -0
  48. package/.agents/workflows/office-hours.md +27 -0
  49. package/.agents/workflows/plan-ceo-review.md +34 -0
  50. package/.agents/workflows/plan-design-review.md +31 -0
  51. package/.agents/workflows/plan-eng-review.md +28 -0
  52. package/.agents/workflows/qa-only.md +28 -0
  53. package/.agents/workflows/qa.md +73 -0
  54. package/.agents/workflows/retro.md +34 -0
  55. package/.agents/workflows/review.md +30 -0
  56. package/.agents/workflows/setup-browser-cookies.md +15 -0
  57. package/.agents/workflows/setup-cookies.md +8 -0
  58. package/.agents/workflows/setup-deploy.md +21 -0
  59. package/.agents/workflows/ship.md +93 -0
  60. package/.agents/workflows/unfreeze.md +12 -0
  61. package/LICENSE +22 -0
  62. package/README.md +189 -0
  63. package/README_KO.md +191 -0
  64. package/bin/install.js +105 -0
  65. package/gstack-origin/.agents/skills/gstack/SKILL.md +651 -0
  66. package/gstack-origin/.agents/skills/gstack-autoplan/SKILL.md +678 -0
  67. package/gstack-origin/.agents/skills/gstack-benchmark/SKILL.md +482 -0
  68. package/gstack-origin/.agents/skills/gstack-browse/SKILL.md +511 -0
  69. package/gstack-origin/.agents/skills/gstack-canary/SKILL.md +486 -0
  70. package/gstack-origin/.agents/skills/gstack-careful/SKILL.md +50 -0
  71. package/gstack-origin/.agents/skills/gstack-cso/SKILL.md +607 -0
  72. package/gstack-origin/.agents/skills/gstack-design-consultation/SKILL.md +615 -0
  73. package/gstack-origin/.agents/skills/gstack-design-review/SKILL.md +988 -0
  74. package/gstack-origin/.agents/skills/gstack-document-release/SKILL.md +604 -0
  75. package/gstack-origin/.agents/skills/gstack-freeze/SKILL.md +67 -0
  76. package/gstack-origin/.agents/skills/gstack-guard/SKILL.md +62 -0
  77. package/gstack-origin/.agents/skills/gstack-investigate/SKILL.md +415 -0
  78. package/gstack-origin/.agents/skills/gstack-land-and-deploy/SKILL.md +873 -0
  79. package/gstack-origin/.agents/skills/gstack-office-hours/SKILL.md +986 -0
  80. package/gstack-origin/.agents/skills/gstack-plan-ceo-review/SKILL.md +1268 -0
  81. package/gstack-origin/.agents/skills/gstack-plan-design-review/SKILL.md +668 -0
  82. package/gstack-origin/.agents/skills/gstack-plan-eng-review/SKILL.md +826 -0
  83. package/gstack-origin/.agents/skills/gstack-qa/SKILL.md +1006 -0
  84. package/gstack-origin/.agents/skills/gstack-qa-only/SKILL.md +626 -0
  85. package/gstack-origin/.agents/skills/gstack-retro/SKILL.md +1065 -0
  86. package/gstack-origin/.agents/skills/gstack-review/SKILL.md +704 -0
  87. package/gstack-origin/.agents/skills/gstack-setup-browser-cookies/SKILL.md +325 -0
  88. package/gstack-origin/.agents/skills/gstack-setup-deploy/SKILL.md +450 -0
  89. package/gstack-origin/.agents/skills/gstack-ship/SKILL.md +1312 -0
  90. package/gstack-origin/.agents/skills/gstack-unfreeze/SKILL.md +36 -0
  91. package/gstack-origin/.agents/skills/gstack-upgrade/SKILL.md +220 -0
  92. package/gstack-origin/.env.example +5 -0
  93. package/gstack-origin/.github/workflows/skill-docs.yml +17 -0
  94. package/gstack-origin/AGENTS.md +49 -0
  95. package/gstack-origin/ARCHITECTURE.md +359 -0
  96. package/gstack-origin/BROWSER.md +271 -0
  97. package/gstack-origin/CHANGELOG.md +800 -0
  98. package/gstack-origin/CLAUDE.md +284 -0
  99. package/gstack-origin/CONTRIBUTING.md +370 -0
  100. package/gstack-origin/ETHOS.md +129 -0
  101. package/gstack-origin/LICENSE +21 -0
  102. package/gstack-origin/README.md +228 -0
  103. package/gstack-origin/SKILL.md +657 -0
  104. package/gstack-origin/SKILL.md.tmpl +281 -0
  105. package/gstack-origin/TODOS.md +564 -0
  106. package/gstack-origin/VERSION +1 -0
  107. package/gstack-origin/autoplan/SKILL.md +689 -0
  108. package/gstack-origin/autoplan/SKILL.md.tmpl +416 -0
  109. package/gstack-origin/benchmark/SKILL.md +489 -0
  110. package/gstack-origin/benchmark/SKILL.md.tmpl +233 -0
  111. package/gstack-origin/bin/dev-setup +68 -0
  112. package/gstack-origin/bin/dev-teardown +56 -0
  113. package/gstack-origin/bin/gstack-analytics +191 -0
  114. package/gstack-origin/bin/gstack-community-dashboard +113 -0
  115. package/gstack-origin/bin/gstack-config +38 -0
  116. package/gstack-origin/bin/gstack-diff-scope +71 -0
  117. package/gstack-origin/bin/gstack-global-discover.ts +591 -0
  118. package/gstack-origin/bin/gstack-repo-mode +93 -0
  119. package/gstack-origin/bin/gstack-review-log +9 -0
  120. package/gstack-origin/bin/gstack-review-read +12 -0
  121. package/gstack-origin/bin/gstack-slug +15 -0
  122. package/gstack-origin/bin/gstack-telemetry-log +158 -0
  123. package/gstack-origin/bin/gstack-telemetry-sync +127 -0
  124. package/gstack-origin/bin/gstack-update-check +196 -0
  125. package/gstack-origin/browse/SKILL.md +517 -0
  126. package/gstack-origin/browse/SKILL.md.tmpl +141 -0
  127. package/gstack-origin/browse/bin/find-browse +21 -0
  128. package/gstack-origin/browse/bin/remote-slug +14 -0
  129. package/gstack-origin/browse/scripts/build-node-server.sh +48 -0
  130. package/gstack-origin/browse/src/browser-manager.ts +634 -0
  131. package/gstack-origin/browse/src/buffers.ts +137 -0
  132. package/gstack-origin/browse/src/bun-polyfill.cjs +109 -0
  133. package/gstack-origin/browse/src/cli.ts +420 -0
  134. package/gstack-origin/browse/src/commands.ts +111 -0
  135. package/gstack-origin/browse/src/config.ts +150 -0
  136. package/gstack-origin/browse/src/cookie-import-browser.ts +417 -0
  137. package/gstack-origin/browse/src/cookie-picker-routes.ts +207 -0
  138. package/gstack-origin/browse/src/cookie-picker-ui.ts +541 -0
  139. package/gstack-origin/browse/src/find-browse.ts +61 -0
  140. package/gstack-origin/browse/src/meta-commands.ts +269 -0
  141. package/gstack-origin/browse/src/platform.ts +17 -0
  142. package/gstack-origin/browse/src/read-commands.ts +335 -0
  143. package/gstack-origin/browse/src/server.ts +369 -0
  144. package/gstack-origin/browse/src/snapshot.ts +398 -0
  145. package/gstack-origin/browse/src/url-validation.ts +91 -0
  146. package/gstack-origin/browse/src/write-commands.ts +352 -0
  147. package/gstack-origin/browse/test/bun-polyfill.test.ts +72 -0
  148. package/gstack-origin/browse/test/commands.test.ts +1836 -0
  149. package/gstack-origin/browse/test/config.test.ts +250 -0
  150. package/gstack-origin/browse/test/cookie-import-browser.test.ts +397 -0
  151. package/gstack-origin/browse/test/cookie-picker-routes.test.ts +205 -0
  152. package/gstack-origin/browse/test/find-browse.test.ts +50 -0
  153. package/gstack-origin/browse/test/fixtures/basic.html +33 -0
  154. package/gstack-origin/browse/test/fixtures/cursor-interactive.html +22 -0
  155. package/gstack-origin/browse/test/fixtures/dialog.html +15 -0
  156. package/gstack-origin/browse/test/fixtures/empty.html +2 -0
  157. package/gstack-origin/browse/test/fixtures/forms.html +55 -0
  158. package/gstack-origin/browse/test/fixtures/qa-eval-checkout.html +108 -0
  159. package/gstack-origin/browse/test/fixtures/qa-eval-spa.html +98 -0
  160. package/gstack-origin/browse/test/fixtures/qa-eval.html +51 -0
  161. package/gstack-origin/browse/test/fixtures/responsive.html +49 -0
  162. package/gstack-origin/browse/test/fixtures/snapshot.html +55 -0
  163. package/gstack-origin/browse/test/fixtures/spa.html +24 -0
  164. package/gstack-origin/browse/test/fixtures/states.html +17 -0
  165. package/gstack-origin/browse/test/fixtures/upload.html +25 -0
  166. package/gstack-origin/browse/test/gstack-config.test.ts +125 -0
  167. package/gstack-origin/browse/test/gstack-update-check.test.ts +467 -0
  168. package/gstack-origin/browse/test/handoff.test.ts +235 -0
  169. package/gstack-origin/browse/test/path-validation.test.ts +63 -0
  170. package/gstack-origin/browse/test/platform.test.ts +37 -0
  171. package/gstack-origin/browse/test/snapshot.test.ts +467 -0
  172. package/gstack-origin/browse/test/test-server.ts +57 -0
  173. package/gstack-origin/browse/test/url-validation.test.ts +72 -0
  174. package/gstack-origin/canary/SKILL.md +493 -0
  175. package/gstack-origin/canary/SKILL.md.tmpl +220 -0
  176. package/gstack-origin/careful/SKILL.md +59 -0
  177. package/gstack-origin/careful/SKILL.md.tmpl +57 -0
  178. package/gstack-origin/careful/bin/check-careful.sh +112 -0
  179. package/gstack-origin/codex/SKILL.md +677 -0
  180. package/gstack-origin/codex/SKILL.md.tmpl +356 -0
  181. package/gstack-origin/conductor.json +6 -0
  182. package/gstack-origin/cso/SKILL.md +615 -0
  183. package/gstack-origin/cso/SKILL.md.tmpl +376 -0
  184. package/gstack-origin/design-consultation/SKILL.md +625 -0
  185. package/gstack-origin/design-consultation/SKILL.md.tmpl +369 -0
  186. package/gstack-origin/design-review/SKILL.md +998 -0
  187. package/gstack-origin/design-review/SKILL.md.tmpl +262 -0
  188. package/gstack-origin/docs/images/github-2013.png +0 -0
  189. package/gstack-origin/docs/images/github-2026.png +0 -0
  190. package/gstack-origin/docs/skills.md +877 -0
  191. package/gstack-origin/document-release/SKILL.md +613 -0
  192. package/gstack-origin/document-release/SKILL.md.tmpl +357 -0
  193. package/gstack-origin/freeze/SKILL.md +82 -0
  194. package/gstack-origin/freeze/SKILL.md.tmpl +80 -0
  195. package/gstack-origin/freeze/bin/check-freeze.sh +68 -0
  196. package/gstack-origin/gstack-upgrade/SKILL.md +226 -0
  197. package/gstack-origin/gstack-upgrade/SKILL.md.tmpl +224 -0
  198. package/gstack-origin/guard/SKILL.md +82 -0
  199. package/gstack-origin/guard/SKILL.md.tmpl +80 -0
  200. package/gstack-origin/investigate/SKILL.md +435 -0
  201. package/gstack-origin/investigate/SKILL.md.tmpl +196 -0
  202. package/gstack-origin/land-and-deploy/SKILL.md +880 -0
  203. package/gstack-origin/land-and-deploy/SKILL.md.tmpl +575 -0
  204. package/gstack-origin/office-hours/SKILL.md +996 -0
  205. package/gstack-origin/office-hours/SKILL.md.tmpl +624 -0
  206. package/gstack-origin/package.json +55 -0
  207. package/gstack-origin/plan-ceo-review/SKILL.md +1277 -0
  208. package/gstack-origin/plan-ceo-review/SKILL.md.tmpl +838 -0
  209. package/gstack-origin/plan-design-review/SKILL.md +676 -0
  210. package/gstack-origin/plan-design-review/SKILL.md.tmpl +314 -0
  211. package/gstack-origin/plan-eng-review/SKILL.md +836 -0
  212. package/gstack-origin/plan-eng-review/SKILL.md.tmpl +279 -0
  213. package/gstack-origin/qa/SKILL.md +1016 -0
  214. package/gstack-origin/qa/SKILL.md.tmpl +316 -0
  215. package/gstack-origin/qa/references/issue-taxonomy.md +85 -0
  216. package/gstack-origin/qa/templates/qa-report-template.md +126 -0
  217. package/gstack-origin/qa-only/SKILL.md +633 -0
  218. package/gstack-origin/qa-only/SKILL.md.tmpl +101 -0
  219. package/gstack-origin/retro/SKILL.md +1072 -0
  220. package/gstack-origin/retro/SKILL.md.tmpl +833 -0
  221. package/gstack-origin/review/SKILL.md +849 -0
  222. package/gstack-origin/review/SKILL.md.tmpl +259 -0
  223. package/gstack-origin/review/TODOS-format.md +62 -0
  224. package/gstack-origin/review/checklist.md +190 -0
  225. package/gstack-origin/review/design-checklist.md +132 -0
  226. package/gstack-origin/review/greptile-triage.md +220 -0
  227. package/gstack-origin/scripts/analytics.ts +190 -0
  228. package/gstack-origin/scripts/dev-skill.ts +82 -0
  229. package/gstack-origin/scripts/eval-compare.ts +96 -0
  230. package/gstack-origin/scripts/eval-list.ts +116 -0
  231. package/gstack-origin/scripts/eval-select.ts +86 -0
  232. package/gstack-origin/scripts/eval-summary.ts +187 -0
  233. package/gstack-origin/scripts/eval-watch.ts +172 -0
  234. package/gstack-origin/scripts/gen-skill-docs.ts +2414 -0
  235. package/gstack-origin/scripts/skill-check.ts +167 -0
  236. package/gstack-origin/setup +269 -0
  237. package/gstack-origin/setup-browser-cookies/SKILL.md +330 -0
  238. package/gstack-origin/setup-browser-cookies/SKILL.md.tmpl +74 -0
  239. package/gstack-origin/setup-deploy/SKILL.md +459 -0
  240. package/gstack-origin/setup-deploy/SKILL.md.tmpl +220 -0
  241. package/gstack-origin/ship/SKILL.md +1457 -0
  242. package/gstack-origin/ship/SKILL.md.tmpl +528 -0
  243. package/gstack-origin/supabase/config.sh +10 -0
  244. package/gstack-origin/supabase/functions/community-pulse/index.ts +59 -0
  245. package/gstack-origin/supabase/functions/telemetry-ingest/index.ts +135 -0
  246. package/gstack-origin/supabase/functions/update-check/index.ts +37 -0
  247. package/gstack-origin/supabase/migrations/001_telemetry.sql +89 -0
  248. package/gstack-origin/test/analytics.test.ts +277 -0
  249. package/gstack-origin/test/codex-e2e.test.ts +197 -0
  250. package/gstack-origin/test/fixtures/coverage-audit-fixture.ts +76 -0
  251. package/gstack-origin/test/fixtures/eval-baselines.json +7 -0
  252. package/gstack-origin/test/fixtures/qa-eval-checkout-ground-truth.json +43 -0
  253. package/gstack-origin/test/fixtures/qa-eval-ground-truth.json +43 -0
  254. package/gstack-origin/test/fixtures/qa-eval-spa-ground-truth.json +43 -0
  255. package/gstack-origin/test/fixtures/review-eval-design-slop.css +86 -0
  256. package/gstack-origin/test/fixtures/review-eval-design-slop.html +41 -0
  257. package/gstack-origin/test/fixtures/review-eval-enum-diff.rb +30 -0
  258. package/gstack-origin/test/fixtures/review-eval-enum.rb +27 -0
  259. package/gstack-origin/test/fixtures/review-eval-vuln.rb +14 -0
  260. package/gstack-origin/test/gemini-e2e.test.ts +173 -0
  261. package/gstack-origin/test/gen-skill-docs.test.ts +1049 -0
  262. package/gstack-origin/test/global-discover.test.ts +187 -0
  263. package/gstack-origin/test/helpers/codex-session-runner.ts +282 -0
  264. package/gstack-origin/test/helpers/e2e-helpers.ts +239 -0
  265. package/gstack-origin/test/helpers/eval-store.test.ts +548 -0
  266. package/gstack-origin/test/helpers/eval-store.ts +689 -0
  267. package/gstack-origin/test/helpers/gemini-session-runner.test.ts +104 -0
  268. package/gstack-origin/test/helpers/gemini-session-runner.ts +201 -0
  269. package/gstack-origin/test/helpers/llm-judge.ts +130 -0
  270. package/gstack-origin/test/helpers/observability.test.ts +283 -0
  271. package/gstack-origin/test/helpers/session-runner.test.ts +96 -0
  272. package/gstack-origin/test/helpers/session-runner.ts +357 -0
  273. package/gstack-origin/test/helpers/skill-parser.ts +206 -0
  274. package/gstack-origin/test/helpers/touchfiles.ts +260 -0
  275. package/gstack-origin/test/hook-scripts.test.ts +373 -0
  276. package/gstack-origin/test/skill-e2e-browse.test.ts +293 -0
  277. package/gstack-origin/test/skill-e2e-deploy.test.ts +279 -0
  278. package/gstack-origin/test/skill-e2e-design.test.ts +614 -0
  279. package/gstack-origin/test/skill-e2e-plan.test.ts +538 -0
  280. package/gstack-origin/test/skill-e2e-qa-bugs.test.ts +194 -0
  281. package/gstack-origin/test/skill-e2e-qa-workflow.test.ts +412 -0
  282. package/gstack-origin/test/skill-e2e-review.test.ts +535 -0
  283. package/gstack-origin/test/skill-e2e-workflow.test.ts +586 -0
  284. package/gstack-origin/test/skill-e2e.test.ts +3325 -0
  285. package/gstack-origin/test/skill-llm-eval.test.ts +787 -0
  286. package/gstack-origin/test/skill-parser.test.ts +179 -0
  287. package/gstack-origin/test/skill-routing-e2e.test.ts +605 -0
  288. package/gstack-origin/test/skill-validation.test.ts +1520 -0
  289. package/gstack-origin/test/telemetry.test.ts +278 -0
  290. package/gstack-origin/test/touchfiles.test.ts +262 -0
  291. package/gstack-origin/unfreeze/SKILL.md +40 -0
  292. package/gstack-origin/unfreeze/SKILL.md.tmpl +38 -0
  293. package/package.json +38 -0
  294. package/scripts/install-antigravity-skill.ps1 +33 -0
  295. package/scripts/install-antigravity-skill.sh +41 -0
  296. package/scripts/sync-gstack-origin.ps1 +37 -0
  297. package/scripts/sync-gstack-origin.sh +35 -0
@@ -0,0 +1,3325 @@
1
+ import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
2
+ import { runSkillTest } from './helpers/session-runner';
3
+ import type { SkillTestResult } from './helpers/session-runner';
4
+ import { outcomeJudge, callJudge } from './helpers/llm-judge';
5
+ import { EvalCollector, judgePassed } from './helpers/eval-store';
6
+ import type { EvalTestEntry } from './helpers/eval-store';
7
+ import { startTestServer } from '../browse/test/test-server';
8
+ import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from './helpers/touchfiles';
9
+ import { spawnSync } from 'child_process';
10
+ import * as fs from 'fs';
11
+ import * as path from 'path';
12
+ import * as os from 'os';
13
+
14
+ const ROOT = path.resolve(import.meta.dir, '..');
15
+
16
+ // Skip unless EVALS=1. Session runner strips CLAUDE* env vars to avoid nested session issues.
17
+ //
18
+ // BLAME PROTOCOL: When an eval fails, do NOT claim "pre-existing" or "not related
19
+ // to our changes" without proof. Run the same eval on main to verify. These tests
20
+ // have invisible couplings — preamble text, SKILL.md content, and timing all affect
21
+ // agent behavior. See CLAUDE.md "E2E eval failure blame protocol" for details.
22
+ const evalsEnabled = !!process.env.EVALS;
23
+ const describeE2E = evalsEnabled ? describe : describe.skip;
24
+
25
+ // --- Diff-based test selection ---
26
+ // When EVALS_ALL is not set, only run tests whose touchfiles were modified.
27
+ // Set EVALS_ALL=1 to force all tests. Set EVALS_BASE to override base branch.
28
+ let selectedTests: string[] | null = null; // null = run all
29
+
30
+ if (evalsEnabled && !process.env.EVALS_ALL) {
31
+ const baseBranch = process.env.EVALS_BASE
32
+ || detectBaseBranch(ROOT)
33
+ || 'main';
34
+ const changedFiles = getChangedFiles(baseBranch, ROOT);
35
+
36
+ if (changedFiles.length > 0) {
37
+ const selection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES);
38
+ selectedTests = selection.selected;
39
+ process.stderr.write(`\nE2E selection (${selection.reason}): ${selection.selected.length}/${Object.keys(E2E_TOUCHFILES).length} tests\n`);
40
+ if (selection.skipped.length > 0) {
41
+ process.stderr.write(` Skipped: ${selection.skipped.join(', ')}\n`);
42
+ }
43
+ process.stderr.write('\n');
44
+ }
45
+ // If changedFiles is empty (e.g., on main branch), selectedTests stays null → run all
46
+ }
47
+
48
+ /** Wrap a describe block to skip entirely if none of its tests are selected. */
49
+ function describeIfSelected(name: string, testNames: string[], fn: () => void) {
50
+ const anySelected = selectedTests === null || testNames.some(t => selectedTests!.includes(t));
51
+ (anySelected ? describeE2E : describe.skip)(name, fn);
52
+ }
53
+
54
+ /** Skip an individual test if not selected (for multi-test describe blocks). */
55
+ function testIfSelected(testName: string, fn: () => Promise<void>, timeout: number) {
56
+ const shouldRun = selectedTests === null || selectedTests.includes(testName);
57
+ (shouldRun ? test : test.skip)(testName, fn, timeout);
58
+ }
59
+
60
+ // Eval result collector — accumulates test results, writes to ~/.gstack-dev/evals/ on finalize
61
+ const evalCollector = evalsEnabled ? new EvalCollector('e2e') : null;
62
+
63
+ // Unique run ID for this E2E session — used for heartbeat + per-run log directory
64
+ const runId = new Date().toISOString().replace(/[:.]/g, '').replace('T', '-').slice(0, 15);
65
+
66
+ /** DRY helper to record an E2E test result into the eval collector. */
67
+ function recordE2E(name: string, suite: string, result: SkillTestResult, extra?: Partial<EvalTestEntry>) {
68
+ // Derive last tool call from transcript for machine-readable diagnostics
69
+ const lastTool = result.toolCalls.length > 0
70
+ ? `${result.toolCalls[result.toolCalls.length - 1].tool}(${JSON.stringify(result.toolCalls[result.toolCalls.length - 1].input).slice(0, 60)})`
71
+ : undefined;
72
+
73
+ evalCollector?.addTest({
74
+ name, suite, tier: 'e2e',
75
+ passed: result.exitReason === 'success' && result.browseErrors.length === 0,
76
+ duration_ms: result.duration,
77
+ cost_usd: result.costEstimate.estimatedCost,
78
+ transcript: result.transcript,
79
+ output: result.output?.slice(0, 2000),
80
+ turns_used: result.costEstimate.turnsUsed,
81
+ browse_errors: result.browseErrors,
82
+ exit_reason: result.exitReason,
83
+ timeout_at_turn: result.exitReason === 'timeout' ? result.costEstimate.turnsUsed : undefined,
84
+ last_tool_call: lastTool,
85
+ ...extra,
86
+ });
87
+ }
88
+
89
+ let testServer: ReturnType<typeof startTestServer>;
90
+ let tmpDir: string;
91
+ const browseBin = path.resolve(ROOT, 'browse', 'dist', 'browse');
92
+
93
+ /**
94
+ * Copy a directory tree recursively (files only, follows structure).
95
+ */
96
+ function copyDirSync(src: string, dest: string) {
97
+ fs.mkdirSync(dest, { recursive: true });
98
+ for (const entry of fs.readdirSync(src, { withFileTypes: true })) {
99
+ const srcPath = path.join(src, entry.name);
100
+ const destPath = path.join(dest, entry.name);
101
+ if (entry.isDirectory()) {
102
+ copyDirSync(srcPath, destPath);
103
+ } else {
104
+ fs.copyFileSync(srcPath, destPath);
105
+ }
106
+ }
107
+ }
108
+
109
+ /**
110
+ * Set up browse shims (binary symlink, find-browse, remote-slug) in a tmpDir.
111
+ */
112
+ function setupBrowseShims(dir: string) {
113
+ // Symlink browse binary
114
+ const binDir = path.join(dir, 'browse', 'dist');
115
+ fs.mkdirSync(binDir, { recursive: true });
116
+ if (fs.existsSync(browseBin)) {
117
+ fs.symlinkSync(browseBin, path.join(binDir, 'browse'));
118
+ }
119
+
120
+ // find-browse shim
121
+ const findBrowseDir = path.join(dir, 'browse', 'bin');
122
+ fs.mkdirSync(findBrowseDir, { recursive: true });
123
+ fs.writeFileSync(
124
+ path.join(findBrowseDir, 'find-browse'),
125
+ `#!/bin/bash\necho "${browseBin}"\n`,
126
+ { mode: 0o755 },
127
+ );
128
+
129
+ // remote-slug shim (returns test-project)
130
+ fs.writeFileSync(
131
+ path.join(findBrowseDir, 'remote-slug'),
132
+ `#!/bin/bash\necho "test-project"\n`,
133
+ { mode: 0o755 },
134
+ );
135
+ }
136
+
137
+ /**
138
+ * Print cost summary after an E2E test.
139
+ */
140
+ function logCost(label: string, result: { costEstimate: { turnsUsed: number; estimatedTokens: number; estimatedCost: number }; duration: number }) {
141
+ const { turnsUsed, estimatedTokens, estimatedCost } = result.costEstimate;
142
+ const durationSec = Math.round(result.duration / 1000);
143
+ console.log(`${label}: $${estimatedCost.toFixed(2)} (${turnsUsed} turns, ${(estimatedTokens / 1000).toFixed(1)}k tokens, ${durationSec}s)`);
144
+ }
145
+
146
+ /**
147
+ * Dump diagnostic info on planted-bug outcome failure (decision 1C).
148
+ */
149
+ function dumpOutcomeDiagnostic(dir: string, label: string, report: string, judgeResult: any) {
150
+ try {
151
+ const transcriptDir = path.join(dir, '.gstack', 'test-transcripts');
152
+ fs.mkdirSync(transcriptDir, { recursive: true });
153
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
154
+ fs.writeFileSync(
155
+ path.join(transcriptDir, `${label}-outcome-${timestamp}.json`),
156
+ JSON.stringify({ label, report, judgeResult }, null, 2),
157
+ );
158
+ } catch { /* non-fatal */ }
159
+ }
160
+
161
+ // Fail fast if Anthropic API is unreachable — don't burn through 13 tests getting ConnectionRefused
162
+ if (evalsEnabled) {
163
+ const check = spawnSync('sh', ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], {
164
+ stdio: 'pipe', timeout: 30_000,
165
+ });
166
+ const output = check.stdout?.toString() || '';
167
+ if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) {
168
+ throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.');
169
+ }
170
+ }
171
+
172
+ describeIfSelected('Skill E2E tests', [
173
+ 'browse-basic', 'browse-snapshot', 'skillmd-setup-discovery',
174
+ 'skillmd-no-local-binary', 'skillmd-outside-git', 'contributor-mode', 'session-awareness',
175
+ ], () => {
176
+ beforeAll(() => {
177
+ testServer = startTestServer();
178
+ tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-'));
179
+ setupBrowseShims(tmpDir);
180
+ });
181
+
182
+ afterAll(() => {
183
+ testServer?.server?.stop();
184
+ try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
185
+ });
186
+
187
+ testIfSelected('browse-basic', async () => {
188
+ const result = await runSkillTest({
189
+ prompt: `You have a browse binary at ${browseBin}. Assign it to B variable and run these commands in sequence:
190
+ 1. $B goto ${testServer.url}
191
+ 2. $B snapshot -i
192
+ 3. $B text
193
+ 4. $B screenshot /tmp/skill-e2e-test.png
194
+ Report the results of each command.`,
195
+ workingDirectory: tmpDir,
196
+ maxTurns: 10,
197
+ timeout: 60_000,
198
+ testName: 'browse-basic',
199
+ runId,
200
+ });
201
+
202
+ logCost('browse basic', result);
203
+ recordE2E('browse basic commands', 'Skill E2E tests', result);
204
+ expect(result.browseErrors).toHaveLength(0);
205
+ expect(result.exitReason).toBe('success');
206
+ }, 90_000);
207
+
208
+ testIfSelected('browse-snapshot', async () => {
209
+ const result = await runSkillTest({
210
+ prompt: `You have a browse binary at ${browseBin}. Assign it to B variable and run:
211
+ 1. $B goto ${testServer.url}
212
+ 2. $B snapshot -i
213
+ 3. $B snapshot -c
214
+ 4. $B snapshot -D
215
+ 5. $B snapshot -i -a -o /tmp/skill-e2e-annotated.png
216
+ Report what each command returned.`,
217
+ workingDirectory: tmpDir,
218
+ maxTurns: 10,
219
+ timeout: 60_000,
220
+ testName: 'browse-snapshot',
221
+ runId,
222
+ });
223
+
224
+ logCost('browse snapshot', result);
225
+ recordE2E('browse snapshot flags', 'Skill E2E tests', result);
226
+ // browseErrors can include false positives from hallucinated paths (e.g. "baltimore" vs "bangalore")
227
+ if (result.browseErrors.length > 0) {
228
+ console.warn('Browse errors (non-fatal):', result.browseErrors);
229
+ }
230
+ expect(result.exitReason).toBe('success');
231
+ }, 90_000);
232
+
233
+ testIfSelected('skillmd-setup-discovery', async () => {
234
+ const skillMd = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
235
+ const setupStart = skillMd.indexOf('## SETUP');
236
+ const setupEnd = skillMd.indexOf('## IMPORTANT');
237
+ const setupBlock = skillMd.slice(setupStart, setupEnd);
238
+
239
+ // Guard: verify we extracted a valid setup block
240
+ expect(setupBlock).toContain('browse/dist/browse');
241
+
242
+ const result = await runSkillTest({
243
+ prompt: `Follow these instructions to find the browse binary and run a basic command.
244
+
245
+ ${setupBlock}
246
+
247
+ After finding the binary, run: $B goto ${testServer.url}
248
+ Then run: $B text
249
+ Report whether it worked.`,
250
+ workingDirectory: tmpDir,
251
+ maxTurns: 10,
252
+ timeout: 60_000,
253
+ testName: 'skillmd-setup-discovery',
254
+ runId,
255
+ });
256
+
257
+ recordE2E('SKILL.md setup block discovery', 'Skill E2E tests', result);
258
+ expect(result.browseErrors).toHaveLength(0);
259
+ expect(result.exitReason).toBe('success');
260
+ }, 90_000);
261
+
262
+ testIfSelected('skillmd-no-local-binary', async () => {
263
+ // Create a tmpdir with no browse binary — no local .claude/skills/gstack/browse/dist/browse
264
+ const emptyDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-empty-'));
265
+
266
+ const skillMd = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
267
+ const setupStart = skillMd.indexOf('## SETUP');
268
+ const setupEnd = skillMd.indexOf('## IMPORTANT');
269
+ const setupBlock = skillMd.slice(setupStart, setupEnd);
270
+
271
+ const result = await runSkillTest({
272
+ prompt: `Follow these instructions exactly. Run the bash code block below and report what it outputs.
273
+
274
+ ${setupBlock}
275
+
276
+ Report the exact output. Do NOT try to fix or install anything — just report what you see.`,
277
+ workingDirectory: emptyDir,
278
+ maxTurns: 5,
279
+ timeout: 30_000,
280
+ testName: 'skillmd-no-local-binary',
281
+ runId,
282
+ });
283
+
284
+ // Setup block should either find the global binary (READY) or show NEEDS_SETUP.
285
+ // On dev machines with gstack installed globally, the fallback path
286
+ // ~/.claude/skills/gstack/browse/dist/browse exists, so we get READY.
287
+ // The important thing is it doesn't crash or give a confusing error.
288
+ const allText = result.output || '';
289
+ recordE2E('SKILL.md setup block (no local binary)', 'Skill E2E tests', result);
290
+ expect(allText).toMatch(/READY|NEEDS_SETUP/);
291
+ expect(result.exitReason).toBe('success');
292
+
293
+ // Clean up
294
+ try { fs.rmSync(emptyDir, { recursive: true, force: true }); } catch {}
295
+ }, 60_000);
296
+
297
+ testIfSelected('skillmd-outside-git', async () => {
298
+ // Create a tmpdir outside any git repo
299
+ const nonGitDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-nogit-'));
300
+
301
+ const skillMd = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
302
+ const setupStart = skillMd.indexOf('## SETUP');
303
+ const setupEnd = skillMd.indexOf('## IMPORTANT');
304
+ const setupBlock = skillMd.slice(setupStart, setupEnd);
305
+
306
+ const result = await runSkillTest({
307
+ prompt: `Follow these instructions exactly. Run the bash code block below and report what it outputs.
308
+
309
+ ${setupBlock}
310
+
311
+ Report the exact output — either "READY: <path>" or "NEEDS_SETUP".`,
312
+ workingDirectory: nonGitDir,
313
+ maxTurns: 5,
314
+ timeout: 30_000,
315
+ testName: 'skillmd-outside-git',
316
+ runId,
317
+ });
318
+
319
+ // Should either find global binary (READY) or show NEEDS_SETUP — not crash
320
+ const allText = result.output || '';
321
+ recordE2E('SKILL.md outside git repo', 'Skill E2E tests', result);
322
+ expect(allText).toMatch(/READY|NEEDS_SETUP/);
323
+
324
+ // Clean up
325
+ try { fs.rmSync(nonGitDir, { recursive: true, force: true }); } catch {}
326
+ }, 60_000);
327
+
328
+ testIfSelected('contributor-mode', async () => {
329
+ const contribDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-contrib-'));
330
+ const logsDir = path.join(contribDir, 'contributor-logs');
331
+ fs.mkdirSync(logsDir, { recursive: true });
332
+
333
+ // Extract contributor mode instructions from generated SKILL.md
334
+ const skillMd = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
335
+ const contribStart = skillMd.indexOf('## Contributor Mode');
336
+ const contribEnd = skillMd.indexOf('\n## ', contribStart + 1);
337
+ const contribBlock = skillMd.slice(contribStart, contribEnd > 0 ? contribEnd : undefined);
338
+
339
+ const result = await runSkillTest({
340
+ prompt: `You are in contributor mode (_CONTRIB=true).
341
+
342
+ ${contribBlock}
343
+
344
+ OVERRIDE: Write contributor logs to ${logsDir}/ instead of ~/.gstack/contributor-logs/
345
+
346
+ Now try this browse command (it will fail — there is no binary at this path):
347
+ /nonexistent/path/browse goto https://example.com
348
+
349
+ This is a gstack issue (the browse binary is missing/misconfigured).
350
+ File a contributor report about this issue. Then tell me what you filed.`,
351
+ workingDirectory: contribDir,
352
+ maxTurns: 8,
353
+ timeout: 60_000,
354
+ testName: 'contributor-mode',
355
+ runId,
356
+ });
357
+
358
+ logCost('contributor mode', result);
359
+ // Override passed: this test intentionally triggers a browse error (nonexistent binary)
360
+ // so browseErrors will be non-empty — that's expected, not a failure
361
+ recordE2E('contributor mode report', 'Skill E2E tests', result, {
362
+ passed: result.exitReason === 'success',
363
+ });
364
+
365
+ // Verify a contributor log was created with expected format
366
+ const logFiles = fs.readdirSync(logsDir).filter(f => f.endsWith('.md'));
367
+ expect(logFiles.length).toBeGreaterThan(0);
368
+
369
+ // Verify new reflection-based format
370
+ const logContent = fs.readFileSync(path.join(logsDir, logFiles[0]), 'utf-8');
371
+ expect(logContent).toContain('Hey gstack team');
372
+ expect(logContent).toContain('What I was trying to do');
373
+ expect(logContent).toContain('What happened instead');
374
+ expect(logContent).toMatch(/rating/i);
375
+ // Verify report has repro steps (agent may use "Steps to reproduce", "Repro Steps", etc.)
376
+ expect(logContent).toMatch(/repro|steps to reproduce|how to reproduce/i);
377
+ // Verify report has date/version footer (agent may format differently)
378
+ expect(logContent).toMatch(/date.*2026|2026.*date/i);
379
+
380
+ // Clean up
381
+ try { fs.rmSync(contribDir, { recursive: true, force: true }); } catch {}
382
+ }, 90_000);
383
+
384
+ testIfSelected('session-awareness', async () => {
385
+ const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-'));
386
+
387
+ // Set up a git repo so there's project/branch context to reference
388
+ const run = (cmd: string, args: string[]) =>
389
+ spawnSync(cmd, args, { cwd: sessionDir, stdio: 'pipe', timeout: 5000 });
390
+ run('git', ['init', '-b', 'main']);
391
+ run('git', ['config', 'user.email', 'test@test.com']);
392
+ run('git', ['config', 'user.name', 'Test']);
393
+ fs.writeFileSync(path.join(sessionDir, 'app.rb'), '# my app\n');
394
+ run('git', ['add', '.']);
395
+ run('git', ['commit', '-m', 'init']);
396
+ run('git', ['checkout', '-b', 'feature/add-payments']);
397
+ // Add a remote so the agent can derive a project name
398
+ run('git', ['remote', 'add', 'origin', 'https://github.com/acme/billing-app.git']);
399
+
400
+ // Extract AskUserQuestion format instructions from generated SKILL.md
401
+ const skillMd = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
402
+ const aqStart = skillMd.indexOf('## AskUserQuestion Format');
403
+ const aqEnd = skillMd.indexOf('\n## ', aqStart + 1);
404
+ const aqBlock = skillMd.slice(aqStart, aqEnd > 0 ? aqEnd : undefined);
405
+
406
+ const outputPath = path.join(sessionDir, 'question-output.md');
407
+
408
+ const result = await runSkillTest({
409
+ prompt: `You are running a gstack skill. The session preamble detected _SESSIONS=4 (the user has 4 gstack windows open).
410
+
411
+ ${aqBlock}
412
+
413
+ You are on branch feature/add-payments in the billing-app project. You were reviewing a plan to add Stripe integration.
414
+
415
+ You've hit a decision point: the plan doesn't specify whether to use Stripe Checkout (hosted) or Stripe Elements (embedded). You need to ask the user which approach to use.
416
+
417
+ Since this is non-interactive, DO NOT actually call AskUserQuestion. Instead, write the EXACT text you would display to the user (the full AskUserQuestion content) to the file: ${outputPath}
418
+
419
+ Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple windows and may not remember what this conversation is about. Re-ground them.`,
420
+ workingDirectory: sessionDir,
421
+ maxTurns: 8,
422
+ timeout: 60_000,
423
+ testName: 'session-awareness',
424
+ runId,
425
+ });
426
+
427
+ logCost('session awareness', result);
428
+ recordE2E('session awareness ELI16', 'Skill E2E tests', result);
429
+
430
+ // Verify the output contains ELI16 re-grounding context
431
+ if (fs.existsSync(outputPath)) {
432
+ const output = fs.readFileSync(outputPath, 'utf-8');
433
+ const lower = output.toLowerCase();
434
+ // Must mention project name
435
+ expect(lower.includes('billing') || lower.includes('acme')).toBe(true);
436
+ // Must mention branch
437
+ expect(lower.includes('payment') || lower.includes('feature')).toBe(true);
438
+ // Must mention what we're working on
439
+ expect(lower.includes('stripe') || lower.includes('checkout') || lower.includes('payment')).toBe(true);
440
+ // Must have a RECOMMENDATION
441
+ expect(output).toContain('RECOMMENDATION');
442
+ } else {
443
+ // Check agent output as fallback
444
+ const output = result.output || '';
445
+ expect(output).toContain('RECOMMENDATION');
446
+ }
447
+
448
+ // Clean up
449
+ try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch {}
450
+ }, 90_000);
451
+ });
452
+
453
+ // --- B4: QA skill E2E ---
454
+
455
+ describeIfSelected('QA skill E2E', ['qa-quick'], () => {
456
+ let qaDir: string;
457
+
458
+ beforeAll(() => {
459
+ testServer = testServer || startTestServer();
460
+ qaDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-qa-'));
461
+ setupBrowseShims(qaDir);
462
+
463
+ // Copy qa skill files into tmpDir
464
+ copyDirSync(path.join(ROOT, 'qa'), path.join(qaDir, 'qa'));
465
+
466
+ // Create report directory
467
+ fs.mkdirSync(path.join(qaDir, 'qa-reports'), { recursive: true });
468
+ });
469
+
470
+ afterAll(() => {
471
+ testServer?.server?.stop();
472
+ try { fs.rmSync(qaDir, { recursive: true, force: true }); } catch {}
473
+ });
474
+
475
+ test('/qa quick completes without browse errors', async () => {
476
+ const result = await runSkillTest({
477
+ prompt: `B="${browseBin}"
478
+
479
+ The test server is already running at: ${testServer.url}
480
+ Target page: ${testServer.url}/basic.html
481
+
482
+ Read the file qa/SKILL.md for the QA workflow instructions.
483
+
484
+ Run a Quick-depth QA test on ${testServer.url}/basic.html
485
+ Do NOT use AskUserQuestion — run Quick tier directly.
486
+ Do NOT try to start a server or discover ports — the URL above is ready.
487
+ Write your report to ${qaDir}/qa-reports/qa-report.md`,
488
+ workingDirectory: qaDir,
489
+ maxTurns: 35,
490
+ timeout: 240_000,
491
+ testName: 'qa-quick',
492
+ runId,
493
+ });
494
+
495
+ logCost('/qa quick', result);
496
+ recordE2E('/qa quick', 'QA skill E2E', result, {
497
+ passed: ['success', 'error_max_turns'].includes(result.exitReason),
498
+ });
499
+ // browseErrors can include false positives from hallucinated paths
500
+ if (result.browseErrors.length > 0) {
501
+ console.warn('/qa quick browse errors (non-fatal):', result.browseErrors);
502
+ }
503
+ // Accept error_max_turns — the agent doing thorough QA work is not a failure
504
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
505
+ }, 300_000);
506
+ });
507
+
508
+ // --- B5: Review skill E2E ---
509
+
510
+ describeIfSelected('Review skill E2E', ['review-sql-injection'], () => {
511
+ let reviewDir: string;
512
+
513
+ beforeAll(() => {
514
+ reviewDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-review-'));
515
+
516
+ // Pre-build a git repo with a vulnerable file on a feature branch (decision 5A)
517
+ const { spawnSync } = require('child_process');
518
+ const run = (cmd: string, args: string[]) =>
519
+ spawnSync(cmd, args, { cwd: reviewDir, stdio: 'pipe', timeout: 5000 });
520
+
521
+ run('git', ['init', '-b', 'main']);
522
+ run('git', ['config', 'user.email', 'test@test.com']);
523
+ run('git', ['config', 'user.name', 'Test']);
524
+
525
+ // Commit a clean base on main
526
+ fs.writeFileSync(path.join(reviewDir, 'app.rb'), '# clean base\nclass App\nend\n');
527
+ run('git', ['add', 'app.rb']);
528
+ run('git', ['commit', '-m', 'initial commit']);
529
+
530
+ // Create feature branch with vulnerable code
531
+ run('git', ['checkout', '-b', 'feature/add-user-controller']);
532
+ const vulnContent = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-vuln.rb'), 'utf-8');
533
+ fs.writeFileSync(path.join(reviewDir, 'user_controller.rb'), vulnContent);
534
+ run('git', ['add', 'user_controller.rb']);
535
+ run('git', ['commit', '-m', 'add user controller']);
536
+
537
+ // Copy review skill files
538
+ fs.copyFileSync(path.join(ROOT, 'review', 'SKILL.md'), path.join(reviewDir, 'review-SKILL.md'));
539
+ fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(reviewDir, 'review-checklist.md'));
540
+ fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(reviewDir, 'review-greptile-triage.md'));
541
+ });
542
+
543
+ afterAll(() => {
544
+ try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
545
+ });
546
+
547
+ test('/review produces findings on SQL injection branch', async () => {
548
+ const result = await runSkillTest({
549
+ prompt: `You are in a git repo on a feature branch with changes against main.
550
+ Read review-SKILL.md for the review workflow instructions.
551
+ Also read review-checklist.md and apply it.
552
+ Run /review on the current diff (git diff main...HEAD).
553
+ Write your review findings to ${reviewDir}/review-output.md`,
554
+ workingDirectory: reviewDir,
555
+ maxTurns: 15,
556
+ timeout: 90_000,
557
+ testName: 'review-sql-injection',
558
+ runId,
559
+ });
560
+
561
+ logCost('/review', result);
562
+ recordE2E('/review SQL injection', 'Review skill E2E', result);
563
+ expect(result.exitReason).toBe('success');
564
+ }, 120_000);
565
+ });
566
+
567
+ // --- Review: Enum completeness E2E ---
568
+
569
+ describeIfSelected('Review enum completeness E2E', ['review-enum-completeness'], () => {
570
+ let enumDir: string;
571
+
572
+ beforeAll(() => {
573
+ enumDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-enum-'));
574
+
575
+ const run = (cmd: string, args: string[]) =>
576
+ spawnSync(cmd, args, { cwd: enumDir, stdio: 'pipe', timeout: 5000 });
577
+
578
+ run('git', ['init', '-b', 'main']);
579
+ run('git', ['config', 'user.email', 'test@test.com']);
580
+ run('git', ['config', 'user.name', 'Test']);
581
+
582
+ // Commit baseline on main — order model with 4 statuses
583
+ const baseContent = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-enum.rb'), 'utf-8');
584
+ fs.writeFileSync(path.join(enumDir, 'order.rb'), baseContent);
585
+ run('git', ['add', 'order.rb']);
586
+ run('git', ['commit', '-m', 'initial order model']);
587
+
588
+ // Feature branch adds "returned" status but misses handlers
589
+ run('git', ['checkout', '-b', 'feature/add-returned-status']);
590
+ const diffContent = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-enum-diff.rb'), 'utf-8');
591
+ fs.writeFileSync(path.join(enumDir, 'order.rb'), diffContent);
592
+ run('git', ['add', 'order.rb']);
593
+ run('git', ['commit', '-m', 'add returned status']);
594
+
595
+ // Copy review skill files
596
+ fs.copyFileSync(path.join(ROOT, 'review', 'SKILL.md'), path.join(enumDir, 'review-SKILL.md'));
597
+ fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(enumDir, 'review-checklist.md'));
598
+ fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(enumDir, 'review-greptile-triage.md'));
599
+ });
600
+
601
+ afterAll(() => {
602
+ try { fs.rmSync(enumDir, { recursive: true, force: true }); } catch {}
603
+ });
604
+
605
+ test('/review catches missing enum handlers for new status value', async () => {
606
+ const result = await runSkillTest({
607
+ prompt: `You are in a git repo on branch feature/add-returned-status with changes against main.
608
+ Read review-SKILL.md for the review workflow instructions.
609
+ Also read review-checklist.md and apply it — pay special attention to the Enum & Value Completeness section.
610
+ Run /review on the current diff (git diff main...HEAD).
611
+ Write your review findings to ${enumDir}/review-output.md
612
+
613
+ The diff adds a new "returned" status to the Order model. Your job is to check if all consumers handle it.`,
614
+ workingDirectory: enumDir,
615
+ maxTurns: 15,
616
+ timeout: 90_000,
617
+ testName: 'review-enum-completeness',
618
+ runId,
619
+ });
620
+
621
+ logCost('/review enum', result);
622
+ recordE2E('/review enum completeness', 'Review enum completeness E2E', result);
623
+ expect(result.exitReason).toBe('success');
624
+
625
+ // Verify the review caught the missing enum handlers
626
+ const reviewPath = path.join(enumDir, 'review-output.md');
627
+ if (fs.existsSync(reviewPath)) {
628
+ const review = fs.readFileSync(reviewPath, 'utf-8');
629
+ // Should mention the missing "returned" handling in at least one of the methods
630
+ const mentionsReturned = review.toLowerCase().includes('returned');
631
+ const mentionsEnum = review.toLowerCase().includes('enum') || review.toLowerCase().includes('status');
632
+ const mentionsCritical = review.toLowerCase().includes('critical');
633
+ expect(mentionsReturned).toBe(true);
634
+ expect(mentionsEnum || mentionsCritical).toBe(true);
635
+ }
636
+ }, 120_000);
637
+ });
638
+
639
+ // --- Review: Design review lite E2E ---
640
+
641
+ describeE2E('Review design lite E2E', () => {
642
+ let designDir: string;
643
+
644
+ beforeAll(() => {
645
+ designDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-design-lite-'));
646
+
647
+ const run = (cmd: string, args: string[]) =>
648
+ spawnSync(cmd, args, { cwd: designDir, stdio: 'pipe', timeout: 5000 });
649
+
650
+ run('git', ['init', '-b', 'main']);
651
+ run('git', ['config', 'user.email', 'test@test.com']);
652
+ run('git', ['config', 'user.name', 'Test']);
653
+
654
+ // Commit clean base on main
655
+ fs.writeFileSync(path.join(designDir, 'index.html'), '<h1>Clean</h1>\n');
656
+ fs.writeFileSync(path.join(designDir, 'styles.css'), 'body { font-size: 16px; }\n');
657
+ run('git', ['add', '.']);
658
+ run('git', ['commit', '-m', 'initial']);
659
+
660
+ // Feature branch adds AI slop CSS + HTML
661
+ run('git', ['checkout', '-b', 'feature/add-landing-page']);
662
+ const slopCss = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-design-slop.css'), 'utf-8');
663
+ const slopHtml = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-design-slop.html'), 'utf-8');
664
+ fs.writeFileSync(path.join(designDir, 'styles.css'), slopCss);
665
+ fs.writeFileSync(path.join(designDir, 'landing.html'), slopHtml);
666
+ run('git', ['add', '.']);
667
+ run('git', ['commit', '-m', 'add landing page']);
668
+
669
+ // Copy review skill files
670
+ fs.copyFileSync(path.join(ROOT, 'review', 'SKILL.md'), path.join(designDir, 'review-SKILL.md'));
671
+ fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(designDir, 'review-checklist.md'));
672
+ fs.copyFileSync(path.join(ROOT, 'review', 'design-checklist.md'), path.join(designDir, 'review-design-checklist.md'));
673
+ fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(designDir, 'review-greptile-triage.md'));
674
+ });
675
+
676
+ afterAll(() => {
677
+ try { fs.rmSync(designDir, { recursive: true, force: true }); } catch {}
678
+ });
679
+
680
+ test('/review catches design anti-patterns in CSS/HTML diff', async () => {
681
+ const result = await runSkillTest({
682
+ prompt: `You are in a git repo on branch feature/add-landing-page with changes against main.
683
+ Read review-SKILL.md for the review workflow instructions.
684
+ Read review-checklist.md for the code review checklist.
685
+ Read review-design-checklist.md for the design review checklist.
686
+ Run /review on the current diff (git diff main...HEAD).
687
+
688
+ The diff adds a landing page with CSS and HTML. Check for both code issues AND design anti-patterns.
689
+ Write your review findings to ${designDir}/review-output.md
690
+
691
+ Important: The design checklist should catch issues like blacklisted fonts, small font sizes, outline:none, !important, AI slop patterns (purple gradients, generic hero copy, 3-column feature grid), etc.`,
692
+ workingDirectory: designDir,
693
+ maxTurns: 15,
694
+ timeout: 120_000,
695
+ testName: 'review-design-lite',
696
+ runId,
697
+ });
698
+
699
+ logCost('/review design lite', result);
700
+ recordE2E('/review design lite', 'Review design lite E2E', result);
701
+ expect(result.exitReason).toBe('success');
702
+
703
+ // Verify the review caught at least 4 of 7 planted design issues
704
+ const reviewPath = path.join(designDir, 'review-output.md');
705
+ if (fs.existsSync(reviewPath)) {
706
+ const review = fs.readFileSync(reviewPath, 'utf-8').toLowerCase();
707
+ let detected = 0;
708
+
709
+ // Issue 1: Blacklisted font (Papyrus) — HIGH
710
+ if (review.includes('papyrus') || review.includes('blacklisted font') || review.includes('font family')) detected++;
711
+ // Issue 2: Body text < 16px — HIGH
712
+ if (review.includes('14px') || review.includes('font-size') || review.includes('font size') || review.includes('body text')) detected++;
713
+ // Issue 3: outline: none — HIGH
714
+ if (review.includes('outline') || review.includes('focus')) detected++;
715
+ // Issue 4: !important — HIGH
716
+ if (review.includes('!important') || review.includes('important')) detected++;
717
+ // Issue 5: Purple gradient — MEDIUM
718
+ if (review.includes('gradient') || review.includes('purple') || review.includes('violet') || review.includes('#6366f1') || review.includes('#8b5cf6')) detected++;
719
+ // Issue 6: Generic hero copy — MEDIUM
720
+ if (review.includes('welcome to') || review.includes('all-in-one') || review.includes('generic') || review.includes('hero copy') || review.includes('ai slop')) detected++;
721
+ // Issue 7: 3-column feature grid — LOW
722
+ if (review.includes('3-column') || review.includes('three-column') || review.includes('feature grid') || review.includes('icon') || review.includes('circle')) detected++;
723
+
724
+ console.log(`Design review detected ${detected}/7 planted issues`);
725
+ expect(detected).toBeGreaterThanOrEqual(4);
726
+ }
727
+ }, 150_000);
728
+ });
729
+
730
+ // --- B6/B7/B8: Planted-bug outcome evals ---
731
+
732
+ // Outcome evals also need ANTHROPIC_API_KEY for the LLM judge
733
+ const hasApiKey = !!process.env.ANTHROPIC_API_KEY;
734
+ const describeOutcome = (evalsEnabled && hasApiKey) ? describe : describe.skip;
735
+
736
+ // Wrap describeOutcome with selection — skip if no planted-bug tests are selected
737
+ const outcomeTestNames = ['qa-b6-static', 'qa-b7-spa', 'qa-b8-checkout'];
738
+ const anyOutcomeSelected = selectedTests === null || outcomeTestNames.some(t => selectedTests!.includes(t));
739
+ (anyOutcomeSelected ? describeOutcome : describe.skip)('Planted-bug outcome evals', () => {
740
+ let outcomeDir: string;
741
+
742
+ beforeAll(() => {
743
+ // Always start fresh — previous tests' agents may have killed the shared server
744
+ try { testServer?.server?.stop(); } catch {}
745
+ testServer = startTestServer();
746
+ outcomeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-outcome-'));
747
+ setupBrowseShims(outcomeDir);
748
+
749
+ // Copy qa skill files
750
+ copyDirSync(path.join(ROOT, 'qa'), path.join(outcomeDir, 'qa'));
751
+ });
752
+
753
+ afterAll(() => {
754
+ testServer?.server?.stop();
755
+ try { fs.rmSync(outcomeDir, { recursive: true, force: true }); } catch {}
756
+ });
757
+
758
+ /**
759
+ * Shared planted-bug eval runner.
760
+ * Gives the agent concise bug-finding instructions (not the full QA workflow),
761
+ * then scores the report with an LLM outcome judge.
762
+ */
763
+ async function runPlantedBugEval(fixture: string, groundTruthFile: string, label: string) {
764
+ // Each test gets its own isolated working directory to prevent cross-contamination
765
+ // (agents reading previous tests' reports and hallucinating those bugs)
766
+ const testWorkDir = fs.mkdtempSync(path.join(os.tmpdir(), `skill-e2e-${label}-`));
767
+ setupBrowseShims(testWorkDir);
768
+ const reportDir = path.join(testWorkDir, 'reports');
769
+ fs.mkdirSync(path.join(reportDir, 'screenshots'), { recursive: true });
770
+ const reportPath = path.join(reportDir, 'qa-report.md');
771
+
772
+ // Direct bug-finding with browse. Keep prompt concise — no reading long SKILL.md docs.
773
+ // "Write early, update later" pattern ensures report exists even if agent hits max turns.
774
+ const targetUrl = `${testServer.url}/${fixture}`;
775
+ const result = await runSkillTest({
776
+ prompt: `Find bugs on this page: ${targetUrl}
777
+
778
+ Browser binary: B="${browseBin}"
779
+
780
+ PHASE 1 — Quick scan (5 commands max):
781
+ $B goto ${targetUrl}
782
+ $B console --errors
783
+ $B snapshot -i
784
+ $B snapshot -c
785
+ $B accessibility
786
+
787
+ PHASE 2 — Write initial report to ${reportPath}:
788
+ Write every bug you found so far. Format each as:
789
+ - Category: functional / visual / accessibility / console
790
+ - Severity: high / medium / low
791
+ - Evidence: what you observed
792
+
793
+ PHASE 3 — Interactive testing (targeted — max 15 commands):
794
+ - Test email: type "user@" (no domain) and blur — does it validate?
795
+ - Test quantity: clear the field entirely — check the total display
796
+ - Test credit card: type a 25-character string — check for overflow
797
+ - Submit the form with zip code empty — does it require zip?
798
+ - Submit a valid form and run $B console --errors
799
+ - After finding more bugs, UPDATE ${reportPath} with new findings
800
+
801
+ PHASE 4 — Finalize report:
802
+ - UPDATE ${reportPath} with ALL bugs found across all phases
803
+ - Include console errors, form validation issues, visual overflow, missing attributes
804
+
805
+ CRITICAL RULES:
806
+ - ONLY test the page at ${targetUrl} — do not navigate to other sites
807
+ - Write the report file in PHASE 2 before doing interactive testing
808
+ - The report MUST exist at ${reportPath} when you finish`,
809
+ workingDirectory: testWorkDir,
810
+ maxTurns: 50,
811
+ timeout: 300_000,
812
+ testName: `qa-${label}`,
813
+ runId,
814
+ });
815
+
816
+ logCost(`/qa ${label}`, result);
817
+
818
+ // Phase 1: browse mechanics. Accept error_max_turns — agent may have written
819
+ // a partial report before running out of turns. What matters is detection rate.
820
+ if (result.browseErrors.length > 0) {
821
+ console.warn(`${label} browse errors:`, result.browseErrors);
822
+ }
823
+ if (result.exitReason !== 'success' && result.exitReason !== 'error_max_turns') {
824
+ throw new Error(`${label}: unexpected exit reason: ${result.exitReason}`);
825
+ }
826
+
827
+ // Phase 2: Outcome evaluation via LLM judge
828
+ const groundTruth = JSON.parse(
829
+ fs.readFileSync(path.join(ROOT, 'test', 'fixtures', groundTruthFile), 'utf-8'),
830
+ );
831
+
832
+ // Read the generated report (try expected path, then glob for any .md in reportDir or workDir)
833
+ let report: string | null = null;
834
+ if (fs.existsSync(reportPath)) {
835
+ report = fs.readFileSync(reportPath, 'utf-8');
836
+ } else {
837
+ // Agent may have named it differently — find any .md in reportDir or testWorkDir
838
+ for (const searchDir of [reportDir, testWorkDir]) {
839
+ try {
840
+ const mdFiles = fs.readdirSync(searchDir).filter(f => f.endsWith('.md'));
841
+ if (mdFiles.length > 0) {
842
+ report = fs.readFileSync(path.join(searchDir, mdFiles[0]), 'utf-8');
843
+ break;
844
+ }
845
+ } catch { /* dir may not exist if agent hit max_turns early */ }
846
+ }
847
+
848
+ // Also check the agent's final output for inline report content
849
+ if (!report && result.output && result.output.length > 100) {
850
+ report = result.output;
851
+ }
852
+ }
853
+
854
+ if (!report) {
855
+ dumpOutcomeDiagnostic(testWorkDir, label, '(no report file found)', { error: 'missing report' });
856
+ recordE2E(`/qa ${label}`, 'Planted-bug outcome evals', result, { error: 'no report generated' });
857
+ throw new Error(`No report file found in ${reportDir}`);
858
+ }
859
+
860
+ const judgeResult = await outcomeJudge(groundTruth, report);
861
+ console.log(`${label} outcome:`, JSON.stringify(judgeResult, null, 2));
862
+
863
+ // Record to eval collector with outcome judge results
864
+ recordE2E(`/qa ${label}`, 'Planted-bug outcome evals', result, {
865
+ passed: judgePassed(judgeResult, groundTruth),
866
+ detection_rate: judgeResult.detection_rate,
867
+ false_positives: judgeResult.false_positives,
868
+ evidence_quality: judgeResult.evidence_quality,
869
+ detected_bugs: judgeResult.detected,
870
+ missed_bugs: judgeResult.missed,
871
+ });
872
+
873
+ // Diagnostic dump on failure (decision 1C)
874
+ if (judgeResult.detection_rate < groundTruth.minimum_detection || judgeResult.false_positives > groundTruth.max_false_positives) {
875
+ dumpOutcomeDiagnostic(testWorkDir, label, report, judgeResult);
876
+ }
877
+
878
+ // Phase 2 assertions
879
+ expect(judgeResult.detection_rate).toBeGreaterThanOrEqual(groundTruth.minimum_detection);
880
+ expect(judgeResult.false_positives).toBeLessThanOrEqual(groundTruth.max_false_positives);
881
+ expect(judgeResult.evidence_quality).toBeGreaterThanOrEqual(2);
882
+ }
883
+
884
+ // B6: Static dashboard — broken link, disabled submit, overflow, missing alt, console error
885
+ test('/qa finds >= 2 of 5 planted bugs (static)', async () => {
886
+ await runPlantedBugEval('qa-eval.html', 'qa-eval-ground-truth.json', 'b6-static');
887
+ }, 360_000);
888
+
889
+ // B7: SPA — broken route, stale state, async race, missing aria, console warning
890
+ test('/qa finds >= 2 of 5 planted SPA bugs', async () => {
891
+ await runPlantedBugEval('qa-eval-spa.html', 'qa-eval-spa-ground-truth.json', 'b7-spa');
892
+ }, 360_000);
893
+
894
+ // B8: Checkout — email regex, NaN total, CC overflow, missing required, stripe error
895
+ test('/qa finds >= 2 of 5 planted checkout bugs', async () => {
896
+ await runPlantedBugEval('qa-eval-checkout.html', 'qa-eval-checkout-ground-truth.json', 'b8-checkout');
897
+ }, 360_000);
898
+
899
+ });
900
+
901
+ // --- Plan CEO Review E2E ---
902
+
903
+ describeIfSelected('Plan CEO Review E2E', ['plan-ceo-review'], () => {
904
+ let planDir: string;
905
+
906
+ beforeAll(() => {
907
+ planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-ceo-'));
908
+ const { spawnSync } = require('child_process');
909
+ const run = (cmd: string, args: string[]) =>
910
+ spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
911
+
912
+ // Init git repo (CEO review SKILL.md has a "System Audit" step that runs git)
913
+ run('git', ['init', '-b', 'main']);
914
+ run('git', ['config', 'user.email', 'test@test.com']);
915
+ run('git', ['config', 'user.name', 'Test']);
916
+
917
+ // Create a simple plan document for the agent to review
918
+ fs.writeFileSync(path.join(planDir, 'plan.md'), `# Plan: Add User Dashboard
919
+
920
+ ## Context
921
+ We're building a new user dashboard that shows recent activity, notifications, and quick actions.
922
+
923
+ ## Changes
924
+ 1. New React component \`UserDashboard\` in \`src/components/\`
925
+ 2. REST API endpoint \`GET /api/dashboard\` returning user stats
926
+ 3. PostgreSQL query for activity aggregation
927
+ 4. Redis cache layer for dashboard data (5min TTL)
928
+
929
+ ## Architecture
930
+ - Frontend: React + TailwindCSS
931
+ - Backend: Express.js REST API
932
+ - Database: PostgreSQL with existing user/activity tables
933
+ - Cache: Redis for dashboard aggregates
934
+
935
+ ## Open questions
936
+ - Should we use WebSocket for real-time updates?
937
+ - How do we handle users with 100k+ activity records?
938
+ `);
939
+
940
+ run('git', ['add', '.']);
941
+ run('git', ['commit', '-m', 'add plan']);
942
+
943
+ // Copy plan-ceo-review skill
944
+ fs.mkdirSync(path.join(planDir, 'plan-ceo-review'), { recursive: true });
945
+ fs.copyFileSync(
946
+ path.join(ROOT, 'plan-ceo-review', 'SKILL.md'),
947
+ path.join(planDir, 'plan-ceo-review', 'SKILL.md'),
948
+ );
949
+ });
950
+
951
+ afterAll(() => {
952
+ try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
953
+ });
954
+
955
+ test('/plan-ceo-review produces structured review output', async () => {
956
+ const result = await runSkillTest({
957
+ prompt: `Read plan-ceo-review/SKILL.md for the review workflow.
958
+
959
+ Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration or system audit steps.
960
+
961
+ Choose HOLD SCOPE mode. Skip any AskUserQuestion calls — this is non-interactive.
962
+ Write your complete review directly to ${planDir}/review-output.md
963
+
964
+ Focus on reviewing the plan content: architecture, error handling, security, and performance.`,
965
+ workingDirectory: planDir,
966
+ maxTurns: 15,
967
+ timeout: 360_000,
968
+ testName: 'plan-ceo-review',
969
+ runId,
970
+ });
971
+
972
+ logCost('/plan-ceo-review', result);
973
+ recordE2E('/plan-ceo-review', 'Plan CEO Review E2E', result, {
974
+ passed: ['success', 'error_max_turns'].includes(result.exitReason),
975
+ });
976
+ // Accept error_max_turns — the CEO review is very thorough and may exceed turns
977
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
978
+
979
+ // Verify the review was written
980
+ const reviewPath = path.join(planDir, 'review-output.md');
981
+ if (fs.existsSync(reviewPath)) {
982
+ const review = fs.readFileSync(reviewPath, 'utf-8');
983
+ expect(review.length).toBeGreaterThan(200);
984
+ }
985
+ }, 420_000);
986
+ });
987
+
988
+ // --- Plan CEO Review (SELECTIVE EXPANSION) E2E ---
989
+
990
+ describeIfSelected('Plan CEO Review SELECTIVE EXPANSION E2E', ['plan-ceo-review-selective'], () => {
991
+ let planDir: string;
992
+
993
+ beforeAll(() => {
994
+ planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-ceo-sel-'));
995
+ const { spawnSync } = require('child_process');
996
+ const run = (cmd: string, args: string[]) =>
997
+ spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
998
+
999
+ run('git', ['init', '-b', 'main']);
1000
+ run('git', ['config', 'user.email', 'test@test.com']);
1001
+ run('git', ['config', 'user.name', 'Test']);
1002
+
1003
+ fs.writeFileSync(path.join(planDir, 'plan.md'), `# Plan: Add User Dashboard
1004
+
1005
+ ## Context
1006
+ We're building a new user dashboard that shows recent activity, notifications, and quick actions.
1007
+
1008
+ ## Changes
1009
+ 1. New React component \`UserDashboard\` in \`src/components/\`
1010
+ 2. REST API endpoint \`GET /api/dashboard\` returning user stats
1011
+ 3. PostgreSQL query for activity aggregation
1012
+ 4. Redis cache layer for dashboard data (5min TTL)
1013
+
1014
+ ## Architecture
1015
+ - Frontend: React + TailwindCSS
1016
+ - Backend: Express.js REST API
1017
+ - Database: PostgreSQL with existing user/activity tables
1018
+ - Cache: Redis for dashboard aggregates
1019
+
1020
+ ## Open questions
1021
+ - Should we use WebSocket for real-time updates?
1022
+ - How do we handle users with 100k+ activity records?
1023
+ `);
1024
+
1025
+ run('git', ['add', '.']);
1026
+ run('git', ['commit', '-m', 'add plan']);
1027
+
1028
+ fs.mkdirSync(path.join(planDir, 'plan-ceo-review'), { recursive: true });
1029
+ fs.copyFileSync(
1030
+ path.join(ROOT, 'plan-ceo-review', 'SKILL.md'),
1031
+ path.join(planDir, 'plan-ceo-review', 'SKILL.md'),
1032
+ );
1033
+ });
1034
+
1035
+ afterAll(() => {
1036
+ try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
1037
+ });
1038
+
1039
+ test('/plan-ceo-review SELECTIVE EXPANSION produces structured review output', async () => {
1040
+ const result = await runSkillTest({
1041
+ prompt: `Read plan-ceo-review/SKILL.md for the review workflow.
1042
+
1043
+ Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration or system audit steps.
1044
+
1045
+ Choose SELECTIVE EXPANSION mode. Skip any AskUserQuestion calls — this is non-interactive.
1046
+ For the cherry-pick ceremony, accept all expansion proposals automatically.
1047
+ Write your complete review directly to ${planDir}/review-output-selective.md
1048
+
1049
+ Focus on reviewing the plan content: architecture, error handling, security, and performance.`,
1050
+ workingDirectory: planDir,
1051
+ maxTurns: 15,
1052
+ timeout: 360_000,
1053
+ testName: 'plan-ceo-review-selective',
1054
+ runId,
1055
+ });
1056
+
1057
+ logCost('/plan-ceo-review (SELECTIVE)', result);
1058
+ recordE2E('/plan-ceo-review-selective', 'Plan CEO Review SELECTIVE EXPANSION E2E', result, {
1059
+ passed: ['success', 'error_max_turns'].includes(result.exitReason),
1060
+ });
1061
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
1062
+
1063
+ const reviewPath = path.join(planDir, 'review-output-selective.md');
1064
+ if (fs.existsSync(reviewPath)) {
1065
+ const review = fs.readFileSync(reviewPath, 'utf-8');
1066
+ expect(review.length).toBeGreaterThan(200);
1067
+ }
1068
+ }, 420_000);
1069
+ });
1070
+
1071
+ // --- Plan Eng Review E2E ---
1072
+
1073
+ describeIfSelected('Plan Eng Review E2E', ['plan-eng-review'], () => {
1074
+ let planDir: string;
1075
+
1076
+ beforeAll(() => {
1077
+ planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-eng-'));
1078
+ const { spawnSync } = require('child_process');
1079
+ const run = (cmd: string, args: string[]) =>
1080
+ spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
1081
+
1082
+ run('git', ['init', '-b', 'main']);
1083
+ run('git', ['config', 'user.email', 'test@test.com']);
1084
+ run('git', ['config', 'user.name', 'Test']);
1085
+
1086
+ // Create a plan with more engineering detail
1087
+ fs.writeFileSync(path.join(planDir, 'plan.md'), `# Plan: Migrate Auth to JWT
1088
+
1089
+ ## Context
1090
+ Replace session-cookie auth with JWT tokens. Currently using express-session + Redis store.
1091
+
1092
+ ## Changes
1093
+ 1. Add \`jsonwebtoken\` package
1094
+ 2. New middleware \`auth/jwt-verify.ts\` replacing \`auth/session-check.ts\`
1095
+ 3. Login endpoint returns { accessToken, refreshToken }
1096
+ 4. Refresh endpoint rotates tokens
1097
+ 5. Migration script to invalidate existing sessions
1098
+
1099
+ ## Files Modified
1100
+ | File | Change |
1101
+ |------|--------|
1102
+ | auth/jwt-verify.ts | NEW: JWT verification middleware |
1103
+ | auth/session-check.ts | DELETED |
1104
+ | routes/login.ts | Return JWT instead of setting cookie |
1105
+ | routes/refresh.ts | NEW: Token refresh endpoint |
1106
+ | middleware/index.ts | Swap session-check for jwt-verify |
1107
+
1108
+ ## Error handling
1109
+ - Expired token: 401 with \`token_expired\` code
1110
+ - Invalid token: 401 with \`invalid_token\` code
1111
+ - Refresh with revoked token: 403
1112
+
1113
+ ## Not in scope
1114
+ - OAuth/OIDC integration
1115
+ - Rate limiting on refresh endpoint
1116
+ `);
1117
+
1118
+ run('git', ['add', '.']);
1119
+ run('git', ['commit', '-m', 'add plan']);
1120
+
1121
+ // Copy plan-eng-review skill
1122
+ fs.mkdirSync(path.join(planDir, 'plan-eng-review'), { recursive: true });
1123
+ fs.copyFileSync(
1124
+ path.join(ROOT, 'plan-eng-review', 'SKILL.md'),
1125
+ path.join(planDir, 'plan-eng-review', 'SKILL.md'),
1126
+ );
1127
+ });
1128
+
1129
+ afterAll(() => {
1130
+ try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
1131
+ });
1132
+
1133
+ test('/plan-eng-review produces structured review output', async () => {
1134
+ const result = await runSkillTest({
1135
+ prompt: `Read plan-eng-review/SKILL.md for the review workflow.
1136
+
1137
+ Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration steps.
1138
+
1139
+ Proceed directly to the full review. Skip any AskUserQuestion calls — this is non-interactive.
1140
+ Write your complete review directly to ${planDir}/review-output.md
1141
+
1142
+ Focus on architecture, code quality, tests, and performance sections.`,
1143
+ workingDirectory: planDir,
1144
+ maxTurns: 15,
1145
+ timeout: 360_000,
1146
+ testName: 'plan-eng-review',
1147
+ runId,
1148
+ });
1149
+
1150
+ logCost('/plan-eng-review', result);
1151
+ recordE2E('/plan-eng-review', 'Plan Eng Review E2E', result, {
1152
+ passed: ['success', 'error_max_turns'].includes(result.exitReason),
1153
+ });
1154
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
1155
+
1156
+ // Verify the review was written
1157
+ const reviewPath = path.join(planDir, 'review-output.md');
1158
+ if (fs.existsSync(reviewPath)) {
1159
+ const review = fs.readFileSync(reviewPath, 'utf-8');
1160
+ expect(review.length).toBeGreaterThan(200);
1161
+ }
1162
+ }, 420_000);
1163
+ });
1164
+
1165
+ // --- Retro E2E ---
1166
+
1167
+ describeIfSelected('Retro E2E', ['retro'], () => {
1168
+ let retroDir: string;
1169
+
1170
+ beforeAll(() => {
1171
+ retroDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-retro-'));
1172
+ const { spawnSync } = require('child_process');
1173
+ const run = (cmd: string, args: string[]) =>
1174
+ spawnSync(cmd, args, { cwd: retroDir, stdio: 'pipe', timeout: 5000 });
1175
+
1176
+ // Create a git repo with varied commit history
1177
+ run('git', ['init', '-b', 'main']);
1178
+ run('git', ['config', 'user.email', 'dev@example.com']);
1179
+ run('git', ['config', 'user.name', 'Dev']);
1180
+
1181
+ // Day 1 commits
1182
+ fs.writeFileSync(path.join(retroDir, 'app.ts'), 'console.log("hello");\n');
1183
+ run('git', ['add', 'app.ts']);
1184
+ run('git', ['commit', '-m', 'feat: initial app setup', '--date', '2026-03-10T09:00:00']);
1185
+
1186
+ fs.writeFileSync(path.join(retroDir, 'auth.ts'), 'export function login() {}\n');
1187
+ run('git', ['add', 'auth.ts']);
1188
+ run('git', ['commit', '-m', 'feat: add auth module', '--date', '2026-03-10T11:00:00']);
1189
+
1190
+ // Day 2 commits
1191
+ fs.writeFileSync(path.join(retroDir, 'app.ts'), 'import { login } from "./auth";\nconsole.log("hello");\nlogin();\n');
1192
+ run('git', ['add', 'app.ts']);
1193
+ run('git', ['commit', '-m', 'fix: wire up auth to app', '--date', '2026-03-11T10:00:00']);
1194
+
1195
+ fs.writeFileSync(path.join(retroDir, 'test.ts'), 'import { test } from "bun:test";\ntest("login", () => {});\n');
1196
+ run('git', ['add', 'test.ts']);
1197
+ run('git', ['commit', '-m', 'test: add login test', '--date', '2026-03-11T14:00:00']);
1198
+
1199
+ // Day 3 commits
1200
+ fs.writeFileSync(path.join(retroDir, 'api.ts'), 'export function getUsers() { return []; }\n');
1201
+ run('git', ['add', 'api.ts']);
1202
+ run('git', ['commit', '-m', 'feat: add users API endpoint', '--date', '2026-03-12T09:30:00']);
1203
+
1204
+ fs.writeFileSync(path.join(retroDir, 'README.md'), '# My App\nA test application.\n');
1205
+ run('git', ['add', 'README.md']);
1206
+ run('git', ['commit', '-m', 'docs: add README', '--date', '2026-03-12T16:00:00']);
1207
+
1208
+ // Copy retro skill
1209
+ fs.mkdirSync(path.join(retroDir, 'retro'), { recursive: true });
1210
+ fs.copyFileSync(
1211
+ path.join(ROOT, 'retro', 'SKILL.md'),
1212
+ path.join(retroDir, 'retro', 'SKILL.md'),
1213
+ );
1214
+ });
1215
+
1216
+ afterAll(() => {
1217
+ try { fs.rmSync(retroDir, { recursive: true, force: true }); } catch {}
1218
+ });
1219
+
1220
+ test('/retro produces analysis from git history', async () => {
1221
+ const result = await runSkillTest({
1222
+ prompt: `Read retro/SKILL.md for instructions on how to run a retrospective.
1223
+
1224
+ Run /retro for the last 7 days of this git repo. Skip any AskUserQuestion calls — this is non-interactive.
1225
+ Write your retrospective report to ${retroDir}/retro-output.md
1226
+
1227
+ Analyze the git history and produce the narrative report as described in the SKILL.md.`,
1228
+ workingDirectory: retroDir,
1229
+ maxTurns: 30,
1230
+ timeout: 300_000,
1231
+ testName: 'retro',
1232
+ runId,
1233
+ });
1234
+
1235
+ logCost('/retro', result);
1236
+ recordE2E('/retro', 'Retro E2E', result, {
1237
+ passed: ['success', 'error_max_turns'].includes(result.exitReason),
1238
+ });
1239
+ // Accept error_max_turns — retro does many git commands to analyze history
1240
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
1241
+
1242
+ // Verify the retro was written
1243
+ const retroPath = path.join(retroDir, 'retro-output.md');
1244
+ if (fs.existsSync(retroPath)) {
1245
+ const retro = fs.readFileSync(retroPath, 'utf-8');
1246
+ expect(retro.length).toBeGreaterThan(100);
1247
+ }
1248
+ }, 420_000);
1249
+ });
1250
+
1251
+ // --- QA-Only E2E (report-only, no fixes) ---
1252
+
1253
+ describeIfSelected('QA-Only skill E2E', ['qa-only-no-fix'], () => {
1254
+ let qaOnlyDir: string;
1255
+
1256
+ beforeAll(() => {
1257
+ testServer = testServer || startTestServer();
1258
+ qaOnlyDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-qa-only-'));
1259
+ setupBrowseShims(qaOnlyDir);
1260
+
1261
+ // Copy qa-only skill files
1262
+ copyDirSync(path.join(ROOT, 'qa-only'), path.join(qaOnlyDir, 'qa-only'));
1263
+
1264
+ // Copy qa templates (qa-only references qa/templates/qa-report-template.md)
1265
+ fs.mkdirSync(path.join(qaOnlyDir, 'qa', 'templates'), { recursive: true });
1266
+ fs.copyFileSync(
1267
+ path.join(ROOT, 'qa', 'templates', 'qa-report-template.md'),
1268
+ path.join(qaOnlyDir, 'qa', 'templates', 'qa-report-template.md'),
1269
+ );
1270
+
1271
+ // Init git repo (qa-only checks for feature branch in diff-aware mode)
1272
+ const { spawnSync } = require('child_process');
1273
+ const run = (cmd: string, args: string[]) =>
1274
+ spawnSync(cmd, args, { cwd: qaOnlyDir, stdio: 'pipe', timeout: 5000 });
1275
+
1276
+ run('git', ['init', '-b', 'main']);
1277
+ run('git', ['config', 'user.email', 'test@test.com']);
1278
+ run('git', ['config', 'user.name', 'Test']);
1279
+ fs.writeFileSync(path.join(qaOnlyDir, 'index.html'), '<h1>Test</h1>\n');
1280
+ run('git', ['add', '.']);
1281
+ run('git', ['commit', '-m', 'initial']);
1282
+ });
1283
+
1284
+ afterAll(() => {
1285
+ try { fs.rmSync(qaOnlyDir, { recursive: true, force: true }); } catch {}
1286
+ });
1287
+
1288
+ test('/qa-only produces report without using Edit tool', async () => {
1289
+ const result = await runSkillTest({
1290
+ prompt: `IMPORTANT: The browse binary is already assigned below as B. Do NOT search for it or run the SKILL.md setup block — just use $B directly.
1291
+
1292
+ B="${browseBin}"
1293
+
1294
+ Read the file qa-only/SKILL.md for the QA-only workflow instructions.
1295
+
1296
+ Run a Quick QA test on ${testServer.url}/qa-eval.html
1297
+ Do NOT use AskUserQuestion — run Quick tier directly.
1298
+ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
1299
+ workingDirectory: qaOnlyDir,
1300
+ maxTurns: 35,
1301
+ allowedTools: ['Bash', 'Read', 'Write', 'Glob'], // NO Edit — the critical guardrail
1302
+ timeout: 180_000,
1303
+ testName: 'qa-only-no-fix',
1304
+ runId,
1305
+ });
1306
+
1307
+ logCost('/qa-only', result);
1308
+
1309
+ // Verify Edit was not used — the critical guardrail for report-only mode.
1310
+ // Glob is read-only and may be used for file discovery (e.g. finding SKILL.md).
1311
+ const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit');
1312
+ if (editCalls.length > 0) {
1313
+ console.warn('qa-only used Edit tool:', editCalls.length, 'times');
1314
+ }
1315
+
1316
+ const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);
1317
+ recordE2E('/qa-only no-fix', 'QA-Only skill E2E', result, {
1318
+ passed: exitOk && editCalls.length === 0,
1319
+ });
1320
+
1321
+ expect(editCalls).toHaveLength(0);
1322
+
1323
+ // Accept error_max_turns — the agent doing thorough QA is not a failure
1324
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
1325
+
1326
+ // Verify git working tree is still clean (no source modifications)
1327
+ const gitStatus = spawnSync('git', ['status', '--porcelain'], {
1328
+ cwd: qaOnlyDir, stdio: 'pipe',
1329
+ });
1330
+ const statusLines = gitStatus.stdout.toString().trim().split('\n').filter(
1331
+ (l: string) => l.trim() && !l.includes('.prompt-tmp') && !l.includes('.gstack/') && !l.includes('qa-reports/'),
1332
+ );
1333
+ expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0);
1334
+ }, 240_000);
1335
+ });
1336
+
1337
+ // --- QA Fix Loop E2E ---
1338
+
1339
+ describeIfSelected('QA Fix Loop E2E', ['qa-fix-loop'], () => {
1340
+ let qaFixDir: string;
1341
+ let qaFixServer: ReturnType<typeof Bun.serve> | null = null;
1342
+
1343
+ beforeAll(() => {
1344
+ qaFixDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-qa-fix-'));
1345
+ setupBrowseShims(qaFixDir);
1346
+
1347
+ // Copy qa skill files
1348
+ copyDirSync(path.join(ROOT, 'qa'), path.join(qaFixDir, 'qa'));
1349
+
1350
+ // Create a simple HTML page with obvious fixable bugs
1351
+ fs.writeFileSync(path.join(qaFixDir, 'index.html'), `<!DOCTYPE html>
1352
+ <html lang="en">
1353
+ <head><meta charset="utf-8"><title>Test App</title></head>
1354
+ <body>
1355
+ <h1>Welcome to Test App</h1>
1356
+ <nav>
1357
+ <a href="/about">About</a>
1358
+ <a href="/nonexistent-broken-page">Help</a> <!-- BUG: broken link -->
1359
+ </nav>
1360
+ <form id="contact">
1361
+ <input type="text" name="name" placeholder="Name">
1362
+ <input type="email" name="email" placeholder="Email">
1363
+ <button type="submit" disabled>Send</button> <!-- BUG: permanently disabled -->
1364
+ </form>
1365
+ <img src="/missing-logo.png"> <!-- BUG: missing alt text -->
1366
+ <script>console.error("TypeError: Cannot read property 'map' of undefined");</script> <!-- BUG: console error -->
1367
+ </body>
1368
+ </html>
1369
+ `);
1370
+
1371
+ // Init git repo with clean working tree
1372
+ const { spawnSync } = require('child_process');
1373
+ const run = (cmd: string, args: string[]) =>
1374
+ spawnSync(cmd, args, { cwd: qaFixDir, stdio: 'pipe', timeout: 5000 });
1375
+
1376
+ run('git', ['init', '-b', 'main']);
1377
+ run('git', ['config', 'user.email', 'test@test.com']);
1378
+ run('git', ['config', 'user.name', 'Test']);
1379
+ run('git', ['add', '.']);
1380
+ run('git', ['commit', '-m', 'initial commit']);
1381
+
1382
+ // Start a local server serving from the working directory so fixes are reflected on refresh
1383
+ qaFixServer = Bun.serve({
1384
+ port: 0,
1385
+ hostname: '127.0.0.1',
1386
+ fetch(req) {
1387
+ const url = new URL(req.url);
1388
+ let filePath = url.pathname === '/' ? '/index.html' : url.pathname;
1389
+ filePath = filePath.replace(/^\//, '');
1390
+ const fullPath = path.join(qaFixDir, filePath);
1391
+ if (!fs.existsSync(fullPath)) {
1392
+ return new Response('Not Found', { status: 404 });
1393
+ }
1394
+ const content = fs.readFileSync(fullPath, 'utf-8');
1395
+ return new Response(content, {
1396
+ headers: { 'Content-Type': 'text/html' },
1397
+ });
1398
+ },
1399
+ });
1400
+ });
1401
+
1402
+ afterAll(() => {
1403
+ qaFixServer?.stop();
1404
+ try { fs.rmSync(qaFixDir, { recursive: true, force: true }); } catch {}
1405
+ });
1406
+
1407
+ test('/qa fix loop finds bugs and commits fixes', async () => {
1408
+ const qaFixUrl = `http://127.0.0.1:${qaFixServer!.port}`;
1409
+
1410
+ const result = await runSkillTest({
1411
+ prompt: `You have a browse binary at ${browseBin}. Assign it to B variable like: B="${browseBin}"
1412
+
1413
+ Read the file qa/SKILL.md for the QA workflow instructions.
1414
+
1415
+ Run a Quick-tier QA test on ${qaFixUrl}
1416
+ The source code for this page is at ${qaFixDir}/index.html — you can fix bugs there.
1417
+ Do NOT use AskUserQuestion — run Quick tier directly.
1418
+ Write your report to ${qaFixDir}/qa-reports/qa-report.md
1419
+
1420
+ This is a test+fix loop: find bugs, fix them in the source code, commit each fix, and re-verify.`,
1421
+ workingDirectory: qaFixDir,
1422
+ maxTurns: 40,
1423
+ allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
1424
+ timeout: 300_000,
1425
+ testName: 'qa-fix-loop',
1426
+ runId,
1427
+ });
1428
+
1429
+ logCost('/qa fix loop', result);
1430
+ recordE2E('/qa fix loop', 'QA Fix Loop E2E', result, {
1431
+ passed: ['success', 'error_max_turns'].includes(result.exitReason),
1432
+ });
1433
+
1434
+ // Accept error_max_turns — fix loop may use many turns
1435
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
1436
+
1437
+ // Verify at least one fix commit was made beyond the initial commit
1438
+ const gitLog = spawnSync('git', ['log', '--oneline'], {
1439
+ cwd: qaFixDir, stdio: 'pipe',
1440
+ });
1441
+ const commits = gitLog.stdout.toString().trim().split('\n');
1442
+ console.log(`/qa fix loop: ${commits.length} commits total (1 initial + ${commits.length - 1} fixes)`);
1443
+ expect(commits.length).toBeGreaterThan(1);
1444
+
1445
+ // Verify Edit tool was used (agent actually modified source code)
1446
+ const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit');
1447
+ expect(editCalls.length).toBeGreaterThan(0);
1448
+ }, 360_000);
1449
+ });
1450
+
1451
+ // --- Plan-Eng-Review Test-Plan Artifact E2E ---
1452
+
1453
+ describeIfSelected('Plan-Eng-Review Test-Plan Artifact E2E', ['plan-eng-review-artifact'], () => {
1454
+ let planDir: string;
1455
+ let projectDir: string;
1456
+
1457
+ beforeAll(() => {
1458
+ planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-artifact-'));
1459
+ const { spawnSync } = require('child_process');
1460
+ const run = (cmd: string, args: string[]) =>
1461
+ spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
1462
+
1463
+ run('git', ['init', '-b', 'main']);
1464
+ run('git', ['config', 'user.email', 'test@test.com']);
1465
+ run('git', ['config', 'user.name', 'Test']);
1466
+
1467
+ // Create base commit on main
1468
+ fs.writeFileSync(path.join(planDir, 'app.ts'), 'export function greet() { return "hello"; }\n');
1469
+ run('git', ['add', '.']);
1470
+ run('git', ['commit', '-m', 'initial']);
1471
+
1472
+ // Create feature branch with changes
1473
+ run('git', ['checkout', '-b', 'feature/add-dashboard']);
1474
+ fs.writeFileSync(path.join(planDir, 'dashboard.ts'), `export function Dashboard() {
1475
+ const data = fetchStats();
1476
+ return { users: data.users, revenue: data.revenue };
1477
+ }
1478
+ function fetchStats() {
1479
+ return fetch('/api/stats').then(r => r.json());
1480
+ }
1481
+ `);
1482
+ fs.writeFileSync(path.join(planDir, 'app.ts'), `import { Dashboard } from "./dashboard";
1483
+ export function greet() { return "hello"; }
1484
+ export function main() { return Dashboard(); }
1485
+ `);
1486
+ run('git', ['add', '.']);
1487
+ run('git', ['commit', '-m', 'feat: add dashboard']);
1488
+
1489
+ // Plan document
1490
+ fs.writeFileSync(path.join(planDir, 'plan.md'), `# Plan: Add Dashboard
1491
+
1492
+ ## Changes
1493
+ 1. New \`dashboard.ts\` with Dashboard component and fetchStats API call
1494
+ 2. Updated \`app.ts\` to import and use Dashboard
1495
+
1496
+ ## Architecture
1497
+ - Dashboard fetches from \`/api/stats\` endpoint
1498
+ - Returns user count and revenue metrics
1499
+ `);
1500
+ run('git', ['add', 'plan.md']);
1501
+ run('git', ['commit', '-m', 'add plan']);
1502
+
1503
+ // Copy plan-eng-review skill
1504
+ fs.mkdirSync(path.join(planDir, 'plan-eng-review'), { recursive: true });
1505
+ fs.copyFileSync(
1506
+ path.join(ROOT, 'plan-eng-review', 'SKILL.md'),
1507
+ path.join(planDir, 'plan-eng-review', 'SKILL.md'),
1508
+ );
1509
+
1510
+ // Set up remote-slug shim and browse shims (plan-eng-review uses remote-slug for artifact path)
1511
+ setupBrowseShims(planDir);
1512
+
1513
+ // Create project directory for artifacts
1514
+ projectDir = path.join(os.homedir(), '.gstack', 'projects', 'test-project');
1515
+ fs.mkdirSync(projectDir, { recursive: true });
1516
+ });
1517
+
1518
+ afterAll(() => {
1519
+ try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
1520
+ // Clean up test-plan artifacts (but not the project dir itself)
1521
+ try {
1522
+ const files = fs.readdirSync(projectDir);
1523
+ for (const f of files) {
1524
+ if (f.includes('test-plan')) {
1525
+ fs.unlinkSync(path.join(projectDir, f));
1526
+ }
1527
+ }
1528
+ } catch {}
1529
+ });
1530
+
1531
+ test('/plan-eng-review writes test-plan artifact to ~/.gstack/projects/', async () => {
1532
+ // Count existing test-plan files before
1533
+ const beforeFiles = fs.readdirSync(projectDir).filter(f => f.includes('test-plan'));
1534
+
1535
+ const result = await runSkillTest({
1536
+ prompt: `Read plan-eng-review/SKILL.md for the review workflow.
1537
+
1538
+ Read plan.md — that's the plan to review. This is a standalone plan with source code in app.ts and dashboard.ts.
1539
+
1540
+ Proceed directly to the full review. Skip any AskUserQuestion calls — this is non-interactive.
1541
+
1542
+ IMPORTANT: After your review, you MUST write the test-plan artifact as described in the "Test Plan Artifact" section of SKILL.md. The remote-slug shim is at ${planDir}/browse/bin/remote-slug.
1543
+
1544
+ Write your review to ${planDir}/review-output.md`,
1545
+ workingDirectory: planDir,
1546
+ maxTurns: 20,
1547
+ allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'],
1548
+ timeout: 360_000,
1549
+ testName: 'plan-eng-review-artifact',
1550
+ runId,
1551
+ });
1552
+
1553
+ logCost('/plan-eng-review artifact', result);
1554
+ recordE2E('/plan-eng-review test-plan artifact', 'Plan-Eng-Review Test-Plan Artifact E2E', result, {
1555
+ passed: ['success', 'error_max_turns'].includes(result.exitReason),
1556
+ });
1557
+
1558
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
1559
+
1560
+ // Verify test-plan artifact was written
1561
+ const afterFiles = fs.readdirSync(projectDir).filter(f => f.includes('test-plan'));
1562
+ const newFiles = afterFiles.filter(f => !beforeFiles.includes(f));
1563
+ console.log(`Test-plan artifacts: ${beforeFiles.length} before, ${afterFiles.length} after, ${newFiles.length} new`);
1564
+
1565
+ if (newFiles.length > 0) {
1566
+ const content = fs.readFileSync(path.join(projectDir, newFiles[0]), 'utf-8');
1567
+ console.log(`Test-plan artifact (${newFiles[0]}): ${content.length} chars`);
1568
+ expect(content.length).toBeGreaterThan(50);
1569
+ } else {
1570
+ console.warn('No test-plan artifact found — agent may not have followed artifact instructions');
1571
+ }
1572
+
1573
+ // Soft assertion: we expect an artifact but agent compliance is not guaranteed
1574
+ expect(newFiles.length).toBeGreaterThanOrEqual(1);
1575
+ }, 420_000);
1576
+ });
1577
+
1578
+ // --- Base branch detection smoke tests ---
1579
+
1580
+ describeIfSelected('Base branch detection', ['review-base-branch', 'ship-base-branch', 'retro-base-branch'], () => {
1581
+ let baseBranchDir: string;
1582
+ const run = (cmd: string, args: string[], cwd: string) =>
1583
+ spawnSync(cmd, args, { cwd, stdio: 'pipe', timeout: 5000 });
1584
+
1585
+ beforeAll(() => {
1586
+ baseBranchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-basebranch-'));
1587
+ });
1588
+
1589
+ afterAll(() => {
1590
+ try { fs.rmSync(baseBranchDir, { recursive: true, force: true }); } catch {}
1591
+ });
1592
+
1593
+ testIfSelected('review-base-branch', async () => {
1594
+ const dir = path.join(baseBranchDir, 'review-base');
1595
+ fs.mkdirSync(dir, { recursive: true });
1596
+
1597
+ // Create git repo with a feature branch off main
1598
+ run('git', ['init'], dir);
1599
+ run('git', ['config', 'user.email', 'test@test.com'], dir);
1600
+ run('git', ['config', 'user.name', 'Test'], dir);
1601
+
1602
+ fs.writeFileSync(path.join(dir, 'app.rb'), '# clean base\nclass App\nend\n');
1603
+ run('git', ['add', 'app.rb'], dir);
1604
+ run('git', ['commit', '-m', 'initial commit'], dir);
1605
+
1606
+ // Create feature branch with a change
1607
+ run('git', ['checkout', '-b', 'feature/test-review'], dir);
1608
+ fs.writeFileSync(path.join(dir, 'app.rb'), '# clean base\nclass App\n def hello; "world"; end\nend\n');
1609
+ run('git', ['add', 'app.rb'], dir);
1610
+ run('git', ['commit', '-m', 'feat: add hello method'], dir);
1611
+
1612
+ // Copy review skill files
1613
+ fs.copyFileSync(path.join(ROOT, 'review', 'SKILL.md'), path.join(dir, 'review-SKILL.md'));
1614
+ fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(dir, 'review-checklist.md'));
1615
+ fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(dir, 'review-greptile-triage.md'));
1616
+
1617
+ const result = await runSkillTest({
1618
+ prompt: `You are in a git repo on a feature branch with changes.
1619
+ Read review-SKILL.md for the review workflow instructions.
1620
+ Also read review-checklist.md and apply it.
1621
+
1622
+ IMPORTANT: Follow Step 0 to detect the base branch. Since there is no remote, gh commands will fail — fall back to main.
1623
+ Then run the review against the detected base branch.
1624
+ Write your findings to ${dir}/review-output.md`,
1625
+ workingDirectory: dir,
1626
+ maxTurns: 15,
1627
+ timeout: 90_000,
1628
+ testName: 'review-base-branch',
1629
+ runId,
1630
+ });
1631
+
1632
+ logCost('/review base-branch', result);
1633
+ recordE2E('/review base branch detection', 'Base branch detection', result);
1634
+ expect(result.exitReason).toBe('success');
1635
+
1636
+ // Verify the review used "base branch" language (from Step 0)
1637
+ const toolOutputs = result.toolCalls.map(tc => tc.output || '').join('\n');
1638
+ const allOutput = (result.output || '') + toolOutputs;
1639
+ // The agent should have run git diff against main (the fallback)
1640
+ const usedGitDiff = result.toolCalls.some(tc =>
1641
+ tc.tool === 'Bash' && typeof tc.input === 'string' && tc.input.includes('git diff')
1642
+ );
1643
+ expect(usedGitDiff).toBe(true);
1644
+ }, 120_000);
1645
+
1646
+ testIfSelected('ship-base-branch', async () => {
1647
+ const dir = path.join(baseBranchDir, 'ship-base');
1648
+ fs.mkdirSync(dir, { recursive: true });
1649
+
1650
+ // Create git repo with feature branch
1651
+ run('git', ['init'], dir);
1652
+ run('git', ['config', 'user.email', 'test@test.com'], dir);
1653
+ run('git', ['config', 'user.name', 'Test'], dir);
1654
+
1655
+ fs.writeFileSync(path.join(dir, 'app.ts'), 'console.log("v1");\n');
1656
+ run('git', ['add', 'app.ts'], dir);
1657
+ run('git', ['commit', '-m', 'initial'], dir);
1658
+
1659
+ run('git', ['checkout', '-b', 'feature/ship-test'], dir);
1660
+ fs.writeFileSync(path.join(dir, 'app.ts'), 'console.log("v2");\n');
1661
+ run('git', ['add', 'app.ts'], dir);
1662
+ run('git', ['commit', '-m', 'feat: update to v2'], dir);
1663
+
1664
+ // Copy ship skill
1665
+ fs.copyFileSync(path.join(ROOT, 'ship', 'SKILL.md'), path.join(dir, 'ship-SKILL.md'));
1666
+
1667
+ const result = await runSkillTest({
1668
+ prompt: `Read ship-SKILL.md for the ship workflow.
1669
+
1670
+ Run ONLY Step 0 (Detect base branch) and Step 1 (Pre-flight) from the ship workflow.
1671
+ Since there is no remote, gh commands will fail — fall back to main.
1672
+
1673
+ After completing Step 0 and Step 1, STOP. Do NOT proceed to Step 2 or beyond.
1674
+ Do NOT push, create PRs, or modify VERSION/CHANGELOG.
1675
+
1676
+ Write a summary of what you detected to ${dir}/ship-preflight.md including:
1677
+ - The detected base branch name
1678
+ - The current branch name
1679
+ - The diff stat against the base branch`,
1680
+ workingDirectory: dir,
1681
+ maxTurns: 10,
1682
+ timeout: 60_000,
1683
+ testName: 'ship-base-branch',
1684
+ runId,
1685
+ });
1686
+
1687
+ logCost('/ship base-branch', result);
1688
+ recordE2E('/ship base branch detection', 'Base branch detection', result);
1689
+ expect(result.exitReason).toBe('success');
1690
+
1691
+ // Verify preflight output was written
1692
+ const preflightPath = path.join(dir, 'ship-preflight.md');
1693
+ if (fs.existsSync(preflightPath)) {
1694
+ const content = fs.readFileSync(preflightPath, 'utf-8');
1695
+ expect(content.length).toBeGreaterThan(20);
1696
+ // Should mention the branch name
1697
+ expect(content.toLowerCase()).toMatch(/main|base/);
1698
+ }
1699
+
1700
+ // Verify no destructive actions — no push, no PR creation
1701
+ const destructiveTools = result.toolCalls.filter(tc =>
1702
+ tc.tool === 'Bash' && typeof tc.input === 'string' &&
1703
+ (tc.input.includes('git push') || tc.input.includes('gh pr create'))
1704
+ );
1705
+ expect(destructiveTools).toHaveLength(0);
1706
+ }, 90_000);
1707
+
1708
+ testIfSelected('retro-base-branch', async () => {
1709
+ const dir = path.join(baseBranchDir, 'retro-base');
1710
+ fs.mkdirSync(dir, { recursive: true });
1711
+
1712
+ // Create git repo with commit history
1713
+ run('git', ['init'], dir);
1714
+ run('git', ['config', 'user.email', 'dev@example.com'], dir);
1715
+ run('git', ['config', 'user.name', 'Dev'], dir);
1716
+
1717
+ fs.writeFileSync(path.join(dir, 'app.ts'), 'console.log("hello");\n');
1718
+ run('git', ['add', 'app.ts'], dir);
1719
+ run('git', ['commit', '-m', 'feat: initial app', '--date', '2026-03-14T09:00:00'], dir);
1720
+
1721
+ fs.writeFileSync(path.join(dir, 'auth.ts'), 'export function login() {}\n');
1722
+ run('git', ['add', 'auth.ts'], dir);
1723
+ run('git', ['commit', '-m', 'feat: add auth', '--date', '2026-03-15T10:00:00'], dir);
1724
+
1725
+ fs.writeFileSync(path.join(dir, 'test.ts'), 'test("it works", () => {});\n');
1726
+ run('git', ['add', 'test.ts'], dir);
1727
+ run('git', ['commit', '-m', 'test: add tests', '--date', '2026-03-16T11:00:00'], dir);
1728
+
1729
+ // Copy retro skill
1730
+ fs.mkdirSync(path.join(dir, 'retro'), { recursive: true });
1731
+ fs.copyFileSync(path.join(ROOT, 'retro', 'SKILL.md'), path.join(dir, 'retro', 'SKILL.md'));
1732
+
1733
+ const result = await runSkillTest({
1734
+ prompt: `Read retro/SKILL.md for instructions on how to run a retrospective.
1735
+
1736
+ IMPORTANT: Follow the "Detect default branch" step first. Since there is no remote, gh will fail — fall back to main.
1737
+ Then use the detected branch name for all git queries.
1738
+
1739
+ Run /retro for the last 7 days of this git repo. Skip any AskUserQuestion calls — this is non-interactive.
1740
+ This is a local-only repo so use the local branch (main) instead of origin/main for all git log commands.
1741
+
1742
+ Write your retrospective to ${dir}/retro-output.md`,
1743
+ workingDirectory: dir,
1744
+ maxTurns: 25,
1745
+ timeout: 240_000,
1746
+ testName: 'retro-base-branch',
1747
+ runId,
1748
+ });
1749
+
1750
+ logCost('/retro base-branch', result);
1751
+ recordE2E('/retro default branch detection', 'Base branch detection', result, {
1752
+ passed: ['success', 'error_max_turns'].includes(result.exitReason),
1753
+ });
1754
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
1755
+
1756
+ // Verify retro output was produced
1757
+ const retroPath = path.join(dir, 'retro-output.md');
1758
+ if (fs.existsSync(retroPath)) {
1759
+ const content = fs.readFileSync(retroPath, 'utf-8');
1760
+ expect(content.length).toBeGreaterThan(100);
1761
+ }
1762
+ }, 300_000);
1763
+ });
1764
+
1765
+ // --- Document-Release skill E2E ---
1766
+
1767
+ describeIfSelected('Document-Release skill E2E', ['document-release'], () => {
1768
+ let docReleaseDir: string;
1769
+
1770
+ beforeAll(() => {
1771
+ docReleaseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-doc-release-'));
1772
+
1773
+ // Copy document-release skill files
1774
+ copyDirSync(path.join(ROOT, 'document-release'), path.join(docReleaseDir, 'document-release'));
1775
+
1776
+ // Init git repo with initial docs
1777
+ const run = (cmd: string, args: string[]) =>
1778
+ spawnSync(cmd, args, { cwd: docReleaseDir, stdio: 'pipe', timeout: 5000 });
1779
+
1780
+ run('git', ['init', '-b', 'main']);
1781
+ run('git', ['config', 'user.email', 'test@test.com']);
1782
+ run('git', ['config', 'user.name', 'Test']);
1783
+
1784
+ // Create initial README with a features list
1785
+ fs.writeFileSync(path.join(docReleaseDir, 'README.md'),
1786
+ '# Test Project\n\n## Features\n\n- Feature A\n- Feature B\n\n## Install\n\n```bash\nnpm install\n```\n');
1787
+
1788
+ // Create initial CHANGELOG that must NOT be clobbered
1789
+ fs.writeFileSync(path.join(docReleaseDir, 'CHANGELOG.md'),
1790
+ '# Changelog\n\n## 1.0.0 — 2026-03-01\n\n- Initial release with Feature A and Feature B\n- Setup CI pipeline\n');
1791
+
1792
+ // Create VERSION file (already bumped)
1793
+ fs.writeFileSync(path.join(docReleaseDir, 'VERSION'), '1.1.0\n');
1794
+
1795
+ run('git', ['add', '.']);
1796
+ run('git', ['commit', '-m', 'initial']);
1797
+
1798
+ // Create feature branch with a code change
1799
+ run('git', ['checkout', '-b', 'feat/add-feature-c']);
1800
+ fs.writeFileSync(path.join(docReleaseDir, 'feature-c.ts'), 'export function featureC() { return "C"; }\n');
1801
+ fs.writeFileSync(path.join(docReleaseDir, 'VERSION'), '1.1.1\n');
1802
+ fs.writeFileSync(path.join(docReleaseDir, 'CHANGELOG.md'),
1803
+ '# Changelog\n\n## 1.1.1 — 2026-03-16\n\n- Added Feature C\n\n## 1.0.0 — 2026-03-01\n\n- Initial release with Feature A and Feature B\n- Setup CI pipeline\n');
1804
+ run('git', ['add', '.']);
1805
+ run('git', ['commit', '-m', 'feat: add feature C']);
1806
+ });
1807
+
1808
+ afterAll(() => {
1809
+ try { fs.rmSync(docReleaseDir, { recursive: true, force: true }); } catch {}
1810
+ });
1811
+
1812
+ test('/document-release updates docs without clobbering CHANGELOG', async () => {
1813
+ const result = await runSkillTest({
1814
+ prompt: `Read the file document-release/SKILL.md for the document-release workflow instructions.
1815
+
1816
+ Run the /document-release workflow on this repo. The base branch is "main".
1817
+
1818
+ IMPORTANT:
1819
+ - Do NOT use AskUserQuestion — auto-approve everything or skip if unsure.
1820
+ - Do NOT push or create PRs (there is no remote).
1821
+ - Do NOT run gh commands (no remote).
1822
+ - Focus on updating README.md to reflect the new Feature C.
1823
+ - Do NOT overwrite or regenerate CHANGELOG entries.
1824
+ - Skip VERSION bump (it's already bumped).
1825
+ - After editing, just commit the changes locally.`,
1826
+ workingDirectory: docReleaseDir,
1827
+ maxTurns: 30,
1828
+ allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
1829
+ timeout: 180_000,
1830
+ testName: 'document-release',
1831
+ runId,
1832
+ });
1833
+
1834
+ logCost('/document-release', result);
1835
+
1836
+ // Read CHANGELOG to verify it was NOT clobbered
1837
+ const changelog = fs.readFileSync(path.join(docReleaseDir, 'CHANGELOG.md'), 'utf-8');
1838
+ const hasOriginalEntries = changelog.includes('Initial release with Feature A and Feature B')
1839
+ && changelog.includes('Setup CI pipeline')
1840
+ && changelog.includes('1.0.0');
1841
+ if (!hasOriginalEntries) {
1842
+ console.warn('CHANGELOG CLOBBERED — original entries missing!');
1843
+ }
1844
+
1845
+ // Check if README was updated
1846
+ const readme = fs.readFileSync(path.join(docReleaseDir, 'README.md'), 'utf-8');
1847
+ const readmeUpdated = readme.includes('Feature C') || readme.includes('feature-c') || readme.includes('feature C');
1848
+
1849
+ const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);
1850
+ recordE2E('/document-release', 'Document-Release skill E2E', result, {
1851
+ passed: exitOk && hasOriginalEntries,
1852
+ });
1853
+
1854
+ // Critical guardrail: CHANGELOG must not be clobbered
1855
+ expect(hasOriginalEntries).toBe(true);
1856
+
1857
+ // Accept error_max_turns — thorough doc review is not a failure
1858
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
1859
+
1860
+ // Informational: did it update README?
1861
+ if (readmeUpdated) {
1862
+ console.log('README updated to include Feature C');
1863
+ } else {
1864
+ console.warn('README was NOT updated — agent may not have found the feature');
1865
+ }
1866
+ }, 240_000);
1867
+ });
1868
+
1869
+ // --- Deferred skill E2E tests (destructive or require interactive UI) ---
1870
+
1871
+ // Deferred tests — only test.todo entries, no selection needed
1872
+ describeE2E('Deferred skill E2E', () => {
1873
+ // Ship is destructive: pushes to remote, creates PRs, modifies VERSION/CHANGELOG
1874
+ test.todo('/ship completes full workflow');
1875
+
1876
+ // Setup-browser-cookies requires interactive browser picker UI
1877
+ test.todo('/setup-browser-cookies imports cookies');
1878
+
1879
+ });
1880
+
1881
+ // --- gstack-upgrade E2E ---
1882
+
1883
+ describeIfSelected('gstack-upgrade E2E', ['gstack-upgrade-happy-path'], () => {
1884
+ let upgradeDir: string;
1885
+ let remoteDir: string;
1886
+
1887
+ beforeAll(() => {
1888
+ upgradeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-upgrade-'));
1889
+ remoteDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-remote-'));
1890
+
1891
+ const run = (cmd: string, args: string[], cwd: string) =>
1892
+ spawnSync(cmd, args, { cwd, stdio: 'pipe', timeout: 5000 });
1893
+
1894
+ // Init the "project" repo
1895
+ run('git', ['init'], upgradeDir);
1896
+ run('git', ['config', 'user.email', 'test@test.com'], upgradeDir);
1897
+ run('git', ['config', 'user.name', 'Test'], upgradeDir);
1898
+
1899
+ // Create mock gstack install directory (local-git type)
1900
+ const mockGstack = path.join(upgradeDir, '.claude', 'skills', 'gstack');
1901
+ fs.mkdirSync(mockGstack, { recursive: true });
1902
+
1903
+ // Init as a git repo
1904
+ run('git', ['init'], mockGstack);
1905
+ run('git', ['config', 'user.email', 'test@test.com'], mockGstack);
1906
+ run('git', ['config', 'user.name', 'Test'], mockGstack);
1907
+
1908
+ // Create bare remote
1909
+ run('git', ['init', '--bare'], remoteDir);
1910
+ run('git', ['remote', 'add', 'origin', remoteDir], mockGstack);
1911
+
1912
+ // Write old version files
1913
+ fs.writeFileSync(path.join(mockGstack, 'VERSION'), '0.5.0\n');
1914
+ fs.writeFileSync(path.join(mockGstack, 'CHANGELOG.md'),
1915
+ '# Changelog\n\n## 0.5.0 — 2026-03-01\n\n- Initial release\n');
1916
+ fs.writeFileSync(path.join(mockGstack, 'setup'),
1917
+ '#!/bin/bash\necho "Setup completed"\n', { mode: 0o755 });
1918
+
1919
+ // Initial commit + push
1920
+ run('git', ['add', '.'], mockGstack);
1921
+ run('git', ['commit', '-m', 'initial'], mockGstack);
1922
+ run('git', ['push', '-u', 'origin', 'HEAD:main'], mockGstack);
1923
+
1924
+ // Create new version (simulate upstream release)
1925
+ fs.writeFileSync(path.join(mockGstack, 'VERSION'), '0.6.0\n');
1926
+ fs.writeFileSync(path.join(mockGstack, 'CHANGELOG.md'),
1927
+ '# Changelog\n\n## 0.6.0 — 2026-03-15\n\n- New feature: interactive design review\n- Fix: snapshot flag validation\n\n## 0.5.0 — 2026-03-01\n\n- Initial release\n');
1928
+ run('git', ['add', '.'], mockGstack);
1929
+ run('git', ['commit', '-m', 'release 0.6.0'], mockGstack);
1930
+ run('git', ['push', 'origin', 'HEAD:main'], mockGstack);
1931
+
1932
+ // Reset working copy back to old version
1933
+ run('git', ['reset', '--hard', 'HEAD~1'], mockGstack);
1934
+
1935
+ // Copy gstack-upgrade skill
1936
+ fs.mkdirSync(path.join(upgradeDir, 'gstack-upgrade'), { recursive: true });
1937
+ fs.copyFileSync(
1938
+ path.join(ROOT, 'gstack-upgrade', 'SKILL.md'),
1939
+ path.join(upgradeDir, 'gstack-upgrade', 'SKILL.md'),
1940
+ );
1941
+
1942
+ // Commit so git repo is clean
1943
+ run('git', ['add', '.'], upgradeDir);
1944
+ run('git', ['commit', '-m', 'initial project'], upgradeDir);
1945
+ });
1946
+
1947
+ afterAll(() => {
1948
+ try { fs.rmSync(upgradeDir, { recursive: true, force: true }); } catch {}
1949
+ try { fs.rmSync(remoteDir, { recursive: true, force: true }); } catch {}
1950
+ });
1951
+
1952
+ testIfSelected('gstack-upgrade-happy-path', async () => {
1953
+ const mockGstack = path.join(upgradeDir, '.claude', 'skills', 'gstack');
1954
+ const result = await runSkillTest({
1955
+ prompt: `Read gstack-upgrade/SKILL.md for the upgrade workflow.
1956
+
1957
+ You are running /gstack-upgrade standalone. The gstack installation is at ./.claude/skills/gstack (local-git type — it has a .git directory with an origin remote).
1958
+
1959
+ Current version: 0.5.0. A new version 0.6.0 is available on origin/main.
1960
+
1961
+ Follow the standalone upgrade flow:
1962
+ 1. Detect install type (local-git)
1963
+ 2. Run git fetch origin && git reset --hard origin/main in the install directory
1964
+ 3. Run the setup script
1965
+ 4. Show what's new from CHANGELOG
1966
+
1967
+ Skip any AskUserQuestion calls — auto-approve the upgrade. Write a summary of what you did to stdout.
1968
+
1969
+ IMPORTANT: The install directory is at ./.claude/skills/gstack — use that exact path.`,
1970
+ workingDirectory: upgradeDir,
1971
+ maxTurns: 20,
1972
+ timeout: 180_000,
1973
+ testName: 'gstack-upgrade-happy-path',
1974
+ runId,
1975
+ });
1976
+
1977
+ logCost('/gstack-upgrade happy path', result);
1978
+
1979
+ // Check that the version was updated
1980
+ const versionAfter = fs.readFileSync(path.join(mockGstack, 'VERSION'), 'utf-8').trim();
1981
+ const output = result.output || '';
1982
+ const mentionsUpgrade = output.toLowerCase().includes('0.6.0') ||
1983
+ output.toLowerCase().includes('upgrade') ||
1984
+ output.toLowerCase().includes('updated');
1985
+
1986
+ recordE2E('/gstack-upgrade happy path', 'gstack-upgrade E2E', result, {
1987
+ passed: versionAfter === '0.6.0' && ['success', 'error_max_turns'].includes(result.exitReason),
1988
+ });
1989
+
1990
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
1991
+ expect(versionAfter).toBe('0.6.0');
1992
+ }, 240_000);
1993
+ });
1994
+
1995
+ // --- Design Consultation E2E ---
1996
+
1997
+ /**
1998
+ * LLM judge for DESIGN.md quality — checks font blacklist compliance,
1999
+ * coherence, specificity, and AI slop avoidance.
2000
+ */
2001
+ async function designQualityJudge(designMd: string): Promise<{ passed: boolean; reasoning: string }> {
2002
+ return callJudge<{ passed: boolean; reasoning: string }>(`You are evaluating a generated DESIGN.md file for quality.
2003
+
2004
+ Evaluate against these criteria — ALL must pass for an overall "passed: true":
2005
+ 1. Does NOT recommend Inter, Roboto, Arial, Helvetica, Open Sans, Lato, Montserrat, or Poppins as primary fonts
2006
+ 2. Aesthetic direction is coherent with color approach (e.g., brutalist aesthetic doesn't pair with expressive color without explanation)
2007
+ 3. Font recommendations include specific font names (not generic like "a sans-serif font")
2008
+ 4. Color palette includes actual hex values, not placeholders like "[hex]"
2009
+ 5. Rationale is provided for major decisions (not just "because it looks good")
2010
+ 6. No AI slop patterns: purple gradients mentioned positively, "3-column feature grid" language, generic marketing speak
2011
+ 7. Product context is reflected in design choices (civic tech → should have appropriate, professional aesthetic)
2012
+
2013
+ DESIGN.md content:
2014
+ \`\`\`
2015
+ ${designMd}
2016
+ \`\`\`
2017
+
2018
+ Return JSON: { "passed": true/false, "reasoning": "one paragraph explaining your evaluation" }`);
2019
+ }
2020
+
2021
+ describeIfSelected('Design Consultation E2E', [
2022
+ 'design-consultation-core', 'design-consultation-research',
2023
+ 'design-consultation-existing', 'design-consultation-preview',
2024
+ ], () => {
2025
+ let designDir: string;
2026
+
2027
+ beforeAll(() => {
2028
+ designDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-design-consultation-'));
2029
+ const { spawnSync } = require('child_process');
2030
+ const run = (cmd: string, args: string[]) =>
2031
+ spawnSync(cmd, args, { cwd: designDir, stdio: 'pipe', timeout: 5000 });
2032
+
2033
+ run('git', ['init', '-b', 'main']);
2034
+ run('git', ['config', 'user.email', 'test@test.com']);
2035
+ run('git', ['config', 'user.name', 'Test']);
2036
+
2037
+ // Create a realistic project context
2038
+ fs.writeFileSync(path.join(designDir, 'README.md'), `# CivicPulse
2039
+
2040
+ A civic tech data platform for government employees to access, visualize, and share public data. Built with Next.js and PostgreSQL.
2041
+
2042
+ ## Features
2043
+ - Real-time data dashboards for municipal budgets
2044
+ - Public records search with faceted filtering
2045
+ - Data export and sharing tools for inter-department collaboration
2046
+ `);
2047
+ fs.writeFileSync(path.join(designDir, 'package.json'), JSON.stringify({
2048
+ name: 'civicpulse',
2049
+ version: '0.1.0',
2050
+ dependencies: { next: '^14.0.0', react: '^18.2.0', 'tailwindcss': '^3.4.0' },
2051
+ }, null, 2));
2052
+
2053
+ run('git', ['add', '.']);
2054
+ run('git', ['commit', '-m', 'initial project setup']);
2055
+
2056
+ // Copy design-consultation skill
2057
+ fs.mkdirSync(path.join(designDir, 'design-consultation'), { recursive: true });
2058
+ fs.copyFileSync(
2059
+ path.join(ROOT, 'design-consultation', 'SKILL.md'),
2060
+ path.join(designDir, 'design-consultation', 'SKILL.md'),
2061
+ );
2062
+ });
2063
+
2064
+ afterAll(() => {
2065
+ try { fs.rmSync(designDir, { recursive: true, force: true }); } catch {}
2066
+ });
2067
+
2068
+ testIfSelected('design-consultation-core', async () => {
2069
+ const result = await runSkillTest({
2070
+ prompt: `Read design-consultation/SKILL.md for the design consultation workflow.
2071
+
2072
+ This is a civic tech data platform called CivicPulse for government employees who need to access public data. Read the README.md for details.
2073
+
2074
+ Skip research — work from your design knowledge. Skip the font preview page. Skip any AskUserQuestion calls — this is non-interactive. Accept your first design system proposal.
2075
+
2076
+ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
2077
+ workingDirectory: designDir,
2078
+ maxTurns: 20,
2079
+ timeout: 360_000,
2080
+ testName: 'design-consultation-core',
2081
+ runId,
2082
+ });
2083
+
2084
+ logCost('/design-consultation core', result);
2085
+
2086
+ const designPath = path.join(designDir, 'DESIGN.md');
2087
+ const claudePath = path.join(designDir, 'CLAUDE.md');
2088
+ const designExists = fs.existsSync(designPath);
2089
+ const claudeExists = fs.existsSync(claudePath);
2090
+ let designContent = '';
2091
+
2092
+ if (designExists) {
2093
+ designContent = fs.readFileSync(designPath, 'utf-8');
2094
+ }
2095
+
2096
+ // Structural checks
2097
+ const requiredSections = ['Product Context', 'Aesthetic', 'Typography', 'Color', 'Spacing', 'Layout', 'Motion'];
2098
+ const missingSections = requiredSections.filter(s => !designContent.toLowerCase().includes(s.toLowerCase()));
2099
+
2100
+ // LLM judge for quality
2101
+ let judgeResult = { passed: false, reasoning: 'judge not run' };
2102
+ if (designExists && designContent.length > 100) {
2103
+ try {
2104
+ judgeResult = await designQualityJudge(designContent);
2105
+ console.log('Design quality judge:', JSON.stringify(judgeResult, null, 2));
2106
+ } catch (err) {
2107
+ console.warn('Judge failed:', err);
2108
+ judgeResult = { passed: true, reasoning: 'judge error — defaulting to pass' };
2109
+ }
2110
+ }
2111
+
2112
+ const structuralPass = designExists && claudeExists && missingSections.length === 0;
2113
+ recordE2E('/design-consultation core', 'Design Consultation E2E', result, {
2114
+ passed: structuralPass && judgeResult.passed && ['success', 'error_max_turns'].includes(result.exitReason),
2115
+ });
2116
+
2117
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
2118
+ expect(designExists).toBe(true);
2119
+ if (designExists) {
2120
+ expect(missingSections).toHaveLength(0);
2121
+ }
2122
+ if (claudeExists) {
2123
+ const claude = fs.readFileSync(claudePath, 'utf-8');
2124
+ expect(claude.toLowerCase()).toContain('design.md');
2125
+ }
2126
+ }, 420_000);
2127
+
2128
+ testIfSelected('design-consultation-research', async () => {
2129
+ // Clean up from previous test
2130
+ try { fs.unlinkSync(path.join(designDir, 'DESIGN.md')); } catch {}
2131
+ try { fs.unlinkSync(path.join(designDir, 'CLAUDE.md')); } catch {}
2132
+
2133
+ const result = await runSkillTest({
2134
+ prompt: `Read design-consultation/SKILL.md for the design consultation workflow.
2135
+
2136
+ This is a civic tech data platform called CivicPulse. Read the README.md.
2137
+
2138
+ DO research what's out there before proposing — search for civic tech and government data platform designs. Skip the font preview page. Skip any AskUserQuestion calls — this is non-interactive.
2139
+
2140
+ Write DESIGN.md to the working directory.`,
2141
+ workingDirectory: designDir,
2142
+ maxTurns: 30,
2143
+ timeout: 360_000,
2144
+ testName: 'design-consultation-research',
2145
+ runId,
2146
+ });
2147
+
2148
+ logCost('/design-consultation research', result);
2149
+
2150
+ const designPath = path.join(designDir, 'DESIGN.md');
2151
+ const designExists = fs.existsSync(designPath);
2152
+ let designContent = '';
2153
+ if (designExists) {
2154
+ designContent = fs.readFileSync(designPath, 'utf-8');
2155
+ }
2156
+
2157
+ // Check if WebSearch was used (may not be available in all envs)
2158
+ const webSearchCalls = result.toolCalls.filter(tc => tc.tool === 'WebSearch');
2159
+ if (webSearchCalls.length > 0) {
2160
+ console.log(`WebSearch used ${webSearchCalls.length} times`);
2161
+ } else {
2162
+ console.warn('WebSearch not used — may be unavailable in test env');
2163
+ }
2164
+
2165
+ // LLM judge
2166
+ let judgeResult = { passed: false, reasoning: 'judge not run' };
2167
+ if (designExists && designContent.length > 100) {
2168
+ try {
2169
+ judgeResult = await designQualityJudge(designContent);
2170
+ console.log('Design quality judge (research):', JSON.stringify(judgeResult, null, 2));
2171
+ } catch (err) {
2172
+ console.warn('Judge failed:', err);
2173
+ judgeResult = { passed: true, reasoning: 'judge error — defaulting to pass' };
2174
+ }
2175
+ }
2176
+
2177
+ recordE2E('/design-consultation research', 'Design Consultation E2E', result, {
2178
+ passed: designExists && ['success', 'error_max_turns'].includes(result.exitReason),
2179
+ });
2180
+
2181
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
2182
+ expect(designExists).toBe(true);
2183
+ }, 420_000);
2184
+
2185
+ testIfSelected('design-consultation-existing', async () => {
2186
+ // Pre-create a minimal DESIGN.md
2187
+ fs.writeFileSync(path.join(designDir, 'DESIGN.md'), `# Design System — CivicPulse
2188
+
2189
+ ## Typography
2190
+ Body: system-ui
2191
+ `);
2192
+
2193
+ const result = await runSkillTest({
2194
+ prompt: `Read design-consultation/SKILL.md for the design consultation workflow.
2195
+
2196
+ There is already a DESIGN.md in this repo. Update it with a complete design system for CivicPulse, a civic tech data platform for government employees.
2197
+
2198
+ Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non-interactive.`,
2199
+ workingDirectory: designDir,
2200
+ maxTurns: 20,
2201
+ timeout: 360_000,
2202
+ testName: 'design-consultation-existing',
2203
+ runId,
2204
+ });
2205
+
2206
+ logCost('/design-consultation existing', result);
2207
+
2208
+ const designPath = path.join(designDir, 'DESIGN.md');
2209
+ const designExists = fs.existsSync(designPath);
2210
+ let designContent = '';
2211
+ if (designExists) {
2212
+ designContent = fs.readFileSync(designPath, 'utf-8');
2213
+ }
2214
+
2215
+ // Should have more content than the minimal version
2216
+ const hasColor = designContent.toLowerCase().includes('color');
2217
+ const hasSpacing = designContent.toLowerCase().includes('spacing');
2218
+
2219
+ recordE2E('/design-consultation existing', 'Design Consultation E2E', result, {
2220
+ passed: designExists && hasColor && hasSpacing && ['success', 'error_max_turns'].includes(result.exitReason),
2221
+ });
2222
+
2223
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
2224
+ expect(designExists).toBe(true);
2225
+ if (designExists) {
2226
+ expect(hasColor).toBe(true);
2227
+ expect(hasSpacing).toBe(true);
2228
+ }
2229
+ }, 420_000);
2230
+
2231
+ testIfSelected('design-consultation-preview', async () => {
2232
+ // Clean up
2233
+ try { fs.unlinkSync(path.join(designDir, 'DESIGN.md')); } catch {}
2234
+
2235
+ const result = await runSkillTest({
2236
+ prompt: `Read design-consultation/SKILL.md for the design consultation workflow.
2237
+
2238
+ This is CivicPulse, a civic tech data platform. Read the README.md.
2239
+
2240
+ Skip research. Skip any AskUserQuestion calls — this is non-interactive. Generate the font and color preview page but write it to ./design-preview.html instead of /tmp/ (do NOT run the open command). Then write DESIGN.md.`,
2241
+ workingDirectory: designDir,
2242
+ maxTurns: 20,
2243
+ timeout: 360_000,
2244
+ testName: 'design-consultation-preview',
2245
+ runId,
2246
+ });
2247
+
2248
+ logCost('/design-consultation preview', result);
2249
+
2250
+ const previewPath = path.join(designDir, 'design-preview.html');
2251
+ const designPath = path.join(designDir, 'DESIGN.md');
2252
+ const previewExists = fs.existsSync(previewPath);
2253
+ const designExists = fs.existsSync(designPath);
2254
+
2255
+ let previewContent = '';
2256
+ if (previewExists) {
2257
+ previewContent = fs.readFileSync(previewPath, 'utf-8');
2258
+ }
2259
+
2260
+ const hasHtml = previewContent.includes('<html') || previewContent.includes('<!DOCTYPE');
2261
+ const hasFontRef = previewContent.includes('font-family') || previewContent.includes('fonts.googleapis') || previewContent.includes('fonts.bunny');
2262
+ const hasColorRef = previewContent.includes('#') && (previewContent.includes('background') || previewContent.includes('color:'));
2263
+
2264
+ // LLM judge on the DESIGN.md
2265
+ let judgeResult = { passed: false, reasoning: 'judge not run' };
2266
+ if (designExists) {
2267
+ const designContent = fs.readFileSync(designPath, 'utf-8');
2268
+ if (designContent.length > 100) {
2269
+ try {
2270
+ judgeResult = await designQualityJudge(designContent);
2271
+ console.log('Design quality judge (preview):', JSON.stringify(judgeResult, null, 2));
2272
+ } catch (err) {
2273
+ console.warn('Judge failed:', err);
2274
+ judgeResult = { passed: true, reasoning: 'judge error — defaulting to pass' };
2275
+ }
2276
+ }
2277
+ }
2278
+
2279
+ recordE2E('/design-consultation preview', 'Design Consultation E2E', result, {
2280
+ passed: previewExists && designExists && hasHtml && ['success', 'error_max_turns'].includes(result.exitReason),
2281
+ });
2282
+
2283
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
2284
+ expect(previewExists).toBe(true);
2285
+ if (previewExists) {
2286
+ expect(hasHtml).toBe(true);
2287
+ expect(hasFontRef).toBe(true);
2288
+ }
2289
+ expect(designExists).toBe(true);
2290
+ }, 420_000);
2291
+ });
2292
+
2293
+ // --- Plan Design Review E2E (plan-mode) ---
2294
+
2295
+ describeIfSelected('Plan Design Review E2E', ['plan-design-review-plan-mode', 'plan-design-review-no-ui-scope'], () => {
2296
+ let reviewDir: string;
2297
+
2298
+ beforeAll(() => {
2299
+ reviewDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-design-'));
2300
+
2301
+ const { spawnSync } = require('child_process');
2302
+ const run = (cmd: string, args: string[]) =>
2303
+ spawnSync(cmd, args, { cwd: reviewDir, stdio: 'pipe', timeout: 5000 });
2304
+
2305
+ run('git', ['init', '-b', 'main']);
2306
+ run('git', ['config', 'user.email', 'test@test.com']);
2307
+ run('git', ['config', 'user.name', 'Test']);
2308
+
2309
+ // Copy plan-design-review skill
2310
+ fs.mkdirSync(path.join(reviewDir, 'plan-design-review'), { recursive: true });
2311
+ fs.copyFileSync(
2312
+ path.join(ROOT, 'plan-design-review', 'SKILL.md'),
2313
+ path.join(reviewDir, 'plan-design-review', 'SKILL.md'),
2314
+ );
2315
+
2316
+ // Create a plan file with intentional design gaps
2317
+ fs.writeFileSync(path.join(reviewDir, 'plan.md'), `# Plan: User Dashboard
2318
+
2319
+ ## Context
2320
+ Build a user dashboard that shows account stats, recent activity, and settings.
2321
+
2322
+ ## Implementation
2323
+ 1. Create a dashboard page at /dashboard
2324
+ 2. Show user stats (posts, followers, engagement rate)
2325
+ 3. Add a recent activity feed
2326
+ 4. Add a settings panel
2327
+ 5. Use a clean, modern UI with cards and icons
2328
+ 6. Add a hero section at the top with a gradient background
2329
+
2330
+ ## Technical Details
2331
+ - React components with Tailwind CSS
2332
+ - API endpoint: GET /api/dashboard
2333
+ - WebSocket for real-time activity updates
2334
+ `);
2335
+
2336
+ run('git', ['add', '.']);
2337
+ run('git', ['commit', '-m', 'initial plan']);
2338
+ });
2339
+
2340
+ afterAll(() => {
2341
+ try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
2342
+ });
2343
+
2344
+ testIfSelected('plan-design-review-plan-mode', async () => {
2345
+ const result = await runSkillTest({
2346
+ prompt: `Read plan-design-review/SKILL.md for the design review workflow.
2347
+
2348
+ Review the plan in ./plan.md. This plan has several design gaps — it uses vague language like "clean, modern UI" and "cards and icons", mentions a "hero section with gradient" (AI slop), and doesn't specify empty states, error states, loading states, responsive behavior, or accessibility.
2349
+
2350
+ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-interactive. Rate each design dimension 0-10 and explain what would make it a 10. Then EDIT plan.md to add the missing design decisions (interaction state table, empty states, responsive behavior, etc.).
2351
+
2352
+ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit. Just read the plan file, review it, and edit it to fix the gaps.`,
2353
+ workingDirectory: reviewDir,
2354
+ maxTurns: 15,
2355
+ timeout: 300_000,
2356
+ testName: 'plan-design-review-plan-mode',
2357
+ runId,
2358
+ });
2359
+
2360
+ logCost('/plan-design-review plan-mode', result);
2361
+
2362
+ // Check that the agent produced design ratings (0-10 scale)
2363
+ const output = result.output || '';
2364
+ const hasRatings = /\d+\/10/.test(output);
2365
+ const hasDesignContent = output.toLowerCase().includes('information architecture') ||
2366
+ output.toLowerCase().includes('interaction state') ||
2367
+ output.toLowerCase().includes('ai slop') ||
2368
+ output.toLowerCase().includes('hierarchy');
2369
+
2370
+ // Check that the plan file was edited (the core new behavior)
2371
+ const planAfter = fs.readFileSync(path.join(reviewDir, 'plan.md'), 'utf-8');
2372
+ const planOriginal = `# Plan: User Dashboard`;
2373
+ const planWasEdited = planAfter.length > 300; // Original is ~450 chars, edited should be much longer
2374
+ const planHasDesignAdditions = planAfter.toLowerCase().includes('empty') ||
2375
+ planAfter.toLowerCase().includes('loading') ||
2376
+ planAfter.toLowerCase().includes('error') ||
2377
+ planAfter.toLowerCase().includes('state') ||
2378
+ planAfter.toLowerCase().includes('responsive') ||
2379
+ planAfter.toLowerCase().includes('accessibility');
2380
+
2381
+ recordE2E('/plan-design-review plan-mode', 'Plan Design Review E2E', result, {
2382
+ passed: hasDesignContent && planWasEdited && ['success', 'error_max_turns'].includes(result.exitReason),
2383
+ });
2384
+
2385
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
2386
+ // Agent should produce design-relevant output about the plan
2387
+ expect(hasDesignContent).toBe(true);
2388
+ // Agent should have edited the plan file to add missing design decisions
2389
+ expect(planWasEdited).toBe(true);
2390
+ expect(planHasDesignAdditions).toBe(true);
2391
+ }, 360_000);
2392
+
2393
+ testIfSelected('plan-design-review-no-ui-scope', async () => {
2394
+ // Write a backend-only plan
2395
+ fs.writeFileSync(path.join(reviewDir, 'backend-plan.md'), `# Plan: Database Migration
2396
+
2397
+ ## Context
2398
+ Migrate user records from PostgreSQL to a new schema with better indexing.
2399
+
2400
+ ## Implementation
2401
+ 1. Create migration to add new columns to users table
2402
+ 2. Backfill data from legacy columns
2403
+ 3. Add database indexes for common query patterns
2404
+ 4. Update ActiveRecord models
2405
+ 5. Run migration in staging first, then production
2406
+ `);
2407
+
2408
+ const result = await runSkillTest({
2409
+ prompt: `Read plan-design-review/SKILL.md for the design review workflow.
2410
+
2411
+ Review the plan in ./backend-plan.md. This is a pure backend database migration plan with no UI changes.
2412
+
2413
+ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-interactive. Write your findings directly to stdout.
2414
+
2415
+ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit.`,
2416
+ workingDirectory: reviewDir,
2417
+ maxTurns: 10,
2418
+ timeout: 180_000,
2419
+ testName: 'plan-design-review-no-ui-scope',
2420
+ runId,
2421
+ });
2422
+
2423
+ logCost('/plan-design-review no-ui-scope', result);
2424
+
2425
+ // Agent should detect no UI scope and exit early
2426
+ const output = result.output || '';
2427
+ const detectsNoUI = output.toLowerCase().includes('no ui') ||
2428
+ output.toLowerCase().includes('no frontend') ||
2429
+ output.toLowerCase().includes('no design') ||
2430
+ output.toLowerCase().includes('not applicable') ||
2431
+ output.toLowerCase().includes('backend');
2432
+
2433
+ recordE2E('/plan-design-review no-ui-scope', 'Plan Design Review E2E', result, {
2434
+ passed: detectsNoUI && ['success', 'error_max_turns'].includes(result.exitReason),
2435
+ });
2436
+
2437
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
2438
+ expect(detectsNoUI).toBe(true);
2439
+ }, 240_000);
2440
+ });
2441
+
2442
+ // --- Design Review E2E (live-site audit + fix) ---
2443
+
2444
+ describeIfSelected('Design Review E2E', ['design-review-fix'], () => {
2445
+ let qaDesignDir: string;
2446
+ let qaDesignServer: ReturnType<typeof Bun.serve> | null = null;
2447
+
2448
+ beforeAll(() => {
2449
+ qaDesignDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-qa-design-'));
2450
+ setupBrowseShims(qaDesignDir);
2451
+
2452
+ const { spawnSync } = require('child_process');
2453
+ const run = (cmd: string, args: string[]) =>
2454
+ spawnSync(cmd, args, { cwd: qaDesignDir, stdio: 'pipe', timeout: 5000 });
2455
+
2456
+ run('git', ['init', '-b', 'main']);
2457
+ run('git', ['config', 'user.email', 'test@test.com']);
2458
+ run('git', ['config', 'user.name', 'Test']);
2459
+
2460
+ // Create HTML/CSS with intentional design issues
2461
+ fs.writeFileSync(path.join(qaDesignDir, 'index.html'), `<!DOCTYPE html>
2462
+ <html lang="en">
2463
+ <head>
2464
+ <meta charset="utf-8">
2465
+ <meta name="viewport" content="width=device-width, initial-scale=1">
2466
+ <title>Design Test App</title>
2467
+ <link rel="stylesheet" href="style.css">
2468
+ </head>
2469
+ <body>
2470
+ <header>
2471
+ <h1 style="font-size: 48px; color: #333;">Welcome</h1>
2472
+ <h2 style="font-size: 47px; color: #334;">Subtitle Here</h2>
2473
+ </header>
2474
+ <main>
2475
+ <div class="card" style="padding: 10px; margin: 20px;">
2476
+ <h3 style="color: blue;">Card Title</h3>
2477
+ <p style="color: #666; font-size: 14px; line-height: 1.2;">Some content here with tight line height.</p>
2478
+ </div>
2479
+ <div class="card" style="padding: 30px; margin: 5px;">
2480
+ <h3 style="color: green;">Another Card</h3>
2481
+ <p style="color: #999; font-size: 16px;">Different spacing and colors for no reason.</p>
2482
+ </div>
2483
+ <button style="background: red; color: white; padding: 5px 10px; border: none;">Click Me</button>
2484
+ <button style="background: #007bff; color: white; padding: 12px 24px; border: none; border-radius: 20px;">Also Click</button>
2485
+ </main>
2486
+ </body>
2487
+ </html>`);
2488
+
2489
+ fs.writeFileSync(path.join(qaDesignDir, 'style.css'), `body {
2490
+ font-family: Arial, sans-serif;
2491
+ margin: 0;
2492
+ padding: 20px;
2493
+ }
2494
+ .card {
2495
+ border: 1px solid #ddd;
2496
+ border-radius: 4px;
2497
+ }
2498
+ `);
2499
+
2500
+ run('git', ['add', '.']);
2501
+ run('git', ['commit', '-m', 'initial design test page']);
2502
+
2503
+ // Start a simple file server for the design test page
2504
+ qaDesignServer = Bun.serve({
2505
+ port: 0,
2506
+ fetch(req) {
2507
+ const url = new URL(req.url);
2508
+ const filePath = path.join(qaDesignDir, url.pathname === '/' ? 'index.html' : url.pathname.slice(1));
2509
+ try {
2510
+ const content = fs.readFileSync(filePath);
2511
+ const ext = path.extname(filePath);
2512
+ const contentType = ext === '.css' ? 'text/css' : ext === '.html' ? 'text/html' : 'text/plain';
2513
+ return new Response(content, { headers: { 'Content-Type': contentType } });
2514
+ } catch {
2515
+ return new Response('Not Found', { status: 404 });
2516
+ }
2517
+ },
2518
+ });
2519
+
2520
+ // Copy design-review skill
2521
+ fs.mkdirSync(path.join(qaDesignDir, 'design-review'), { recursive: true });
2522
+ fs.copyFileSync(
2523
+ path.join(ROOT, 'design-review', 'SKILL.md'),
2524
+ path.join(qaDesignDir, 'design-review', 'SKILL.md'),
2525
+ );
2526
+ });
2527
+
2528
+ afterAll(() => {
2529
+ qaDesignServer?.stop();
2530
+ try { fs.rmSync(qaDesignDir, { recursive: true, force: true }); } catch {}
2531
+ });
2532
+
2533
+ test('Test 7: /design-review audits and fixes design issues', async () => {
2534
+ const serverUrl = `http://localhost:${(qaDesignServer as any)?.port}`;
2535
+
2536
+ const result = await runSkillTest({
2537
+ prompt: `IMPORTANT: The browse binary is already assigned below as B. Do NOT search for it or run the SKILL.md setup block — just use $B directly.
2538
+
2539
+ B="${browseBin}"
2540
+
2541
+ Read design-review/SKILL.md for the design review + fix workflow.
2542
+
2543
+ Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion calls — this is non-interactive. Fix up to 3 issues max. Write your report to ./design-audit.md.`,
2544
+ workingDirectory: qaDesignDir,
2545
+ maxTurns: 30,
2546
+ timeout: 360_000,
2547
+ testName: 'design-review-fix',
2548
+ runId,
2549
+ });
2550
+
2551
+ logCost('/design-review fix', result);
2552
+
2553
+ const reportPath = path.join(qaDesignDir, 'design-audit.md');
2554
+ const reportExists = fs.existsSync(reportPath);
2555
+
2556
+ // Check if any design fix commits were made
2557
+ const gitLog = spawnSync('git', ['log', '--oneline'], {
2558
+ cwd: qaDesignDir, stdio: 'pipe',
2559
+ });
2560
+ const commits = gitLog.stdout.toString().trim().split('\n');
2561
+ const designFixCommits = commits.filter((c: string) => c.includes('style(design)'));
2562
+
2563
+ recordE2E('/design-review fix', 'Design Review E2E', result, {
2564
+ passed: ['success', 'error_max_turns'].includes(result.exitReason),
2565
+ });
2566
+
2567
+ // Accept error_max_turns — the fix loop is complex
2568
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
2569
+
2570
+ // Report and commits are best-effort — log what happened
2571
+ if (reportExists) {
2572
+ const report = fs.readFileSync(reportPath, 'utf-8');
2573
+ console.log(`Design audit report: ${report.length} chars`);
2574
+ } else {
2575
+ console.warn('No design-audit.md generated');
2576
+ }
2577
+ console.log(`Design fix commits: ${designFixCommits.length}`);
2578
+ }, 420_000);
2579
+ });
2580
+
2581
+ // --- Test Bootstrap E2E ---
2582
+
2583
+ describeIfSelected('Test Bootstrap E2E', ['qa-bootstrap'], () => {
2584
+ let bootstrapDir: string;
2585
+ let bootstrapServer: ReturnType<typeof Bun.serve>;
2586
+
2587
+ beforeAll(() => {
2588
+ bootstrapDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-bootstrap-'));
2589
+ setupBrowseShims(bootstrapDir);
2590
+
2591
+ // Copy qa skill files
2592
+ copyDirSync(path.join(ROOT, 'qa'), path.join(bootstrapDir, 'qa'));
2593
+
2594
+ // Create a minimal Node.js project with NO test framework
2595
+ fs.writeFileSync(path.join(bootstrapDir, 'package.json'), JSON.stringify({
2596
+ name: 'test-bootstrap-app',
2597
+ version: '1.0.0',
2598
+ type: 'module',
2599
+ }, null, 2));
2600
+
2601
+ // Create a simple app file with a bug
2602
+ fs.writeFileSync(path.join(bootstrapDir, 'app.js'), `
2603
+ export function add(a, b) { return a + b; }
2604
+ export function subtract(a, b) { return a - b; }
2605
+ export function divide(a, b) { return a / b; } // BUG: no zero check
2606
+ `);
2607
+
2608
+ // Create a simple HTML page with a bug
2609
+ fs.writeFileSync(path.join(bootstrapDir, 'index.html'), `<!DOCTYPE html>
2610
+ <html lang="en">
2611
+ <head><meta charset="utf-8"><title>Bootstrap Test</title></head>
2612
+ <body>
2613
+ <h1>Test App</h1>
2614
+ <a href="/nonexistent-page">Broken Link</a>
2615
+ <script>console.error("ReferenceError: undefinedVar is not defined");</script>
2616
+ </body>
2617
+ </html>
2618
+ `);
2619
+
2620
+ // Init git repo
2621
+ const run = (cmd: string, args: string[]) =>
2622
+ spawnSync(cmd, args, { cwd: bootstrapDir, stdio: 'pipe', timeout: 5000 });
2623
+ run('git', ['init', '-b', 'main']);
2624
+ run('git', ['config', 'user.email', 'test@test.com']);
2625
+ run('git', ['config', 'user.name', 'Test']);
2626
+ run('git', ['add', '.']);
2627
+ run('git', ['commit', '-m', 'initial commit']);
2628
+
2629
+ // Serve from working directory
2630
+ bootstrapServer = Bun.serve({
2631
+ port: 0,
2632
+ hostname: '127.0.0.1',
2633
+ fetch(req) {
2634
+ const url = new URL(req.url);
2635
+ let filePath = url.pathname === '/' ? '/index.html' : url.pathname;
2636
+ filePath = filePath.replace(/^\//, '');
2637
+ const fullPath = path.join(bootstrapDir, filePath);
2638
+ if (!fs.existsSync(fullPath)) {
2639
+ return new Response('Not Found', { status: 404 });
2640
+ }
2641
+ const content = fs.readFileSync(fullPath, 'utf-8');
2642
+ return new Response(content, {
2643
+ headers: { 'Content-Type': 'text/html' },
2644
+ });
2645
+ },
2646
+ });
2647
+ });
2648
+
2649
+ afterAll(() => {
2650
+ bootstrapServer?.stop();
2651
+ try { fs.rmSync(bootstrapDir, { recursive: true, force: true }); } catch {}
2652
+ });
2653
+
2654
+ test('/qa bootstrap + regression test on zero-test project', async () => {
2655
+ const serverUrl = `http://127.0.0.1:${bootstrapServer!.port}`;
2656
+
2657
+ const result = await runSkillTest({
2658
+ prompt: `You have a browse binary at ${browseBin}. Assign it to B variable like: B="${browseBin}"
2659
+
2660
+ Read the file qa/SKILL.md for the QA workflow instructions.
2661
+
2662
+ Run a Quick-tier QA test on ${serverUrl}
2663
+ The source code for this page is at ${bootstrapDir}/index.html — you can fix bugs there.
2664
+ Do NOT use AskUserQuestion — for any AskUserQuestion prompts, choose the RECOMMENDED option automatically.
2665
+ Write your report to ${bootstrapDir}/qa-reports/qa-report.md
2666
+
2667
+ This project has NO test framework. When the bootstrap asks, pick vitest (option A).
2668
+ This is a test+fix loop: find bugs, fix them, write regression tests, commit each fix.`,
2669
+ workingDirectory: bootstrapDir,
2670
+ maxTurns: 50,
2671
+ allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
2672
+ timeout: 420_000,
2673
+ testName: 'qa-bootstrap',
2674
+ runId,
2675
+ });
2676
+
2677
+ logCost('/qa bootstrap', result);
2678
+ recordE2E('/qa bootstrap + regression test', 'Test Bootstrap E2E', result, {
2679
+ passed: ['success', 'error_max_turns'].includes(result.exitReason),
2680
+ });
2681
+
2682
+ expect(['success', 'error_max_turns']).toContain(result.exitReason);
2683
+
2684
+ // Verify bootstrap created test infrastructure
2685
+ const hasTestConfig = fs.existsSync(path.join(bootstrapDir, 'vitest.config.ts'))
2686
+ || fs.existsSync(path.join(bootstrapDir, 'vitest.config.js'))
2687
+ || fs.existsSync(path.join(bootstrapDir, 'jest.config.js'))
2688
+ || fs.existsSync(path.join(bootstrapDir, 'jest.config.ts'));
2689
+ console.log(`Test config created: ${hasTestConfig}`);
2690
+
2691
+ const hasTestingMd = fs.existsSync(path.join(bootstrapDir, 'TESTING.md'));
2692
+ console.log(`TESTING.md created: ${hasTestingMd}`);
2693
+
2694
+ // Check for bootstrap commit
2695
+ const gitLog = spawnSync('git', ['log', '--oneline', '--grep=bootstrap'], {
2696
+ cwd: bootstrapDir, stdio: 'pipe',
2697
+ });
2698
+ const bootstrapCommits = gitLog.stdout.toString().trim();
2699
+ console.log(`Bootstrap commits: ${bootstrapCommits || 'none'}`);
2700
+
2701
+ // Check for regression test commits
2702
+ const regressionLog = spawnSync('git', ['log', '--oneline', '--grep=test(qa)'], {
2703
+ cwd: bootstrapDir, stdio: 'pipe',
2704
+ });
2705
+ const regressionCommits = regressionLog.stdout.toString().trim();
2706
+ console.log(`Regression test commits: ${regressionCommits || 'none'}`);
2707
+
2708
+ // Verify at least the bootstrap happened (fix commits are bonus)
2709
+ const allCommits = spawnSync('git', ['log', '--oneline'], {
2710
+ cwd: bootstrapDir, stdio: 'pipe',
2711
+ });
2712
+ const totalCommits = allCommits.stdout.toString().trim().split('\n').length;
2713
+ console.log(`Total commits: ${totalCommits}`);
2714
+ expect(totalCommits).toBeGreaterThan(1); // At least initial + bootstrap
2715
+ }, 420_000);
2716
+ });
2717
+
2718
+ // --- Test Coverage Audit E2E ---
2719
+
2720
+ describeIfSelected('Test Coverage Audit E2E', ['ship-coverage-audit'], () => {
2721
+ let coverageDir: string;
2722
+
2723
+ beforeAll(() => {
2724
+ coverageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-coverage-'));
2725
+
2726
+ // Copy ship skill files
2727
+ copyDirSync(path.join(ROOT, 'ship'), path.join(coverageDir, 'ship'));
2728
+ copyDirSync(path.join(ROOT, 'review'), path.join(coverageDir, 'review'));
2729
+
2730
+ // Use shared fixture for billing project with coverage gaps
2731
+ const { createCoverageAuditFixture } = require('./fixtures/coverage-audit-fixture');
2732
+ createCoverageAuditFixture(coverageDir);
2733
+ });
2734
+
2735
+ afterAll(() => {
2736
+ try { fs.rmSync(coverageDir, { recursive: true, force: true }); } catch {}
2737
+ });
2738
+
2739
+ test('/ship Step 3.4 produces coverage diagram', async () => {
2740
+ const result = await runSkillTest({
2741
+ prompt: `Read the file ship/SKILL.md for the ship workflow instructions.
2742
+
2743
+ You are on the feature/billing branch. The base branch is main.
2744
+ This is a test project — there is no remote, no PR to create.
2745
+
2746
+ ONLY run Step 3.4 (Test Coverage Audit) from the ship workflow.
2747
+ Skip all other steps (tests, evals, review, version, changelog, commit, push, PR).
2748
+
2749
+ The source code is in ${coverageDir}/src/billing.ts.
2750
+ Existing tests are in ${coverageDir}/test/billing.test.ts.
2751
+ The test command is: echo "tests pass" (mocked — just pretend tests pass).
2752
+
2753
+ Produce the ASCII coverage diagram showing which code paths are tested and which have gaps.
2754
+ Do NOT generate new tests — just produce the diagram and coverage summary.
2755
+ Output the diagram directly.`,
2756
+ workingDirectory: coverageDir,
2757
+ maxTurns: 15,
2758
+ allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
2759
+ timeout: 120_000,
2760
+ testName: 'ship-coverage-audit',
2761
+ runId,
2762
+ });
2763
+
2764
+ logCost('/ship coverage audit', result);
2765
+ recordE2E('/ship Step 3.4 coverage audit', 'Test Coverage Audit E2E', result, {
2766
+ passed: result.exitReason === 'success',
2767
+ });
2768
+
2769
+ expect(result.exitReason).toBe('success');
2770
+
2771
+ // Check output contains coverage diagram elements
2772
+ const output = result.output || '';
2773
+ const outputLower = output.toLowerCase();
2774
+ const hasGap = outputLower.includes('gap') || outputLower.includes('no test');
2775
+ const hasTested = outputLower.includes('tested') || output.includes('✓') || output.includes('★');
2776
+ const hasCoverage = outputLower.includes('coverage') || outputLower.includes('paths tested');
2777
+
2778
+ console.log(`Output has GAP markers: ${hasGap}`);
2779
+ console.log(`Output has TESTED markers: ${hasTested}`);
2780
+ console.log(`Output has coverage summary: ${hasCoverage}`);
2781
+
2782
+ // The agent MUST produce a coverage diagram with gap and tested markers
2783
+ expect(hasGap || hasTested).toBe(true);
2784
+
2785
+ // At minimum, the agent should have read the source and test files
2786
+ const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
2787
+ expect(readCalls.length).toBeGreaterThan(0);
2788
+ }, 180_000);
2789
+ });
2790
+
2791
+ // --- Review Coverage Audit E2E ---
2792
+
2793
+ describeIfSelected('Review Coverage Audit E2E', ['review-coverage-audit'], () => {
2794
+ let reviewCoverageDir: string;
2795
+
2796
+ beforeAll(() => {
2797
+ reviewCoverageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-review-coverage-'));
2798
+
2799
+ // Copy review skill files
2800
+ copyDirSync(path.join(ROOT, 'review'), path.join(reviewCoverageDir, 'review'));
2801
+
2802
+ // Use shared fixture for billing project with coverage gaps
2803
+ const { createCoverageAuditFixture } = require('./fixtures/coverage-audit-fixture');
2804
+ createCoverageAuditFixture(reviewCoverageDir);
2805
+ });
2806
+
2807
+ afterAll(() => {
2808
+ try { fs.rmSync(reviewCoverageDir, { recursive: true, force: true }); } catch {}
2809
+ });
2810
+
2811
+ test('/review Step 4.75 produces coverage diagram', async () => {
2812
+ const result = await runSkillTest({
2813
+ prompt: `Read the file review/SKILL.md for the review workflow instructions.
2814
+
2815
+ You are on the feature/billing branch. The base branch is main.
2816
+ This is a test project — there is no remote, no PR to create.
2817
+
2818
+ ONLY run Step 4.75 (Test Coverage Diagram) from the review workflow.
2819
+ Skip all other steps (scope drift, checklist, design review, fix-first, etc.).
2820
+
2821
+ The source code is in ${reviewCoverageDir}/src/billing.ts.
2822
+ Existing tests are in ${reviewCoverageDir}/test/billing.test.ts.
2823
+
2824
+ Produce the ASCII coverage diagram showing which code paths are tested and which have gaps.
2825
+ Output the diagram directly.`,
2826
+ workingDirectory: reviewCoverageDir,
2827
+ maxTurns: 15,
2828
+ allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
2829
+ timeout: 120_000,
2830
+ testName: 'review-coverage-audit',
2831
+ runId,
2832
+ });
2833
+
2834
+ logCost('/review coverage audit', result);
2835
+ recordE2E('/review Step 4.75 coverage audit', 'Review Coverage Audit E2E', result, {
2836
+ passed: result.exitReason === 'success',
2837
+ });
2838
+
2839
+ expect(result.exitReason).toBe('success');
2840
+
2841
+ // Check output contains coverage diagram elements
2842
+ const output = result.output || '';
2843
+ const outputLower = output.toLowerCase();
2844
+ const hasGap = outputLower.includes('gap') || outputLower.includes('no test');
2845
+ const hasTested = outputLower.includes('tested') || output.includes('✓') || output.includes('★');
2846
+ const hasCoverage = outputLower.includes('coverage') || outputLower.includes('paths tested');
2847
+
2848
+ console.log(`Output has GAP markers: ${hasGap}`);
2849
+ console.log(`Output has TESTED markers: ${hasTested}`);
2850
+ console.log(`Output has coverage summary: ${hasCoverage}`);
2851
+
2852
+ // The agent MUST produce a coverage diagram with gap and tested markers
2853
+ expect(hasGap || hasTested).toBe(true);
2854
+
2855
+ // At minimum, the agent should have read the source and test files
2856
+ const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
2857
+ expect(readCalls.length).toBeGreaterThan(0);
2858
+ }, 180_000);
2859
+ });
2860
+
2861
+ // --- Plan Eng Review Coverage Audit E2E ---
2862
+
2863
+ describeIfSelected('Plan Eng Review Coverage Audit E2E', ['plan-eng-coverage-audit'], () => {
2864
+ let planCoverageDir: string;
2865
+
2866
+ beforeAll(() => {
2867
+ planCoverageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-coverage-'));
2868
+
2869
+ // Copy plan-eng-review skill files
2870
+ copyDirSync(path.join(ROOT, 'plan-eng-review'), path.join(planCoverageDir, 'plan-eng-review'));
2871
+
2872
+ // Use shared fixture for billing project with coverage gaps
2873
+ const { createCoverageAuditFixture } = require('./fixtures/coverage-audit-fixture');
2874
+ createCoverageAuditFixture(planCoverageDir);
2875
+ });
2876
+
2877
+ afterAll(() => {
2878
+ try { fs.rmSync(planCoverageDir, { recursive: true, force: true }); } catch {}
2879
+ });
2880
+
2881
+ test('/plan-eng-review coverage audit traces plan codepaths', async () => {
2882
+ const result = await runSkillTest({
2883
+ prompt: `Read the file plan-eng-review/SKILL.md for the plan review workflow instructions.
2884
+
2885
+ You are on the feature/billing branch. The base branch is main.
2886
+ This is a test project — there is no remote, no PR to create.
2887
+
2888
+ ONLY run the Test Coverage Audit section from the plan review workflow.
2889
+ Skip all other steps (architecture, code quality, performance, etc.).
2890
+
2891
+ The source code is in ${planCoverageDir}/src/billing.ts.
2892
+ Existing tests are in ${planCoverageDir}/test/billing.test.ts.
2893
+
2894
+ Produce the ASCII coverage diagram showing which code paths are tested and which have gaps.
2895
+ Output the diagram directly.`,
2896
+ workingDirectory: planCoverageDir,
2897
+ maxTurns: 15,
2898
+ allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
2899
+ timeout: 120_000,
2900
+ testName: 'plan-eng-coverage-audit',
2901
+ runId,
2902
+ });
2903
+
2904
+ logCost('/plan-eng-review coverage audit', result);
2905
+ recordE2E('/plan-eng-review coverage audit', 'Plan Eng Review Coverage Audit E2E', result, {
2906
+ passed: result.exitReason === 'success',
2907
+ });
2908
+
2909
+ expect(result.exitReason).toBe('success');
2910
+
2911
+ // Check output contains coverage diagram elements
2912
+ const output = result.output || '';
2913
+ const outputLower = output.toLowerCase();
2914
+ const hasGap = outputLower.includes('gap') || outputLower.includes('no test');
2915
+ const hasTested = outputLower.includes('tested') || output.includes('✓') || output.includes('★');
2916
+ const hasCoverage = outputLower.includes('coverage') || outputLower.includes('paths tested');
2917
+
2918
+ console.log(`Output has GAP markers: ${hasGap}`);
2919
+ console.log(`Output has TESTED markers: ${hasTested}`);
2920
+ console.log(`Output has coverage summary: ${hasCoverage}`);
2921
+
2922
+ // The agent MUST produce a coverage diagram with gap and tested markers
2923
+ expect(hasGap || hasTested).toBe(true);
2924
+
2925
+ // At minimum, the agent should have read the source and test files
2926
+ const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
2927
+ expect(readCalls.length).toBeGreaterThan(0);
2928
+ }, 180_000);
2929
+ });
2930
+
2931
+ // --- Triage E2E ---
2932
+
2933
+ describeIfSelected('Test Failure Triage E2E', ['ship-triage'], () => {
2934
+ let triageDir: string;
2935
+
2936
+ beforeAll(() => {
2937
+ triageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-triage-'));
2938
+
2939
+ // Copy ship skill files
2940
+ copyDirSync(path.join(ROOT, 'ship'), path.join(triageDir, 'ship'));
2941
+
2942
+ const run = (cmd: string, args: string[]) =>
2943
+ spawnSync(cmd, args, { cwd: triageDir, stdio: 'pipe', timeout: 5000 });
2944
+
2945
+ // Init git repo
2946
+ run('git', ['init', '-b', 'main']);
2947
+ run('git', ['config', 'user.email', 'test@test.com']);
2948
+ run('git', ['config', 'user.name', 'Test']);
2949
+
2950
+ // Create a project with a pre-existing test failure on main
2951
+ fs.writeFileSync(path.join(triageDir, 'package.json'), JSON.stringify({
2952
+ name: 'triage-test-app',
2953
+ version: '1.0.0',
2954
+ scripts: { test: 'node test/run.js' },
2955
+ }, null, 2));
2956
+
2957
+ fs.mkdirSync(path.join(triageDir, 'src'), { recursive: true });
2958
+ fs.mkdirSync(path.join(triageDir, 'test'), { recursive: true });
2959
+
2960
+ // Source with a bug that exists on main (pre-existing)
2961
+ fs.writeFileSync(path.join(triageDir, 'src', 'math.js'), `
2962
+ module.exports = {
2963
+ add: (a, b) => a + b,
2964
+ divide: (a, b) => a / b, // BUG: no zero-division check (pre-existing)
2965
+ };
2966
+ `);
2967
+
2968
+ // Test file that catches the pre-existing bug
2969
+ fs.writeFileSync(path.join(triageDir, 'test', 'math.test.js'), `
2970
+ const { add, divide } = require('../src/math');
2971
+
2972
+ // This test passes
2973
+ if (add(2, 3) !== 5) { console.error('FAIL: add(2,3) should be 5'); process.exit(1); }
2974
+ console.log('PASS: add');
2975
+
2976
+ // This test FAILS — pre-existing bug (divide by zero returns Infinity, not an error)
2977
+ try {
2978
+ const result = divide(10, 0);
2979
+ if (result === Infinity) { console.error('FAIL: divide(10,0) should throw, got Infinity'); process.exit(1); }
2980
+ } catch(e) {
2981
+ console.log('PASS: divide zero check');
2982
+ }
2983
+ `);
2984
+
2985
+ // Test runner — each test in a subprocess so one failure doesn't kill the other
2986
+ fs.writeFileSync(path.join(triageDir, 'test', 'run.js'), `
2987
+ const { execSync } = require('child_process');
2988
+ const path = require('path');
2989
+ let failures = 0;
2990
+ for (const f of ['math.test.js', 'string.test.js']) {
2991
+ try {
2992
+ execSync('node ' + path.join(__dirname, f), { stdio: 'inherit' });
2993
+ } catch (e) {
2994
+ failures++;
2995
+ }
2996
+ }
2997
+ if (failures > 0) process.exit(1);
2998
+ `);
2999
+
3000
+ // Commit on main with the pre-existing bug
3001
+ run('git', ['add', '.']);
3002
+ run('git', ['commit', '-m', 'initial: math utils with tests']);
3003
+
3004
+ // Create feature branch
3005
+ run('git', ['checkout', '-b', 'feature/string-utils']);
3006
+
3007
+ // Add new code with a new bug (in-branch)
3008
+ fs.writeFileSync(path.join(triageDir, 'src', 'string.js'), `
3009
+ module.exports = {
3010
+ capitalize: (s) => s.charAt(0).toUpperCase() + s.slice(1),
3011
+ reverse: (s) => s.split('').reverse().join(''),
3012
+ truncate: (s, len) => s.substring(0, len), // BUG: no null check (in-branch)
3013
+ };
3014
+ `);
3015
+
3016
+ // Add test that catches the in-branch bug
3017
+ fs.writeFileSync(path.join(triageDir, 'test', 'string.test.js'), `
3018
+ const { capitalize, reverse, truncate } = require('../src/string');
3019
+
3020
+ if (capitalize('hello') !== 'Hello') { console.error('FAIL: capitalize'); process.exit(1); }
3021
+ console.log('PASS: capitalize');
3022
+
3023
+ if (reverse('abc') !== 'cba') { console.error('FAIL: reverse'); process.exit(1); }
3024
+ console.log('PASS: reverse');
3025
+
3026
+ // This test FAILS — in-branch bug (null input causes TypeError)
3027
+ try {
3028
+ truncate(null, 5);
3029
+ console.log('PASS: truncate null');
3030
+ } catch(e) {
3031
+ console.error('FAIL: truncate(null, 5) threw: ' + e.message);
3032
+ process.exit(1);
3033
+ }
3034
+ `);
3035
+
3036
+ run('git', ['add', '.']);
3037
+ run('git', ['commit', '-m', 'feat: add string utilities']);
3038
+ });
3039
+
3040
+ afterAll(() => {
3041
+ try { fs.rmSync(triageDir, { recursive: true, force: true }); } catch {}
3042
+ });
3043
+
3044
+ test('/ship triage correctly classifies in-branch vs pre-existing failures', async () => {
3045
+ const result = await runSkillTest({
3046
+ prompt: `Read the file ship/SKILL.md for the ship workflow instructions.
3047
+
3048
+ You are on the feature/string-utils branch. The base branch is main.
3049
+ This is a test project — there is no remote, no PR to create.
3050
+
3051
+ Run the tests first:
3052
+ \`\`\`bash
3053
+ cd ${triageDir} && node test/run.js
3054
+ \`\`\`
3055
+
3056
+ The tests will fail. Now run ONLY the Test Failure Ownership Triage (Steps T1-T4) from the ship workflow.
3057
+
3058
+ For each failing test, classify it as:
3059
+ - **In-branch**: caused by changes on this branch (feature/string-utils)
3060
+ - **Pre-existing**: existed before this branch (present on main)
3061
+
3062
+ Use git diff origin/main...HEAD (or git diff main...HEAD since there's no remote) to determine which files changed on this branch.
3063
+
3064
+ Output your classification for each failure clearly, labeling each as "IN-BRANCH" or "PRE-EXISTING" with your reasoning.
3065
+
3066
+ This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixing now.`,
3067
+ workingDirectory: triageDir,
3068
+ maxTurns: 20,
3069
+ allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
3070
+ timeout: 180_000,
3071
+ testName: 'ship-triage',
3072
+ runId,
3073
+ });
3074
+
3075
+ logCost('/ship triage', result);
3076
+
3077
+ const output = result.output || '';
3078
+ const outputLower = output.toLowerCase();
3079
+
3080
+ // The triage should identify the string/truncate failure as in-branch
3081
+ const hasInBranch = outputLower.includes('in-branch') || outputLower.includes('in branch') || outputLower.includes('introduced');
3082
+ // The triage should identify the math/divide failure as pre-existing
3083
+ const hasPreExisting = outputLower.includes('pre-existing') || outputLower.includes('pre existing') || outputLower.includes('existed before');
3084
+
3085
+ console.log(`Output identifies IN-BRANCH failures: ${hasInBranch}`);
3086
+ console.log(`Output identifies PRE-EXISTING failures: ${hasPreExisting}`);
3087
+
3088
+ // Check that the string/truncate bug is classified as in-branch
3089
+ const mentionsTruncate = outputLower.includes('truncate') || outputLower.includes('string');
3090
+ const mentionsDivide = outputLower.includes('divide') || outputLower.includes('math');
3091
+
3092
+ console.log(`Mentions truncate/string (in-branch bug): ${mentionsTruncate}`);
3093
+ console.log(`Mentions divide/math (pre-existing bug): ${mentionsDivide}`);
3094
+
3095
+ // Verify BOTH failure classes are exercised (not just detected):
3096
+ // The test runner must have actually run both test files
3097
+ const ranMathTest = output.includes('math.test') || output.includes('FAIL: divide');
3098
+ const ranStringTest = output.includes('string.test') || output.includes('FAIL: truncate');
3099
+ console.log(`Ran math test file (pre-existing failure): ${ranMathTest}`);
3100
+ console.log(`Ran string test file (in-branch failure): ${ranStringTest}`);
3101
+
3102
+ recordE2E('/ship triage', 'Test Failure Triage E2E', result, {
3103
+ passed: result.exitReason === 'success' && hasInBranch && hasPreExisting,
3104
+ has_in_branch_classification: hasInBranch,
3105
+ has_pre_existing_classification: hasPreExisting,
3106
+ mentions_truncate: mentionsTruncate,
3107
+ mentions_divide: mentionsDivide,
3108
+ ran_both_test_files: ranMathTest && ranStringTest,
3109
+ });
3110
+
3111
+ expect(result.exitReason).toBe('success');
3112
+ // Must classify at least one failure as in-branch AND one as pre-existing
3113
+ expect(hasInBranch).toBe(true);
3114
+ expect(hasPreExisting).toBe(true);
3115
+ // Must mention the specific bugs
3116
+ expect(mentionsTruncate).toBe(true);
3117
+ expect(mentionsDivide).toBe(true);
3118
+ // Must have actually run both test files (exercises both failure classes)
3119
+ expect(ranMathTest).toBe(true);
3120
+ expect(ranStringTest).toBe(true);
3121
+ }, 240_000);
3122
+ });
3123
+
3124
+ // --- Codex skill E2E ---
3125
+
3126
+ describeIfSelected('Codex skill E2E', ['codex-review'], () => {
3127
+ let codexDir: string;
3128
+
3129
+ beforeAll(() => {
3130
+ codexDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-codex-'));
3131
+
3132
+ const run = (cmd: string, args: string[]) =>
3133
+ spawnSync(cmd, args, { cwd: codexDir, stdio: 'pipe', timeout: 5000 });
3134
+
3135
+ run('git', ['init', '-b', 'main']);
3136
+ run('git', ['config', 'user.email', 'test@test.com']);
3137
+ run('git', ['config', 'user.name', 'Test']);
3138
+
3139
+ // Commit a clean base on main
3140
+ fs.writeFileSync(path.join(codexDir, 'app.rb'), '# clean base\nclass App\nend\n');
3141
+ run('git', ['add', 'app.rb']);
3142
+ run('git', ['commit', '-m', 'initial commit']);
3143
+
3144
+ // Create feature branch with vulnerable code (reuse review fixture)
3145
+ run('git', ['checkout', '-b', 'feature/add-vuln']);
3146
+ const vulnContent = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-vuln.rb'), 'utf-8');
3147
+ fs.writeFileSync(path.join(codexDir, 'user_controller.rb'), vulnContent);
3148
+ run('git', ['add', 'user_controller.rb']);
3149
+ run('git', ['commit', '-m', 'add vulnerable controller']);
3150
+
3151
+ // Copy the codex skill file
3152
+ fs.copyFileSync(path.join(ROOT, 'codex', 'SKILL.md'), path.join(codexDir, 'codex-SKILL.md'));
3153
+ });
3154
+
3155
+ afterAll(() => {
3156
+ try { fs.rmSync(codexDir, { recursive: true, force: true }); } catch {}
3157
+ });
3158
+
3159
+ test('/codex review produces findings and GATE verdict', async () => {
3160
+ // Check codex is available — skip if not installed
3161
+ const codexCheck = spawnSync('which', ['codex'], { stdio: 'pipe', timeout: 3000 });
3162
+ if (codexCheck.status !== 0) {
3163
+ console.warn('codex CLI not installed — skipping E2E test');
3164
+ return;
3165
+ }
3166
+
3167
+ const result = await runSkillTest({
3168
+ prompt: `You are in a git repo on branch feature/add-vuln with changes against main.
3169
+ Read codex-SKILL.md for the /codex skill instructions.
3170
+ Run /codex review to review the current diff against main.
3171
+ Write the full output (including the GATE verdict) to ${codexDir}/codex-output.md`,
3172
+ workingDirectory: codexDir,
3173
+ maxTurns: 10,
3174
+ timeout: 300_000,
3175
+ testName: 'codex-review',
3176
+ runId,
3177
+ });
3178
+
3179
+ logCost('/codex review', result);
3180
+ recordE2E('/codex review', 'Codex skill E2E', result);
3181
+ expect(result.exitReason).toBe('success');
3182
+
3183
+ // Check that output file was created with review content
3184
+ const outputPath = path.join(codexDir, 'codex-output.md');
3185
+ if (fs.existsSync(outputPath)) {
3186
+ const output = fs.readFileSync(outputPath, 'utf-8');
3187
+ // Should contain the CODEX SAYS header or GATE verdict
3188
+ const hasCodexOutput = output.includes('CODEX') || output.includes('GATE') || output.includes('codex');
3189
+ expect(hasCodexOutput).toBe(true);
3190
+ }
3191
+ }, 360_000);
3192
+ });
3193
+
3194
+ // --- Office Hours Spec Review E2E ---
3195
+
3196
+ describeIfSelected('Office Hours Spec Review E2E', ['office-hours-spec-review'], () => {
3197
+ let ohDir: string;
3198
+
3199
+ beforeAll(() => {
3200
+ ohDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-oh-spec-'));
3201
+ const run = (cmd: string, args: string[]) =>
3202
+ spawnSync(cmd, args, { cwd: ohDir, stdio: 'pipe', timeout: 5000 });
3203
+
3204
+ run('git', ['init', '-b', 'main']);
3205
+ run('git', ['config', 'user.email', 'test@test.com']);
3206
+ run('git', ['config', 'user.name', 'Test']);
3207
+ fs.writeFileSync(path.join(ohDir, 'README.md'), '# Test Project\n');
3208
+ run('git', ['add', '.']);
3209
+ run('git', ['commit', '-m', 'init']);
3210
+
3211
+ // Copy office-hours skill
3212
+ fs.mkdirSync(path.join(ohDir, 'office-hours'), { recursive: true });
3213
+ fs.copyFileSync(
3214
+ path.join(ROOT, 'office-hours', 'SKILL.md'),
3215
+ path.join(ohDir, 'office-hours', 'SKILL.md'),
3216
+ );
3217
+ });
3218
+
3219
+ afterAll(() => {
3220
+ try { fs.rmSync(ohDir, { recursive: true, force: true }); } catch {}
3221
+ });
3222
+
3223
+ test('/office-hours SKILL.md contains spec review loop', async () => {
3224
+ const result = await runSkillTest({
3225
+ prompt: `Read office-hours/SKILL.md. I want to understand the spec review loop.
3226
+
3227
+ Summarize what the "Spec Review Loop" section does — specifically:
3228
+ 1. How many dimensions does the reviewer check?
3229
+ 2. What tool is used to dispatch the reviewer?
3230
+ 3. What's the maximum number of iterations?
3231
+ 4. What metrics are tracked?
3232
+
3233
+ Write your summary to ${ohDir}/spec-review-summary.md`,
3234
+ workingDirectory: ohDir,
3235
+ maxTurns: 8,
3236
+ timeout: 120_000,
3237
+ testName: 'office-hours-spec-review',
3238
+ runId,
3239
+ });
3240
+
3241
+ logCost('/office-hours spec review', result);
3242
+ recordE2E('/office-hours-spec-review', 'Office Hours Spec Review E2E', result);
3243
+ expect(result.exitReason).toBe('success');
3244
+
3245
+ const summaryPath = path.join(ohDir, 'spec-review-summary.md');
3246
+ if (fs.existsSync(summaryPath)) {
3247
+ const summary = fs.readFileSync(summaryPath, 'utf-8').toLowerCase();
3248
+ // Verify the agent understood the key concepts
3249
+ expect(summary).toMatch(/5.*dimension|dimension.*5|completeness|consistency|clarity|scope|feasibility/);
3250
+ expect(summary).toMatch(/agent|subagent/);
3251
+ expect(summary).toMatch(/3.*iteration|iteration.*3|maximum.*3/);
3252
+ }
3253
+ }, 180_000);
3254
+ });
3255
+
3256
+ // --- Plan CEO Review Benefits-From E2E ---
3257
+
3258
+ describeIfSelected('Plan CEO Review Benefits-From E2E', ['plan-ceo-review-benefits'], () => {
3259
+ let benefitsDir: string;
3260
+
3261
+ beforeAll(() => {
3262
+ benefitsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-benefits-'));
3263
+ const run = (cmd: string, args: string[]) =>
3264
+ spawnSync(cmd, args, { cwd: benefitsDir, stdio: 'pipe', timeout: 5000 });
3265
+
3266
+ run('git', ['init', '-b', 'main']);
3267
+ run('git', ['config', 'user.email', 'test@test.com']);
3268
+ run('git', ['config', 'user.name', 'Test']);
3269
+ fs.writeFileSync(path.join(benefitsDir, 'README.md'), '# Test Project\n');
3270
+ run('git', ['add', '.']);
3271
+ run('git', ['commit', '-m', 'init']);
3272
+
3273
+ // Copy plan-ceo-review skill
3274
+ fs.mkdirSync(path.join(benefitsDir, 'plan-ceo-review'), { recursive: true });
3275
+ fs.copyFileSync(
3276
+ path.join(ROOT, 'plan-ceo-review', 'SKILL.md'),
3277
+ path.join(benefitsDir, 'plan-ceo-review', 'SKILL.md'),
3278
+ );
3279
+ });
3280
+
3281
+ afterAll(() => {
3282
+ try { fs.rmSync(benefitsDir, { recursive: true, force: true }); } catch {}
3283
+ });
3284
+
3285
+ test('/plan-ceo-review SKILL.md contains prerequisite skill offer', async () => {
3286
+ const result = await runSkillTest({
3287
+ prompt: `Read plan-ceo-review/SKILL.md. Search for sections about "Prerequisite" or "office-hours" or "design doc found".
3288
+
3289
+ Summarize what happens when no design doc is found — specifically:
3290
+ 1. Is /office-hours offered as a prerequisite?
3291
+ 2. What options does the user get?
3292
+ 3. Is there a mid-session detection for when the user seems lost?
3293
+
3294
+ Write your summary to ${benefitsDir}/benefits-summary.md`,
3295
+ workingDirectory: benefitsDir,
3296
+ maxTurns: 8,
3297
+ timeout: 120_000,
3298
+ testName: 'plan-ceo-review-benefits',
3299
+ runId,
3300
+ });
3301
+
3302
+ logCost('/plan-ceo-review benefits-from', result);
3303
+ recordE2E('/plan-ceo-review-benefits', 'Plan CEO Review Benefits-From E2E', result);
3304
+ expect(result.exitReason).toBe('success');
3305
+
3306
+ const summaryPath = path.join(benefitsDir, 'benefits-summary.md');
3307
+ if (fs.existsSync(summaryPath)) {
3308
+ const summary = fs.readFileSync(summaryPath, 'utf-8').toLowerCase();
3309
+ // Verify the agent understood the skill chaining
3310
+ expect(summary).toMatch(/office.hours/);
3311
+ expect(summary).toMatch(/design doc|no design/i);
3312
+ }
3313
+ }, 180_000);
3314
+ });
3315
+
3316
+ // Module-level afterAll — finalize eval collector after all tests complete
3317
+ afterAll(async () => {
3318
+ if (evalCollector) {
3319
+ try {
3320
+ await evalCollector.finalize();
3321
+ } catch (err) {
3322
+ console.error('Failed to save eval results:', err);
3323
+ }
3324
+ }
3325
+ });