@haystackeditor/cli 0.10.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/commands/config.js +81 -66
- package/dist/commands/dismiss.js +2 -1
- package/dist/commands/request-review.d.ts +15 -0
- package/dist/commands/request-review.js +95 -0
- package/dist/commands/setup.d.ts +1 -0
- package/dist/commands/setup.js +274 -2
- package/dist/commands/submit.js +1 -1
- package/dist/commands/triage.js +115 -23
- package/dist/index.d.ts +1 -0
- package/dist/index.js +43 -18
- package/dist/triage/prompts.js +13 -5
- package/dist/triage/runner.js +1 -1
- package/dist/triage/types.d.ts +1 -1
- package/dist/types.d.ts +132 -26
- package/dist/types.js +42 -5
- package/dist/utils/analysis-api.d.ts +16 -0
- package/dist/utils/analysis-api.js +23 -5
- package/dist/utils/telemetry.d.ts +17 -0
- package/dist/utils/telemetry.js +109 -0
- package/package.json +1 -1
- package/dist/commands/secrets.d.ts +0 -33
- package/dist/commands/secrets.js +0 -216
- package/dist/tools/fixtures.d.ts +0 -38
- package/dist/tools/fixtures.js +0 -199
- package/dist/tools/setup.d.ts +0 -43
- package/dist/tools/setup.js +0 -597
- package/dist/utils/skill.d.ts +0 -10
- package/dist/utils/skill.js +0 -1671
package/dist/utils/skill.js
DELETED
|
@@ -1,1671 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Create the .agents/skills/haystack.md file for agent discovery
|
|
3
|
-
* and .claude/commands/haystack.md for Claude Code slash command
|
|
4
|
-
*/
|
|
5
|
-
import * as fs from 'fs/promises';
|
|
6
|
-
import * as path from 'path';
|
|
7
|
-
/**
|
|
8
|
-
* Claude Code slash command - invoked with /haystack
|
|
9
|
-
*/
|
|
10
|
-
const CLAUDE_COMMAND_CONTENT = `# Set Up Haystack Verification
|
|
11
|
-
|
|
12
|
-
Follow .agents/skills/setup-haystack.md to set up Haystack verification for this repo.
|
|
13
|
-
`;
|
|
14
|
-
const SKILL_CONTENT = `# Haystack Verification Setup
|
|
15
|
-
|
|
16
|
-
**Your job**: Help the verification system understand this app so it can visually verify PRs work correctly.
|
|
17
|
-
|
|
18
|
-
The verification system has two parts:
|
|
19
|
-
1. **Planner** - An AI that explores the codebase to figure out what to test
|
|
20
|
-
2. **Executor** - Takes screenshots based on the plan
|
|
21
|
-
|
|
22
|
-
Your \`.haystack.json\` flows feed into the **corpus** - they're hints for the Planner about what routes exist, what selectors to use, and what the core user journey looks like.
|
|
23
|
-
|
|
24
|
-
---
|
|
25
|
-
|
|
26
|
-
## Step 1: Generate Base Config
|
|
27
|
-
|
|
28
|
-
\`\`\`bash
|
|
29
|
-
npx @haystackeditor/cli init --yes
|
|
30
|
-
\`\`\`
|
|
31
|
-
|
|
32
|
-
This detects your dev server command, port, and service dependencies.
|
|
33
|
-
|
|
34
|
-
---
|
|
35
|
-
|
|
36
|
-
## Step 2: Understand the App's Core Feature
|
|
37
|
-
|
|
38
|
-
Before writing any flows, answer this:
|
|
39
|
-
|
|
40
|
-
**What is the ONE main thing users do in this app?**
|
|
41
|
-
|
|
42
|
-
Read the codebase to understand the core user journey:
|
|
43
|
-
- E-commerce: browse products → add to cart → checkout
|
|
44
|
-
- Dashboard: view metrics → filter data → export
|
|
45
|
-
- Editor: create document → edit → save
|
|
46
|
-
- Social: view feed → create post → interact
|
|
47
|
-
- SaaS: sign up → configure → use feature
|
|
48
|
-
|
|
49
|
-
Your flows should describe THIS journey, not just "pages load".
|
|
50
|
-
|
|
51
|
-
---
|
|
52
|
-
|
|
53
|
-
## Step 3: Verify ALL Services Have Flows (CRITICAL)
|
|
54
|
-
|
|
55
|
-
**Every service in \`.haystack.json\` must have at least one verification flow.** This is the most common mistake - adding services but forgetting to add flows that exercise them.
|
|
56
|
-
|
|
57
|
-
### Check for Uncovered Services
|
|
58
|
-
|
|
59
|
-
\`\`\`bash
|
|
60
|
-
# List all services
|
|
61
|
-
grep -A1 '"services"' .haystack.json | grep -E '^\\s+"[^"]+":' | sed 's/.*"\\([^"]*\\)".*/\\1/'
|
|
62
|
-
|
|
63
|
-
# List all flows
|
|
64
|
-
grep '"name":' .haystack.json
|
|
65
|
-
|
|
66
|
-
# MANUAL CHECK: Does every service have a flow?
|
|
67
|
-
\`\`\`
|
|
68
|
-
|
|
69
|
-
### Types of Services
|
|
70
|
-
|
|
71
|
-
| Service Type | How to Verify |
|
|
72
|
-
|--------------|---------------|
|
|
73
|
-
| **Web frontend** | UI flows (navigate, wait_for, screenshot) |
|
|
74
|
-
| **API/Worker** | UI flows that hit API endpoints |
|
|
75
|
-
| **CLI/Batch** | Backend flows with golden inputs |
|
|
76
|
-
| **Analysis pipeline** | Backend flows with known PR input |
|
|
77
|
-
|
|
78
|
-
### Backend Verification (CLI tools, analysis pipelines, batch jobs)
|
|
79
|
-
|
|
80
|
-
**Not everything is a web page.** For CLI tools and batch processes, you need:
|
|
81
|
-
|
|
82
|
-
1. **Golden input** - A known-good input that produces predictable output
|
|
83
|
-
2. **Run command** - How to execute the service
|
|
84
|
-
3. **Output assertion** - What to check in the output
|
|
85
|
-
|
|
86
|
-
\`\`\`json
|
|
87
|
-
{
|
|
88
|
-
"name": "Backend pipeline - golden input",
|
|
89
|
-
"description": "Run pipeline on known input to verify it works",
|
|
90
|
-
"trigger": "on_change",
|
|
91
|
-
"watch_patterns": ["packages/my-pipeline/src/**"],
|
|
92
|
-
"service": "my-pipeline",
|
|
93
|
-
"type": "backend",
|
|
94
|
-
"steps": [
|
|
95
|
-
{
|
|
96
|
-
"action": "run",
|
|
97
|
-
"command": "pnpm start",
|
|
98
|
-
"env": { "INPUT_ID": "known-good-input-123" },
|
|
99
|
-
"timeout": 120
|
|
100
|
-
},
|
|
101
|
-
{ "action": "assert_exit_code", "code": 0 },
|
|
102
|
-
{ "action": "assert_output_contains", "pattern": "Processing complete" }
|
|
103
|
-
]
|
|
104
|
-
}
|
|
105
|
-
\`\`\`
|
|
106
|
-
|
|
107
|
-
---
|
|
108
|
-
|
|
109
|
-
## STOP - Confirm Golden URLs and Golden Data
|
|
110
|
-
|
|
111
|
-
**If you identified backend services that need verification, you MUST ask the user with a proposed example:**
|
|
112
|
-
|
|
113
|
-
First, explore the codebase to find:
|
|
114
|
-
- Example inputs used in tests, scripts, or documentation
|
|
115
|
-
- Default/demo values in config files or environment variables
|
|
116
|
-
- IDs or URLs referenced in comments or READMEs
|
|
117
|
-
|
|
118
|
-
Then propose a specific example:
|
|
119
|
-
|
|
120
|
-
> I found these backend services that need verification:
|
|
121
|
-
> - **[service name]**: [what it does]
|
|
122
|
-
>
|
|
123
|
-
> For backend verification, I need **golden test data** - stable inputs that produce predictable output.
|
|
124
|
-
>
|
|
125
|
-
> **Based on what I found in the codebase, here's my suggestion:**
|
|
126
|
-
> - **Golden URL/Input**: \`[specific example you found, e.g., a test ID, demo endpoint, or sample file]\`
|
|
127
|
-
> - **Command**: \`[the command to run with this input]\`
|
|
128
|
-
> - **Expected Output**: \`[what success looks like based on the code]\`
|
|
129
|
-
>
|
|
130
|
-
> Does this look right? Or should I use different test data?
|
|
131
|
-
|
|
132
|
-
**Wait for the user to confirm or provide alternatives before adding backend flows.**
|
|
133
|
-
|
|
134
|
-
---
|
|
135
|
-
|
|
136
|
-
## Step 4: Assess Data Needs
|
|
137
|
-
|
|
138
|
-
\`\`\`bash
|
|
139
|
-
# Find API calls
|
|
140
|
-
grep -r "fetch(\\|useQuery\\|useSWR\\|axios" src/ --include="*.tsx" --include="*.ts" | wc -l
|
|
141
|
-
|
|
142
|
-
# Find dynamic routes
|
|
143
|
-
grep -r "useParams\\|router.query\\|\\[.*\\]" src/ --include="*.tsx" | head -10
|
|
144
|
-
|
|
145
|
-
# Find external domains
|
|
146
|
-
grep -r "https://" src/ --include="*.ts" --include="*.tsx" | grep -v node_modules | head -10
|
|
147
|
-
\`\`\`
|
|
148
|
-
|
|
149
|
-
---
|
|
150
|
-
|
|
151
|
-
## STOP - Ask About Test Data Strategy
|
|
152
|
-
|
|
153
|
-
**You MUST ask the user before proceeding:**
|
|
154
|
-
|
|
155
|
-
> I analyzed the codebase and found:
|
|
156
|
-
> - X API fetch calls
|
|
157
|
-
> - Y dynamic routes with parameters
|
|
158
|
-
> - These external domains: [list them]
|
|
159
|
-
>
|
|
160
|
-
> **How should I handle test data for verification?**
|
|
161
|
-
>
|
|
162
|
-
> 1. **Passthrough** - Let API calls through to real servers (add domains to \`network.allow\`)
|
|
163
|
-
> 2. **Staging URL** - Point flows at your staging/demo environment
|
|
164
|
-
> 3. **Local fixtures** - JSON files in \`fixtures/\` directory
|
|
165
|
-
> 4. **Skip data pages** - Only test static pages that don't need API data
|
|
166
|
-
|
|
167
|
-
**Wait for the user's response before continuing.**
|
|
168
|
-
|
|
169
|
-
---
|
|
170
|
-
|
|
171
|
-
## STOP - Confirm Frontend Golden URLs
|
|
172
|
-
|
|
173
|
-
**Goldens are URLs with real staging data** - they let verification test pages that need actual content to render properly. You need MULTIPLE goldens to cover different scenarios.
|
|
174
|
-
|
|
175
|
-
**Before writing flows for dynamic routes, explore the codebase to find example data:**
|
|
176
|
-
|
|
177
|
-
1. Check test files (\`*.test.ts\`, \`cypress/\`, \`playwright/\`) for example IDs/URLs
|
|
178
|
-
2. Check README/docs for demo links or example usage
|
|
179
|
-
3. Check seed data, fixtures, or mock files
|
|
180
|
-
4. Check \`.env.example\` or config for staging URLs
|
|
181
|
-
5. Check existing routes to understand the URL format
|
|
182
|
-
|
|
183
|
-
**Then propose MULTIPLE golden URLs (aim for 3-5) covering different scenarios:**
|
|
184
|
-
|
|
185
|
-
> I found these dynamic routes that need staging data to render:
|
|
186
|
-
>
|
|
187
|
-
> **[Route Name]** (\`/path/:param1/:param2\`)
|
|
188
|
-
>
|
|
189
|
-
> | Golden | URL | Why |
|
|
190
|
-
> |--------|-----|-----|
|
|
191
|
-
> | Large dataset | \`/path/example/large-123\` | Tests performance with many items |
|
|
192
|
-
> | Typical case | \`/path/example/medium-456\` | Common user scenario |
|
|
193
|
-
> | Edge case | \`/path/example/empty-789\` | Tests empty/minimal state |
|
|
194
|
-
> | Error state | \`/path/example/error-000\` | Tests error handling UI |
|
|
195
|
-
>
|
|
196
|
-
> **Evidence:**
|
|
197
|
-
> - Found \`large-123\` in \`cypress/fixtures/test-data.json\`
|
|
198
|
-
> - Found \`medium-456\` in README as demo example
|
|
199
|
-
> - Found \`empty-789\` in \`src/tests/edge-cases.test.ts\`
|
|
200
|
-
>
|
|
201
|
-
> Does this look right?
|
|
202
|
-
> 1. ✅ Yes, use these goldens
|
|
203
|
-
> 2. 🔄 Use different URLs (I'll provide them)
|
|
204
|
-
|
|
205
|
-
**Add goldens to \`verification.goldens\` in \`.haystack.json\` using this EXACT format:**
|
|
206
|
-
|
|
207
|
-
\`\`\`json
|
|
208
|
-
{
|
|
209
|
-
"verification": {
|
|
210
|
-
"goldens": {
|
|
211
|
-
"frontend": [
|
|
212
|
-
{
|
|
213
|
-
"id": "large-dataset",
|
|
214
|
-
"description": "100+ items - tests rendering performance",
|
|
215
|
-
"route": "/the/actual/url/with/data"
|
|
216
|
-
},
|
|
217
|
-
{
|
|
218
|
-
"id": "typical-case",
|
|
219
|
-
"description": "Standard 10-20 items",
|
|
220
|
-
"route": "/another/url/with/typical/data"
|
|
221
|
-
},
|
|
222
|
-
{
|
|
223
|
-
"id": "empty-state",
|
|
224
|
-
"description": "No items - tests empty state UI",
|
|
225
|
-
"route": "/url/that/shows/empty/state"
|
|
226
|
-
}
|
|
227
|
-
],
|
|
228
|
-
"backend": [
|
|
229
|
-
{
|
|
230
|
-
"id": "golden-input-1",
|
|
231
|
-
"description": "Known-good input for pipeline testing",
|
|
232
|
-
"input": { "type": "github_pr", "value": "owner/repo#123" }
|
|
233
|
-
}
|
|
234
|
-
]
|
|
235
|
-
},
|
|
236
|
-
"commands": [...]
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
\`\`\`
|
|
240
|
-
|
|
241
|
-
**DO NOT:**
|
|
242
|
-
- Ask the user what format to use - USE THIS FORMAT
|
|
243
|
-
- Put goldens at the top level - they go inside \`verification.goldens\`
|
|
244
|
-
- Use a flat object like \`{ "goldens": { "name": "url" } }\` - use the array format above
|
|
245
|
-
- Embed golden URLs only in flow steps - also add them to the goldens section for discoverability
|
|
246
|
-
|
|
247
|
-
**CRITICAL**:
|
|
248
|
-
- Propose MULTIPLE goldens (3-5), not just one
|
|
249
|
-
- Cover different scenarios: large, typical, empty, error states
|
|
250
|
-
- Always propose SPECIFIC URLs, never ask the user to freestyle
|
|
251
|
-
- Show WHERE you found the example data (file path, line if possible)
|
|
252
|
-
- Make sure URLs match the app's actual route format (check \`<Route path=\`)
|
|
253
|
-
- Wait for confirmation before writing flows with these URLs
|
|
254
|
-
|
|
255
|
-
---
|
|
256
|
-
|
|
257
|
-
## Step 5: Write Flows
|
|
258
|
-
|
|
259
|
-
Flows tell the Planner about your app's routes, UI elements, and user journeys.
|
|
260
|
-
|
|
261
|
-
### Structure
|
|
262
|
-
|
|
263
|
-
\`\`\`yaml
|
|
264
|
-
flows:
|
|
265
|
-
- name: "Descriptive name of what this tests"
|
|
266
|
-
trigger: always # or on_change with watch_patterns
|
|
267
|
-
steps:
|
|
268
|
-
- action: navigate
|
|
269
|
-
url: "/"
|
|
270
|
-
- action: wait_for
|
|
271
|
-
selector: "[data-testid='specific-element']"
|
|
272
|
-
- action: click
|
|
273
|
-
selector: "[data-testid='button']"
|
|
274
|
-
- action: screenshot
|
|
275
|
-
name: "result"
|
|
276
|
-
\`\`\`
|
|
277
|
-
|
|
278
|
-
### What to Include
|
|
279
|
-
|
|
280
|
-
**Core journey flow** (trigger: always):
|
|
281
|
-
- The main thing users do in your app
|
|
282
|
-
- Multiple steps with interactions (click, type)
|
|
283
|
-
- Waits for meaningful state changes
|
|
284
|
-
|
|
285
|
-
**Route coverage flows**:
|
|
286
|
-
- One flow per major route
|
|
287
|
-
- Uses specific selectors the Planner can learn from
|
|
288
|
-
- \`watch_patterns\` to only run when relevant files change
|
|
289
|
-
|
|
290
|
-
### Finding Good Selectors
|
|
291
|
-
|
|
292
|
-
\`\`\`bash
|
|
293
|
-
# Find data-testid attributes
|
|
294
|
-
grep -r "data-testid" src/ --include="*.tsx" | head -20
|
|
295
|
-
|
|
296
|
-
# Find aria-labels
|
|
297
|
-
grep -r "aria-label" src/ --include="*.tsx" | head -20
|
|
298
|
-
|
|
299
|
-
# Find component class names
|
|
300
|
-
grep -r "className=" src/components/ --include="*.tsx" | head -20
|
|
301
|
-
\`\`\`
|
|
302
|
-
|
|
303
|
-
Use specific selectors like:
|
|
304
|
-
- \`[data-testid='dashboard-chart']\`
|
|
305
|
-
- \`[aria-label='Submit form']\`
|
|
306
|
-
- \`.pricing-table\`
|
|
307
|
-
- \`button[type='submit']\`
|
|
308
|
-
|
|
309
|
-
If good selectors don't exist, add \`data-testid\` to key components.
|
|
310
|
-
|
|
311
|
-
### watch_patterns
|
|
312
|
-
|
|
313
|
-
For flows that only matter when certain files change:
|
|
314
|
-
|
|
315
|
-
\`\`\`yaml
|
|
316
|
-
- name: "Settings page works"
|
|
317
|
-
trigger: on_change
|
|
318
|
-
watch_patterns:
|
|
319
|
-
- "src/pages/settings/**"
|
|
320
|
-
- "src/components/settings/**"
|
|
321
|
-
steps:
|
|
322
|
-
- action: navigate
|
|
323
|
-
url: "/settings"
|
|
324
|
-
# ...
|
|
325
|
-
\`\`\`
|
|
326
|
-
|
|
327
|
-
---
|
|
328
|
-
|
|
329
|
-
## Step 6: Configure Fixtures (if needed)
|
|
330
|
-
|
|
331
|
-
Based on user's answer in Step 3:
|
|
332
|
-
|
|
333
|
-
**Passthrough**:
|
|
334
|
-
\`\`\`yaml
|
|
335
|
-
network:
|
|
336
|
-
allow:
|
|
337
|
-
- "api.example.com"
|
|
338
|
-
- "cdn.example.com"
|
|
339
|
-
\`\`\`
|
|
340
|
-
|
|
341
|
-
**Staging URL**:
|
|
342
|
-
\`\`\`yaml
|
|
343
|
-
flows:
|
|
344
|
-
- name: "Dashboard with real data"
|
|
345
|
-
steps:
|
|
346
|
-
- action: navigate
|
|
347
|
-
url: "https://staging.example.com/dashboard"
|
|
348
|
-
\`\`\`
|
|
349
|
-
|
|
350
|
-
**Local fixtures**:
|
|
351
|
-
\`\`\`yaml
|
|
352
|
-
fixtures:
|
|
353
|
-
- pattern: "/api/user"
|
|
354
|
-
source: "file://fixtures/user.json"
|
|
355
|
-
- pattern: "/api/data/*"
|
|
356
|
-
source: "file://fixtures/data.json"
|
|
357
|
-
\`\`\`
|
|
358
|
-
|
|
359
|
-
---
|
|
360
|
-
|
|
361
|
-
## Step 7: Commit
|
|
362
|
-
|
|
363
|
-
\`\`\`bash
|
|
364
|
-
git add .haystack.json fixtures/
|
|
365
|
-
git commit -m "Add Haystack verification"
|
|
366
|
-
\`\`\`
|
|
367
|
-
|
|
368
|
-
Done! The Planner will use your flows to understand the app and create verification plans for PRs.
|
|
369
|
-
`;
|
|
370
|
-
const REFERENCE_CONTENT = `# Haystack Reference
|
|
371
|
-
|
|
372
|
-
Reference material for \`.haystack.json\` configuration. Only consult when needed for a specific step.
|
|
373
|
-
|
|
374
|
-
---
|
|
375
|
-
|
|
376
|
-
## Fixture Patterns
|
|
377
|
-
|
|
378
|
-
### Option 1: Passthrough (Recommended - Easiest!)
|
|
379
|
-
\`\`\`yaml
|
|
380
|
-
# Let API calls through to real servers - no mocking needed
|
|
381
|
-
fixtures:
|
|
382
|
-
- pattern: "/api/*"
|
|
383
|
-
source: passthrough
|
|
384
|
-
|
|
385
|
-
network:
|
|
386
|
-
allow:
|
|
387
|
-
- "api.example.com"
|
|
388
|
-
- "cdn.example.com"
|
|
389
|
-
\`\`\`
|
|
390
|
-
|
|
391
|
-
**This is the easiest option.** Just allow the domains your app calls and let real APIs handle requests. No fixtures to maintain.
|
|
392
|
-
|
|
393
|
-
### Option 2: Staging/Demo URL
|
|
394
|
-
\`\`\`yaml
|
|
395
|
-
# Point flows directly at staging - no fixtures needed
|
|
396
|
-
flows:
|
|
397
|
-
- name: "Dashboard loads"
|
|
398
|
-
url: "https://staging.example.com/dashboard"
|
|
399
|
-
wait_for_selector: ".dashboard-content"
|
|
400
|
-
|
|
401
|
-
network:
|
|
402
|
-
allow:
|
|
403
|
-
- "staging.example.com"
|
|
404
|
-
\`\`\`
|
|
405
|
-
|
|
406
|
-
### Option 3: Pre-signed URLs (S3/Cloud Storage)
|
|
407
|
-
\`\`\`yaml
|
|
408
|
-
fixtures:
|
|
409
|
-
- pattern: "/api/data"
|
|
410
|
-
source: "$FIXTURE_DATA_URL" # Pre-signed URL from CI
|
|
411
|
-
- pattern: "/api/users"
|
|
412
|
-
source: "$FIXTURE_USERS_URL"
|
|
413
|
-
|
|
414
|
-
secrets:
|
|
415
|
-
- FIXTURE_DATA_URL
|
|
416
|
-
- FIXTURE_USERS_URL
|
|
417
|
-
\`\`\`
|
|
418
|
-
|
|
419
|
-
See "Pre-signed URL Setup" below for CI configuration.
|
|
420
|
-
|
|
421
|
-
### Option 4: Local Fixtures (Simple Apps Only)
|
|
422
|
-
\`\`\`yaml
|
|
423
|
-
fixtures:
|
|
424
|
-
- pattern: "/api/user"
|
|
425
|
-
source: "file://fixtures/user.json"
|
|
426
|
-
- pattern: "/api/settings"
|
|
427
|
-
source: "file://fixtures/settings.json"
|
|
428
|
-
\`\`\`
|
|
429
|
-
|
|
430
|
-
Create matching JSON files in \`fixtures/\` directory. Only use for small, stable data.
|
|
431
|
-
|
|
432
|
-
### Option 5: HTTP Endpoint (Self-hosted)
|
|
433
|
-
\`\`\`yaml
|
|
434
|
-
fixtures:
|
|
435
|
-
- pattern: "/api/*"
|
|
436
|
-
source: "https://fixtures.yourcompany.com/api"
|
|
437
|
-
headers:
|
|
438
|
-
Authorization: "Bearer $FIXTURES_TOKEN"
|
|
439
|
-
|
|
440
|
-
network:
|
|
441
|
-
allow:
|
|
442
|
-
- "fixtures.yourcompany.com"
|
|
443
|
-
\`\`\`
|
|
444
|
-
|
|
445
|
-
---
|
|
446
|
-
|
|
447
|
-
## Pre-signed URL Setup
|
|
448
|
-
|
|
449
|
-
Generate temporary URLs in CI, pass to Haystack. No cloud credentials in sandbox.
|
|
450
|
-
|
|
451
|
-
**Note:** URLs valid for 24 hours. Run on push to main to keep URLs fresh.
|
|
452
|
-
|
|
453
|
-
### AWS S3
|
|
454
|
-
\`\`\`bash
|
|
455
|
-
# Generate pre-signed URL (valid 24 hours)
|
|
456
|
-
aws s3 presign s3://my-bucket/fixtures/data.json --expires-in 86400
|
|
457
|
-
\`\`\`
|
|
458
|
-
|
|
459
|
-
### Google Cloud Storage
|
|
460
|
-
\`\`\`bash
|
|
461
|
-
# Generate signed URL (valid 24 hours)
|
|
462
|
-
gcloud storage sign-url gs://my-bucket/fixtures/data.json --duration=24h
|
|
463
|
-
\`\`\`
|
|
464
|
-
|
|
465
|
-
### Azure Blob Storage
|
|
466
|
-
\`\`\`bash
|
|
467
|
-
# Generate SAS URL (valid 24 hours)
|
|
468
|
-
az storage blob generate-sas --account-name myaccount --container fixtures \\
|
|
469
|
-
--name data.json --permissions r --expiry $(date -u -d '+1 day' +%Y-%m-%dT%H:%MZ) \\
|
|
470
|
-
--full-uri
|
|
471
|
-
\`\`\`
|
|
472
|
-
|
|
473
|
-
### Cloudflare R2 / DigitalOcean Spaces / MinIO
|
|
474
|
-
\`\`\`bash
|
|
475
|
-
# S3-compatible - use aws cli with custom endpoint
|
|
476
|
-
aws s3 presign s3://my-bucket/data.json --expires-in 86400 \\
|
|
477
|
-
--endpoint-url https://your-r2-endpoint.com
|
|
478
|
-
\`\`\`
|
|
479
|
-
|
|
480
|
-
---
|
|
481
|
-
|
|
482
|
-
### GitHub Actions Example
|
|
483
|
-
\`\`\`yaml
|
|
484
|
-
# Run on push to main to keep URLs fresh
|
|
485
|
-
on:
|
|
486
|
-
push:
|
|
487
|
-
branches: [main]
|
|
488
|
-
|
|
489
|
-
jobs:
|
|
490
|
-
update-fixture-urls:
|
|
491
|
-
runs-on: ubuntu-latest
|
|
492
|
-
permissions:
|
|
493
|
-
id-token: write # OIDC - no long-lived secrets
|
|
494
|
-
steps:
|
|
495
|
-
- uses: aws-actions/configure-aws-credentials@v4
|
|
496
|
-
with:
|
|
497
|
-
role-to-assume: arn:aws:iam::123456789:role/haystack-fixtures
|
|
498
|
-
aws-region: us-east-1
|
|
499
|
-
|
|
500
|
-
- name: Generate pre-signed URLs (valid 24 hours)
|
|
501
|
-
run: |
|
|
502
|
-
URL=$(aws s3 presign s3://my-bucket/fixtures/data.json --expires-in 86400)
|
|
503
|
-
haystack secrets set FIXTURE_DATA_URL "$URL"
|
|
504
|
-
\`\`\`
|
|
505
|
-
|
|
506
|
-
### General Approach
|
|
507
|
-
1. Use OIDC to get temporary cloud credentials (no long-lived secrets)
|
|
508
|
-
2. Generate pre-signed/signed URL for each fixture file (24 hour expiry)
|
|
509
|
-
3. Store with \`haystack secrets set FIXTURE_URL "$URL"\`
|
|
510
|
-
4. Run on push to main to keep URLs fresh
|
|
511
|
-
|
|
512
|
-
---
|
|
513
|
-
|
|
514
|
-
## Flow Examples
|
|
515
|
-
|
|
516
|
-
### Basic Page Flow
|
|
517
|
-
\`\`\`yaml
|
|
518
|
-
flows:
|
|
519
|
-
- name: "Dashboard loads"
|
|
520
|
-
description: "Verify dashboard renders correctly"
|
|
521
|
-
trigger: always
|
|
522
|
-
watch_patterns:
|
|
523
|
-
- "src/components/dashboard/**"
|
|
524
|
-
steps:
|
|
525
|
-
- action: navigate
|
|
526
|
-
url: "/dashboard"
|
|
527
|
-
- action: wait_for
|
|
528
|
-
selector: "[data-testid='dashboard-content']"
|
|
529
|
-
- action: assert_no_errors
|
|
530
|
-
- action: screenshot
|
|
531
|
-
name: "dashboard"
|
|
532
|
-
\`\`\`
|
|
533
|
-
|
|
534
|
-
### Interactive Flow (click)
|
|
535
|
-
\`\`\`yaml
|
|
536
|
-
- name: "Modal opens"
|
|
537
|
-
trigger: on_change
|
|
538
|
-
watch_patterns:
|
|
539
|
-
- "src/components/settings/**"
|
|
540
|
-
steps:
|
|
541
|
-
- action: navigate
|
|
542
|
-
url: "/settings"
|
|
543
|
-
- action: wait_for
|
|
544
|
-
selector: ".settings-page"
|
|
545
|
-
- action: click
|
|
546
|
-
selector: "button[aria-label='Settings']"
|
|
547
|
-
- action: wait_for
|
|
548
|
-
selector: "[role='dialog']"
|
|
549
|
-
- action: screenshot
|
|
550
|
-
name: "settings-modal"
|
|
551
|
-
\`\`\`
|
|
552
|
-
|
|
553
|
-
### Form Flow (type)
|
|
554
|
-
\`\`\`yaml
|
|
555
|
-
- name: "Contact form works"
|
|
556
|
-
trigger: on_change
|
|
557
|
-
watch_patterns:
|
|
558
|
-
- "src/components/contact/**"
|
|
559
|
-
steps:
|
|
560
|
-
- action: navigate
|
|
561
|
-
url: "/contact"
|
|
562
|
-
- action: wait_for
|
|
563
|
-
selector: "form"
|
|
564
|
-
- action: type
|
|
565
|
-
selector: "input[name='email']"
|
|
566
|
-
value: "test@example.com"
|
|
567
|
-
- action: click
|
|
568
|
-
selector: "button[type='submit']"
|
|
569
|
-
- action: wait_for
|
|
570
|
-
selector: ".success-message"
|
|
571
|
-
\`\`\`
|
|
572
|
-
|
|
573
|
-
### Backend API Flow
|
|
574
|
-
\`\`\`yaml
|
|
575
|
-
- name: "API health check"
|
|
576
|
-
trigger: always
|
|
577
|
-
steps:
|
|
578
|
-
- action: http_request
|
|
579
|
-
method: GET
|
|
580
|
-
url: "http://localhost:3001/health"
|
|
581
|
-
- action: assert_status
|
|
582
|
-
status: 200
|
|
583
|
-
\`\`\`
|
|
584
|
-
|
|
585
|
-
---
|
|
586
|
-
|
|
587
|
-
## Finding Good Selectors
|
|
588
|
-
|
|
589
|
-
**Priority order:**
|
|
590
|
-
1. \`[data-testid='feature-name']\` - Best
|
|
591
|
-
2. \`[role='main']\`, \`[aria-label='...']\` - Semantic
|
|
592
|
-
3. \`.specific-class-name\` - Component-specific
|
|
593
|
-
4. Avoid: \`#root\`, \`div\`, \`h1\` - Too generic
|
|
594
|
-
|
|
595
|
-
**How to find:**
|
|
596
|
-
\`\`\`bash
|
|
597
|
-
# Find data-testid attributes
|
|
598
|
-
grep -r "data-testid" src/ --include="*.tsx"
|
|
599
|
-
|
|
600
|
-
# Find class names
|
|
601
|
-
grep -r "className=" src/components/Dashboard.tsx
|
|
602
|
-
|
|
603
|
-
# Find semantic roles
|
|
604
|
-
grep -r "role=\\|aria-label=" src/ --include="*.tsx"
|
|
605
|
-
\`\`\`
|
|
606
|
-
|
|
607
|
-
---
|
|
608
|
-
|
|
609
|
-
## Config Structure
|
|
610
|
-
|
|
611
|
-
⚠️ **IMPORTANT**: \`flows\` must be at TOP LEVEL, not nested under \`verification\`!
|
|
612
|
-
|
|
613
|
-
\`\`\`yaml
|
|
614
|
-
version: "1"
|
|
615
|
-
name: my-app
|
|
616
|
-
|
|
617
|
-
dev_server:
|
|
618
|
-
command: pnpm dev
|
|
619
|
-
port: 3000
|
|
620
|
-
ready_pattern: "ready|Local:|started" # Regex for server ready
|
|
621
|
-
|
|
622
|
-
# Verification commands - MUST include build
|
|
623
|
-
verification:
|
|
624
|
-
commands:
|
|
625
|
-
- name: build
|
|
626
|
-
run: pnpm build # ← REQUIRED
|
|
627
|
-
- name: lint
|
|
628
|
-
run: pnpm lint
|
|
629
|
-
- name: typecheck
|
|
630
|
-
run: pnpm tsc --noEmit
|
|
631
|
-
|
|
632
|
-
network:
|
|
633
|
-
allow:
|
|
634
|
-
- "api.example.com"
|
|
635
|
-
|
|
636
|
-
# ⚠️ flows at TOP LEVEL - NOT under verification!
|
|
637
|
-
flows:
|
|
638
|
-
- name: "Page loads"
|
|
639
|
-
trigger: always
|
|
640
|
-
steps:
|
|
641
|
-
- action: navigate
|
|
642
|
-
url: "/"
|
|
643
|
-
- action: wait_for
|
|
644
|
-
selector: "[data-testid='main']"
|
|
645
|
-
- action: screenshot
|
|
646
|
-
name: "home"
|
|
647
|
-
|
|
648
|
-
secrets:
|
|
649
|
-
- API_TOKEN
|
|
650
|
-
|
|
651
|
-
fixtures:
|
|
652
|
-
- pattern: "/api/*"
|
|
653
|
-
source: passthrough
|
|
654
|
-
\`\`\`
|
|
655
|
-
|
|
656
|
-
---
|
|
657
|
-
|
|
658
|
-
## Monorepo Configuration
|
|
659
|
-
|
|
660
|
-
\`\`\`yaml
|
|
661
|
-
version: "1"
|
|
662
|
-
name: my-monorepo
|
|
663
|
-
|
|
664
|
-
services:
|
|
665
|
-
frontend:
|
|
666
|
-
root: ./
|
|
667
|
-
command: pnpm dev
|
|
668
|
-
port: 3000
|
|
669
|
-
ready_pattern: "Local:"
|
|
670
|
-
|
|
671
|
-
api:
|
|
672
|
-
root: packages/api
|
|
673
|
-
command: pnpm dev
|
|
674
|
-
port: 8080
|
|
675
|
-
ready_pattern: "listening"
|
|
676
|
-
|
|
677
|
-
worker:
|
|
678
|
-
root: infra/worker
|
|
679
|
-
command: pnpm wrangler dev
|
|
680
|
-
port: 8787
|
|
681
|
-
\`\`\`
|
|
682
|
-
|
|
683
|
-
---
|
|
684
|
-
|
|
685
|
-
## Flow Triggers
|
|
686
|
-
|
|
687
|
-
| Trigger | When it runs |
|
|
688
|
-
|---------|--------------|
|
|
689
|
-
| \`always\` | Every PR |
|
|
690
|
-
| \`on_change\` | Only when \`watch_patterns\` match changed files |
|
|
691
|
-
`;
|
|
692
|
-
const PREPARE_VERIFICATION_CONTENT = `# Prepare Codebase for Verification
|
|
693
|
-
|
|
694
|
-
**Your job**: Make this codebase easy to verify by adding semantic identifiers that the verification system can target.
|
|
695
|
-
|
|
696
|
-
The verification Planner needs to find UI elements by selectors. Generic selectors like \`div\` or \`.flex\` are useless. Your job is to add meaningful identifiers throughout the codebase.
|
|
697
|
-
|
|
698
|
-
---
|
|
699
|
-
|
|
700
|
-
## ⚠️ CRITICAL RULE: Every Button MUST Have \`data-testid\` and \`aria-label\`
|
|
701
|
-
|
|
702
|
-
**This is non-negotiable.** Every \`<button>\` element in the codebase MUST have:
|
|
703
|
-
1. \`data-testid="descriptive-name"\` - For automated testing/verification
|
|
704
|
-
2. \`aria-label="Action description"\` - For accessibility AND verification
|
|
705
|
-
|
|
706
|
-
**Why?** The verification system clicks buttons by their \`data-testid\` or \`aria-label\`. Without these, buttons are invisible to automation. We learned this the hard way - a theme toggle button without these attributes broke our entire verification pipeline.
|
|
707
|
-
|
|
708
|
-
\`\`\`tsx
|
|
709
|
-
// ❌ WRONG - No identifiers, verification cannot click this
|
|
710
|
-
<button onClick={toggleTheme}>
|
|
711
|
-
<SunIcon />
|
|
712
|
-
</button>
|
|
713
|
-
|
|
714
|
-
// ✅ CORRECT - Both data-testid AND aria-label
|
|
715
|
-
<button
|
|
716
|
-
onClick={toggleTheme}
|
|
717
|
-
data-testid="theme-toggle"
|
|
718
|
-
aria-label="Toggle dark mode"
|
|
719
|
-
>
|
|
720
|
-
<SunIcon />
|
|
721
|
-
</button>
|
|
722
|
-
\`\`\`
|
|
723
|
-
|
|
724
|
-
**For button wrapper components**, pass through these props:
|
|
725
|
-
\`\`\`tsx
|
|
726
|
-
interface ButtonProps {
|
|
727
|
-
'data-testid'?: string;
|
|
728
|
-
'aria-label'?: string;
|
|
729
|
-
// ...other props
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
function IconButton({ 'data-testid': testId, 'aria-label': ariaLabel, ...props }: ButtonProps) {
|
|
733
|
-
return (
|
|
734
|
-
<button data-testid={testId} aria-label={ariaLabel} {...props}>
|
|
735
|
-
{props.children}
|
|
736
|
-
</button>
|
|
737
|
-
);
|
|
738
|
-
}
|
|
739
|
-
\`\`\`
|
|
740
|
-
|
|
741
|
-
---
|
|
742
|
-
|
|
743
|
-
## What to Add
|
|
744
|
-
|
|
745
|
-
### 1. \`aria-label\` on Interactive Elements
|
|
746
|
-
|
|
747
|
-
Every clickable/interactive element should have an aria-label describing what it does:
|
|
748
|
-
|
|
749
|
-
\`\`\`tsx
|
|
750
|
-
// Before
|
|
751
|
-
<button onClick={onSave}>💾</button>
|
|
752
|
-
<button onClick={() => setOpen(true)}>
|
|
753
|
-
<MenuIcon />
|
|
754
|
-
</button>
|
|
755
|
-
|
|
756
|
-
// After
|
|
757
|
-
<button onClick={onSave} aria-label="Save document">💾</button>
|
|
758
|
-
<button onClick={() => setOpen(true)} aria-label="Open menu">
|
|
759
|
-
<MenuIcon />
|
|
760
|
-
</button>
|
|
761
|
-
\`\`\`
|
|
762
|
-
|
|
763
|
-
**Target elements:**
|
|
764
|
-
- Buttons (especially icon-only buttons)
|
|
765
|
-
- Links without descriptive text
|
|
766
|
-
- Toggle switches
|
|
767
|
-
- Dropdown triggers
|
|
768
|
-
- Modal open/close buttons
|
|
769
|
-
- Form submit buttons
|
|
770
|
-
|
|
771
|
-
### 2. \`data-testid\` on Key Sections
|
|
772
|
-
|
|
773
|
-
Major UI sections should have data-testid for easy targeting:
|
|
774
|
-
|
|
775
|
-
\`\`\`tsx
|
|
776
|
-
// Before
|
|
777
|
-
<div className="flex flex-col p-4">
|
|
778
|
-
<h1>Dashboard</h1>
|
|
779
|
-
{/* content */}
|
|
780
|
-
</div>
|
|
781
|
-
|
|
782
|
-
// After
|
|
783
|
-
<div className="flex flex-col p-4" data-testid="dashboard-container">
|
|
784
|
-
<h1>Dashboard</h1>
|
|
785
|
-
{/* content */}
|
|
786
|
-
</div>
|
|
787
|
-
\`\`\`
|
|
788
|
-
|
|
789
|
-
**Target sections:**
|
|
790
|
-
- Page containers (dashboard, settings, profile)
|
|
791
|
-
- Navigation bars/sidebars
|
|
792
|
-
- Modal/dialog content
|
|
793
|
-
- Form containers
|
|
794
|
-
- Data tables/lists
|
|
795
|
-
- Card components
|
|
796
|
-
- Loading states
|
|
797
|
-
- Error states
|
|
798
|
-
- Empty states
|
|
799
|
-
|
|
800
|
-
### 3. \`role\` Attributes for Semantic Structure
|
|
801
|
-
|
|
802
|
-
Add ARIA roles where HTML semantics aren't clear:
|
|
803
|
-
|
|
804
|
-
\`\`\`tsx
|
|
805
|
-
// Before
|
|
806
|
-
<div className="modal-overlay">
|
|
807
|
-
<div className="modal-content">
|
|
808
|
-
|
|
809
|
-
// After
|
|
810
|
-
<div className="modal-overlay" role="presentation">
|
|
811
|
-
<div className="modal-content" role="dialog" aria-modal="true">
|
|
812
|
-
\`\`\`
|
|
813
|
-
|
|
814
|
-
**Common roles:**
|
|
815
|
-
- \`role="dialog"\` - Modals/dialogs
|
|
816
|
-
- \`role="navigation"\` - Nav sections
|
|
817
|
-
- \`role="main"\` - Main content area
|
|
818
|
-
- \`role="alert"\` - Error/success messages
|
|
819
|
-
- \`role="status"\` - Loading indicators
|
|
820
|
-
- \`role="tablist"\`, \`role="tab"\`, \`role="tabpanel"\` - Tab interfaces
|
|
821
|
-
|
|
822
|
-
### 4. State Indicators
|
|
823
|
-
|
|
824
|
-
Add attributes that indicate UI state:
|
|
825
|
-
|
|
826
|
-
\`\`\`tsx
|
|
827
|
-
// Before
|
|
828
|
-
<button onClick={toggle}>
|
|
829
|
-
{isOpen ? 'Close' : 'Open'}
|
|
830
|
-
</button>
|
|
831
|
-
|
|
832
|
-
// After
|
|
833
|
-
<button
|
|
834
|
-
onClick={toggle}
|
|
835
|
-
aria-expanded={isOpen}
|
|
836
|
-
aria-label={isOpen ? 'Close panel' : 'Open panel'}
|
|
837
|
-
>
|
|
838
|
-
{isOpen ? 'Close' : 'Open'}
|
|
839
|
-
</button>
|
|
840
|
-
\`\`\`
|
|
841
|
-
|
|
842
|
-
**State attributes:**
|
|
843
|
-
- \`aria-expanded\` - Collapsible sections
|
|
844
|
-
- \`aria-selected\` - Selected items in lists
|
|
845
|
-
- \`aria-checked\` - Checkboxes/toggles
|
|
846
|
-
- \`aria-disabled\` - Disabled elements
|
|
847
|
-
- \`aria-busy\` - Loading states
|
|
848
|
-
- \`data-state="loading|error|success"\` - Custom states
|
|
849
|
-
|
|
850
|
-
### 5. Form Accessibility
|
|
851
|
-
|
|
852
|
-
Forms should have proper labels and descriptions:
|
|
853
|
-
|
|
854
|
-
\`\`\`tsx
|
|
855
|
-
// Before
|
|
856
|
-
<input type="email" placeholder="Email" />
|
|
857
|
-
<span className="text-red-500">{error}</span>
|
|
858
|
-
|
|
859
|
-
// After
|
|
860
|
-
<input
|
|
861
|
-
type="email"
|
|
862
|
-
placeholder="Email"
|
|
863
|
-
aria-label="Email address"
|
|
864
|
-
aria-describedby={error ? "email-error" : undefined}
|
|
865
|
-
aria-invalid={!!error}
|
|
866
|
-
/>
|
|
867
|
-
{error && <span id="email-error" role="alert" className="text-red-500">{error}</span>}
|
|
868
|
-
\`\`\`
|
|
869
|
-
|
|
870
|
-
---
|
|
871
|
-
|
|
872
|
-
## Step 1: Scan for Missing Identifiers
|
|
873
|
-
|
|
874
|
-
**Start with buttons - these are the most critical:**
|
|
875
|
-
|
|
876
|
-
\`\`\`bash
|
|
877
|
-
# CRITICAL: Find ALL buttons missing data-testid (fix ALL of these!)
|
|
878
|
-
grep -rn "<button" src/ --include="*.tsx" | grep -v "data-testid" | head -30
|
|
879
|
-
|
|
880
|
-
# CRITICAL: Find ALL buttons missing aria-label (fix ALL of these!)
|
|
881
|
-
grep -rn "<button" src/ --include="*.tsx" | grep -v "aria-label" | head -30
|
|
882
|
-
|
|
883
|
-
# Find icon-only buttons (MUST have aria-label since no visible text)
|
|
884
|
-
grep -rn "<button.*Icon\\|<button.*>.*</.*Icon>" src/ --include="*.tsx" | head -20
|
|
885
|
-
|
|
886
|
-
# Find button wrapper components that need to pass through data-testid/aria-label
|
|
887
|
-
grep -rn "function.*Button\\|const.*Button.*=" src/ --include="*.tsx" | head -10
|
|
888
|
-
|
|
889
|
-
# Find modals/dialogs without role
|
|
890
|
-
grep -rn "modal\\|dialog\\|Modal\\|Dialog" src/ --include="*.tsx" | grep -v "role=" | head -20
|
|
891
|
-
|
|
892
|
-
# Find forms without proper labeling
|
|
893
|
-
grep -rn "<input\\|<select\\|<textarea" src/ --include="*.tsx" | grep -v "aria-label\\|id=" | head -20
|
|
894
|
-
|
|
895
|
-
# Find major components (likely need data-testid)
|
|
896
|
-
ls src/components/ src/pages/ 2>/dev/null
|
|
897
|
-
\`\`\`
|
|
898
|
-
|
|
899
|
-
**Every button found without \`data-testid\` or \`aria-label\` MUST be fixed.**
|
|
900
|
-
|
|
901
|
-
---
|
|
902
|
-
|
|
903
|
-
## Step 2: Prioritize by Impact
|
|
904
|
-
|
|
905
|
-
Focus on elements the verification system is most likely to need:
|
|
906
|
-
|
|
907
|
-
**High Priority (do first):**
|
|
908
|
-
1. Navigation elements (header, sidebar, menu buttons)
|
|
909
|
-
2. Primary actions (submit buttons, save buttons, CTAs)
|
|
910
|
-
3. Modal triggers and dialogs
|
|
911
|
-
4. Form inputs and submit buttons
|
|
912
|
-
5. Page-level containers
|
|
913
|
-
|
|
914
|
-
**Medium Priority:**
|
|
915
|
-
1. Toggle switches and checkboxes
|
|
916
|
-
2. Dropdown menus
|
|
917
|
-
3. Tab interfaces
|
|
918
|
-
4. Cards and list items
|
|
919
|
-
5. Loading/error states
|
|
920
|
-
|
|
921
|
-
**Lower Priority:**
|
|
922
|
-
1. Decorative elements
|
|
923
|
-
2. Static content sections
|
|
924
|
-
3. Footer links
|
|
925
|
-
|
|
926
|
-
---
|
|
927
|
-
|
|
928
|
-
## Step 3: Add Identifiers Systematically
|
|
929
|
-
|
|
930
|
-
Go component by component. For each component:
|
|
931
|
-
|
|
932
|
-
1. **Check the component's purpose** - What does it DO?
|
|
933
|
-
2. **Add aria-label** to interactive elements describing the ACTION
|
|
934
|
-
3. **Add data-testid** to the container if it's a major section
|
|
935
|
-
4. **Add role** if the semantic HTML isn't clear
|
|
936
|
-
5. **Add state attributes** if the component has dynamic states
|
|
937
|
-
|
|
938
|
-
### Naming Conventions
|
|
939
|
-
|
|
940
|
-
**aria-label**: Describe the action, not the element
|
|
941
|
-
- ✅ \`aria-label="Close modal"\`
|
|
942
|
-
- ✅ \`aria-label="Submit contact form"\`
|
|
943
|
-
- ❌ \`aria-label="Button"\`
|
|
944
|
-
- ❌ \`aria-label="Click here"\`
|
|
945
|
-
|
|
946
|
-
**data-testid**: Use kebab-case, describe the section
|
|
947
|
-
- ✅ \`data-testid="user-profile-card"\`
|
|
948
|
-
- ✅ \`data-testid="search-results-list"\`
|
|
949
|
-
- ❌ \`data-testid="div1"\`
|
|
950
|
-
- ❌ \`data-testid="container"\`
|
|
951
|
-
|
|
952
|
-
---
|
|
953
|
-
|
|
954
|
-
## Step 4: Verify Coverage
|
|
955
|
-
|
|
956
|
-
After adding identifiers, check coverage:
|
|
957
|
-
|
|
958
|
-
\`\`\`bash
|
|
959
|
-
# Count aria-labels added
|
|
960
|
-
grep -r "aria-label" src/ --include="*.tsx" | wc -l
|
|
961
|
-
|
|
962
|
-
# Count data-testid added
|
|
963
|
-
grep -r "data-testid" src/ --include="*.tsx" | wc -l
|
|
964
|
-
|
|
965
|
-
# Count role attributes
|
|
966
|
-
grep -r "role=" src/ --include="*.tsx" | wc -l
|
|
967
|
-
|
|
968
|
-
# List all data-testid values (check for meaningful names)
|
|
969
|
-
grep -oh 'data-testid="[^"]*"' src/ -r --include="*.tsx" | sort -u
|
|
970
|
-
\`\`\`
|
|
971
|
-
|
|
972
|
-
---
|
|
973
|
-
|
|
974
|
-
## Step 5: Commit
|
|
975
|
-
|
|
976
|
-
\`\`\`bash
|
|
977
|
-
git add src/
|
|
978
|
-
git commit -m "Add accessibility attributes for verification
|
|
979
|
-
|
|
980
|
-
- Added aria-labels to interactive elements
|
|
981
|
-
- Added data-testid to major sections
|
|
982
|
-
- Added ARIA roles for semantic structure
|
|
983
|
-
- Added state indicators (aria-expanded, etc.)"
|
|
984
|
-
\`\`\`
|
|
985
|
-
|
|
986
|
-
---
|
|
987
|
-
|
|
988
|
-
## Quick Reference: Common Patterns
|
|
989
|
-
|
|
990
|
-
### Icon Button
|
|
991
|
-
\`\`\`tsx
|
|
992
|
-
<button onClick={onAction} aria-label="Descriptive action name">
|
|
993
|
-
<Icon />
|
|
994
|
-
</button>
|
|
995
|
-
\`\`\`
|
|
996
|
-
|
|
997
|
-
### Modal
|
|
998
|
-
\`\`\`tsx
|
|
999
|
-
<div role="dialog" aria-modal="true" aria-labelledby="modal-title" data-testid="settings-modal">
|
|
1000
|
-
<h2 id="modal-title">Settings</h2>
|
|
1001
|
-
<button onClick={onClose} aria-label="Close settings">×</button>
|
|
1002
|
-
</div>
|
|
1003
|
-
\`\`\`
|
|
1004
|
-
|
|
1005
|
-
### Navigation
|
|
1006
|
-
\`\`\`tsx
|
|
1007
|
-
<nav aria-label="Main navigation" data-testid="main-nav">
|
|
1008
|
-
<a href="/dashboard" aria-current={isActive ? "page" : undefined}>Dashboard</a>
|
|
1009
|
-
</nav>
|
|
1010
|
-
\`\`\`
|
|
1011
|
-
|
|
1012
|
-
### Toggle
|
|
1013
|
-
\`\`\`tsx
|
|
1014
|
-
<button
|
|
1015
|
-
onClick={toggle}
|
|
1016
|
-
aria-pressed={isOn}
|
|
1017
|
-
aria-label={\`\${isOn ? 'Disable' : 'Enable'} notifications\`}
|
|
1018
|
-
>
|
|
1019
|
-
{isOn ? 'On' : 'Off'}
|
|
1020
|
-
</button>
|
|
1021
|
-
\`\`\`
|
|
1022
|
-
|
|
1023
|
-
### Loading State
|
|
1024
|
-
\`\`\`tsx
|
|
1025
|
-
<div data-testid="content-area" aria-busy={isLoading}>
|
|
1026
|
-
{isLoading ? (
|
|
1027
|
-
<div role="status" aria-label="Loading content">
|
|
1028
|
-
<Spinner />
|
|
1029
|
-
</div>
|
|
1030
|
-
) : (
|
|
1031
|
-
content
|
|
1032
|
-
)}
|
|
1033
|
-
</div>
|
|
1034
|
-
\`\`\`
|
|
1035
|
-
|
|
1036
|
-
### Form Field
|
|
1037
|
-
\`\`\`tsx
|
|
1038
|
-
<div data-testid="email-field">
|
|
1039
|
-
<label htmlFor="email">Email</label>
|
|
1040
|
-
<input
|
|
1041
|
-
id="email"
|
|
1042
|
-
type="email"
|
|
1043
|
-
aria-describedby={error ? "email-error" : "email-hint"}
|
|
1044
|
-
aria-invalid={!!error}
|
|
1045
|
-
/>
|
|
1046
|
-
<span id="email-hint">We'll never share your email</span>
|
|
1047
|
-
{error && <span id="email-error" role="alert">{error}</span>}
|
|
1048
|
-
</div>
|
|
1049
|
-
\`\`\`
|
|
1050
|
-
|
|
1051
|
-
### Expandable Section
|
|
1052
|
-
\`\`\`tsx
|
|
1053
|
-
<div data-testid="faq-section">
|
|
1054
|
-
<button
|
|
1055
|
-
onClick={() => setExpanded(!expanded)}
|
|
1056
|
-
aria-expanded={expanded}
|
|
1057
|
-
aria-controls="faq-content"
|
|
1058
|
-
>
|
|
1059
|
-
FAQ
|
|
1060
|
-
</button>
|
|
1061
|
-
<div id="faq-content" hidden={!expanded}>
|
|
1062
|
-
{content}
|
|
1063
|
-
</div>
|
|
1064
|
-
</div>
|
|
1065
|
-
\`\`\`
|
|
1066
|
-
`;
|
|
1067
|
-
const PREPARE_HAYSTACK_COMMAND = `# Prepare Codebase for Verification
|
|
1068
|
-
|
|
1069
|
-
Follow .agents/skills/prepare-haystack.md to add accessibility attributes that make verification easier.
|
|
1070
|
-
|
|
1071
|
-
Run this BEFORE /setup-haystack to ensure your codebase has good selectors.
|
|
1072
|
-
`;
|
|
1073
|
-
const SECRETS_COMMAND_CONTENT = `# Set Up Haystack Secrets
|
|
1074
|
-
|
|
1075
|
-
Follow .agents/skills/setup-haystack-secrets.md to configure secrets (API keys, credentials) for Haystack verification.
|
|
1076
|
-
`;
|
|
1077
|
-
const SECRETS_SKILL_CONTENT = `# Set Up Haystack Secrets
|
|
1078
|
-
|
|
1079
|
-
**Your job**: Help the user configure secrets needed for Haystack verification, especially LLM API keys for backend services.
|
|
1080
|
-
|
|
1081
|
-
---
|
|
1082
|
-
|
|
1083
|
-
## Step 1: Detect Services That Need Secrets
|
|
1084
|
-
|
|
1085
|
-
Scan the codebase to identify services that might need API keys or credentials:
|
|
1086
|
-
|
|
1087
|
-
\`\`\`bash
|
|
1088
|
-
# Find LLM/AI SDK usage
|
|
1089
|
-
grep -rn "openai\\|anthropic\\|bedrock\\|OPENAI_API_KEY\\|ANTHROPIC_API_KEY" --include="*.ts" --include="*.py" --include="*.js" | grep -v node_modules | head -20
|
|
1090
|
-
|
|
1091
|
-
# Find environment variable references
|
|
1092
|
-
grep -rn "process.env\\.\\|os.environ\\|env\\." --include="*.ts" --include="*.py" --include="*.js" | grep -v node_modules | grep -iE "key|secret|token|api" | head -20
|
|
1093
|
-
|
|
1094
|
-
# Check for .env.example or similar
|
|
1095
|
-
cat .env.example .env.sample 2>/dev/null || echo "No .env example found"
|
|
1096
|
-
|
|
1097
|
-
# Check existing secrets in .haystack.json
|
|
1098
|
-
grep -A10 '"secrets"' .haystack.json 2>/dev/null || echo "No secrets configured yet"
|
|
1099
|
-
\`\`\`
|
|
1100
|
-
|
|
1101
|
-
---
|
|
1102
|
-
|
|
1103
|
-
## Step 2: Categorize the Secrets Found
|
|
1104
|
-
|
|
1105
|
-
Group secrets by type and service:
|
|
1106
|
-
|
|
1107
|
-
| Category | Examples | Storage Recommendation |
|
|
1108
|
-
|----------|----------|------------------------|
|
|
1109
|
-
| **LLM API Keys** | \`OPENAI_API_KEY\`, \`ANTHROPIC_API_KEY\` | Haystack Secrets (encrypted) |
|
|
1110
|
-
| **Cloud Credentials** | \`AWS_ACCESS_KEY_ID\`, \`GCP_SERVICE_ACCOUNT\` | Use OIDC in CI, not static keys |
|
|
1111
|
-
| **Database** | \`DATABASE_URL\`, \`REDIS_URL\` | Haystack Secrets or passthrough |
|
|
1112
|
-
| **Third-party APIs** | \`STRIPE_KEY\`, \`SENDGRID_KEY\` | Haystack Secrets |
|
|
1113
|
-
| **Internal Services** | \`API_TOKEN\`, \`WEBHOOK_SECRET\` | Haystack Secrets |
|
|
1114
|
-
|
|
1115
|
-
---
|
|
1116
|
-
|
|
1117
|
-
## STOP - Present Findings and Ask User
|
|
1118
|
-
|
|
1119
|
-
**You MUST present what you found and ask the user before proceeding:**
|
|
1120
|
-
|
|
1121
|
-
> I scanned the codebase and found these services that may need secrets for verification:
|
|
1122
|
-
>
|
|
1123
|
-
> **Services detected:**
|
|
1124
|
-
> - [Service 1]: Uses \`OPENAI_API_KEY\` for [purpose]
|
|
1125
|
-
> - [Service 2]: Uses \`DATABASE_URL\` for [purpose]
|
|
1126
|
-
> - ...
|
|
1127
|
-
>
|
|
1128
|
-
> **Questions:**
|
|
1129
|
-
>
|
|
1130
|
-
> 1. **LLM API Keys**: Which provider do you use?
|
|
1131
|
-
> - OpenAI (\`OPENAI_API_KEY\`)
|
|
1132
|
-
> - Anthropic (\`ANTHROPIC_API_KEY\`)
|
|
1133
|
-
> - AWS Bedrock (use OIDC, no static keys needed)
|
|
1134
|
-
> - Other: ___
|
|
1135
|
-
>
|
|
1136
|
-
> 2. **For each secret**, should verification use:
|
|
1137
|
-
> - **Real credentials** (stored securely in Haystack Secrets)
|
|
1138
|
-
> - **Test/sandbox credentials** (separate keys for verification only)
|
|
1139
|
-
> - **Mock/skip** (service not needed for visual verification)
|
|
1140
|
-
>
|
|
1141
|
-
> 3. **Do you have separate test credentials?**
|
|
1142
|
-
> - Yes, I have sandbox/test API keys
|
|
1143
|
-
> - No, I'll use production keys (with usage limits)
|
|
1144
|
-
> - I need to create test credentials first
|
|
1145
|
-
|
|
1146
|
-
**Wait for the user's response before continuing.**
|
|
1147
|
-
|
|
1148
|
-
---
|
|
1149
|
-
|
|
1150
|
-
## Step 3: Configure Secrets in .haystack.json
|
|
1151
|
-
|
|
1152
|
-
Based on user's answers, add secrets to the config:
|
|
1153
|
-
|
|
1154
|
-
\`\`\`json
|
|
1155
|
-
{
|
|
1156
|
-
"secrets": {
|
|
1157
|
-
"OPENAI_API_KEY": {
|
|
1158
|
-
"description": "OpenAI API key for analysis service",
|
|
1159
|
-
"required": true,
|
|
1160
|
-
"services": ["analysis"]
|
|
1161
|
-
},
|
|
1162
|
-
"DATABASE_URL": {
|
|
1163
|
-
"description": "PostgreSQL connection string",
|
|
1164
|
-
"required": false,
|
|
1165
|
-
"services": ["api"]
|
|
1166
|
-
}
|
|
1167
|
-
}
|
|
1168
|
-
}
|
|
1169
|
-
\`\`\`
|
|
1170
|
-
|
|
1171
|
-
### Secret Properties
|
|
1172
|
-
|
|
1173
|
-
| Property | Description |
|
|
1174
|
-
|----------|-------------|
|
|
1175
|
-
| \`description\` | What this secret is for (shown to user) |
|
|
1176
|
-
| \`required\` | If \`true\`, verification fails without it |
|
|
1177
|
-
| \`services\` | Which services need this secret (for scoping) |
|
|
1178
|
-
|
|
1179
|
-
---
|
|
1180
|
-
|
|
1181
|
-
## Step 4: Guide User to Store Secrets
|
|
1182
|
-
|
|
1183
|
-
### Option A: Haystack CLI (Recommended)
|
|
1184
|
-
|
|
1185
|
-
\`\`\`bash
|
|
1186
|
-
# Store secrets securely (encrypted at rest)
|
|
1187
|
-
npx @haystackeditor/cli secrets set OPENAI_API_KEY
|
|
1188
|
-
# Prompts for value, never shown in terminal history
|
|
1189
|
-
|
|
1190
|
-
# Verify it's stored
|
|
1191
|
-
npx @haystackeditor/cli secrets list
|
|
1192
|
-
\`\`\`
|
|
1193
|
-
|
|
1194
|
-
### Option B: Environment Variables (Local Development)
|
|
1195
|
-
|
|
1196
|
-
\`\`\`bash
|
|
1197
|
-
# Add to .env.local (gitignored)
|
|
1198
|
-
echo 'OPENAI_API_KEY=sk-...' >> .env.local
|
|
1199
|
-
|
|
1200
|
-
# Make sure .env.local is in .gitignore
|
|
1201
|
-
grep -q '.env.local' .gitignore || echo '.env.local' >> .gitignore
|
|
1202
|
-
\`\`\`
|
|
1203
|
-
|
|
1204
|
-
### Option C: CI/CD Secrets (GitHub Actions)
|
|
1205
|
-
|
|
1206
|
-
\`\`\`yaml
|
|
1207
|
-
# .github/workflows/haystack.yml
|
|
1208
|
-
env:
|
|
1209
|
-
OPENAI_API_KEY: \${{ secrets.OPENAI_API_KEY }}
|
|
1210
|
-
\`\`\`
|
|
1211
|
-
|
|
1212
|
-
---
|
|
1213
|
-
|
|
1214
|
-
## Step 5: Handle Cloud Provider Credentials
|
|
1215
|
-
|
|
1216
|
-
**Never store long-lived cloud credentials.** Use OIDC instead:
|
|
1217
|
-
|
|
1218
|
-
### AWS (Bedrock, S3, etc.)
|
|
1219
|
-
|
|
1220
|
-
\`\`\`yaml
|
|
1221
|
-
# GitHub Actions with OIDC - no secrets needed!
|
|
1222
|
-
jobs:
|
|
1223
|
-
verify:
|
|
1224
|
-
permissions:
|
|
1225
|
-
id-token: write
|
|
1226
|
-
contents: read
|
|
1227
|
-
steps:
|
|
1228
|
-
- uses: aws-actions/configure-aws-credentials@v4
|
|
1229
|
-
with:
|
|
1230
|
-
role-to-assume: arn:aws:iam::123456789:role/haystack-verify
|
|
1231
|
-
aws-region: us-west-2
|
|
1232
|
-
\`\`\`
|
|
1233
|
-
|
|
1234
|
-
### GCP (Vertex AI, Cloud Storage, etc.)
|
|
1235
|
-
|
|
1236
|
-
\`\`\`yaml
|
|
1237
|
-
jobs:
|
|
1238
|
-
verify:
|
|
1239
|
-
permissions:
|
|
1240
|
-
id-token: write
|
|
1241
|
-
steps:
|
|
1242
|
-
- uses: google-github-actions/auth@v2
|
|
1243
|
-
with:
|
|
1244
|
-
workload_identity_provider: projects/123/locations/global/...
|
|
1245
|
-
service_account: haystack-verify@project.iam.gserviceaccount.com
|
|
1246
|
-
\`\`\`
|
|
1247
|
-
|
|
1248
|
-
---
|
|
1249
|
-
|
|
1250
|
-
## Step 6: Test Secret Access
|
|
1251
|
-
|
|
1252
|
-
After storing secrets, verify they're accessible:
|
|
1253
|
-
|
|
1254
|
-
\`\`\`bash
|
|
1255
|
-
# Test locally
|
|
1256
|
-
npx @haystackeditor/cli secrets test
|
|
1257
|
-
|
|
1258
|
-
# Or check manually
|
|
1259
|
-
npx @haystackeditor/cli secrets get OPENAI_API_KEY --masked
|
|
1260
|
-
# Shows: sk-...xxxx (last 4 chars only)
|
|
1261
|
-
\`\`\`
|
|
1262
|
-
|
|
1263
|
-
---
|
|
1264
|
-
|
|
1265
|
-
## Step 7: Update .haystack.json
|
|
1266
|
-
|
|
1267
|
-
Add the secrets configuration:
|
|
1268
|
-
|
|
1269
|
-
\`\`\`bash
|
|
1270
|
-
# Show what to add
|
|
1271
|
-
cat << 'EOF'
|
|
1272
|
-
Add this to your .haystack.json:
|
|
1273
|
-
|
|
1274
|
-
{
|
|
1275
|
-
"secrets": {
|
|
1276
|
-
"OPENAI_API_KEY": {
|
|
1277
|
-
"description": "OpenAI API key for LLM analysis",
|
|
1278
|
-
"required": true
|
|
1279
|
-
}
|
|
1280
|
-
}
|
|
1281
|
-
}
|
|
1282
|
-
EOF
|
|
1283
|
-
\`\`\`
|
|
1284
|
-
|
|
1285
|
-
---
|
|
1286
|
-
|
|
1287
|
-
## Common Patterns
|
|
1288
|
-
|
|
1289
|
-
### Analysis Pipeline with LLM
|
|
1290
|
-
|
|
1291
|
-
\`\`\`json
|
|
1292
|
-
{
|
|
1293
|
-
"services": {
|
|
1294
|
-
"analysis": {
|
|
1295
|
-
"root": "packages/analysis",
|
|
1296
|
-
"command": "pnpm start",
|
|
1297
|
-
"env": {
|
|
1298
|
-
"PR_IDENTIFIER": "$PR_IDENTIFIER"
|
|
1299
|
-
}
|
|
1300
|
-
}
|
|
1301
|
-
},
|
|
1302
|
-
"secrets": {
|
|
1303
|
-
"OPENAI_API_KEY": {
|
|
1304
|
-
"description": "OpenAI API for code analysis",
|
|
1305
|
-
"required": true,
|
|
1306
|
-
"services": ["analysis"]
|
|
1307
|
-
}
|
|
1308
|
-
}
|
|
1309
|
-
}
|
|
1310
|
-
\`\`\`
|
|
1311
|
-
|
|
1312
|
-
### Multiple LLM Providers
|
|
1313
|
-
|
|
1314
|
-
\`\`\`json
|
|
1315
|
-
{
|
|
1316
|
-
"secrets": {
|
|
1317
|
-
"OPENAI_API_KEY": {
|
|
1318
|
-
"description": "OpenAI for embeddings",
|
|
1319
|
-
"services": ["search"]
|
|
1320
|
-
},
|
|
1321
|
-
"ANTHROPIC_API_KEY": {
|
|
1322
|
-
"description": "Claude for code review",
|
|
1323
|
-
"services": ["review"]
|
|
1324
|
-
}
|
|
1325
|
-
}
|
|
1326
|
-
}
|
|
1327
|
-
\`\`\`
|
|
1328
|
-
|
|
1329
|
-
### Database + API Keys
|
|
1330
|
-
|
|
1331
|
-
\`\`\`json
|
|
1332
|
-
{
|
|
1333
|
-
"secrets": {
|
|
1334
|
-
"DATABASE_URL": {
|
|
1335
|
-
"description": "PostgreSQL for test database",
|
|
1336
|
-
"required": true
|
|
1337
|
-
},
|
|
1338
|
-
"STRIPE_TEST_KEY": {
|
|
1339
|
-
"description": "Stripe test mode key",
|
|
1340
|
-
"required": false
|
|
1341
|
-
}
|
|
1342
|
-
}
|
|
1343
|
-
}
|
|
1344
|
-
\`\`\`
|
|
1345
|
-
|
|
1346
|
-
---
|
|
1347
|
-
|
|
1348
|
-
## Security Best Practices
|
|
1349
|
-
|
|
1350
|
-
1. **Use test/sandbox credentials** when available (OpenAI, Stripe, etc. offer these)
|
|
1351
|
-
2. **Set usage limits** on API keys used for verification
|
|
1352
|
-
3. **Never commit secrets** - use \`.gitignore\` for \`.env.local\`
|
|
1353
|
-
4. **Rotate regularly** - especially if verification runs on external PRs
|
|
1354
|
-
5. **Scope narrowly** - use \`services\` array to limit which services see which secrets
|
|
1355
|
-
6. **Prefer OIDC** for cloud providers instead of static credentials
|
|
1356
|
-
|
|
1357
|
-
---
|
|
1358
|
-
|
|
1359
|
-
## Troubleshooting
|
|
1360
|
-
|
|
1361
|
-
### "Secret not found" errors
|
|
1362
|
-
|
|
1363
|
-
\`\`\`bash
|
|
1364
|
-
# Check if secret is stored
|
|
1365
|
-
npx @haystackeditor/cli secrets list
|
|
1366
|
-
|
|
1367
|
-
# Check if it's in the right scope
|
|
1368
|
-
npx @haystackeditor/cli secrets get SECRET_NAME --show-metadata
|
|
1369
|
-
\`\`\`
|
|
1370
|
-
|
|
1371
|
-
### "Permission denied" for cloud resources
|
|
1372
|
-
|
|
1373
|
-
- Verify OIDC trust policy includes your repo
|
|
1374
|
-
- Check IAM role has required permissions
|
|
1375
|
-
- Ensure GitHub Actions has \`id-token: write\` permission
|
|
1376
|
-
|
|
1377
|
-
### Secrets work locally but not in CI
|
|
1378
|
-
|
|
1379
|
-
- Secrets stored via CLI are user-scoped by default
|
|
1380
|
-
- For CI, use GitHub Secrets or organization-level Haystack secrets
|
|
1381
|
-
- Check \`npx @haystackeditor/cli secrets list --scope=org\`
|
|
1382
|
-
`;
|
|
1383
|
-
const SETUP_RULES_CONTENT = `# Set Up Review Policies & Rules
|
|
1384
|
-
|
|
1385
|
-
**Your job**: Help the user configure two systems that control how Haystack evaluates their PRs:
|
|
1386
|
-
|
|
1387
|
-
1. **Review Policies** — deterministic, path-based triggers that **always** require human review
|
|
1388
|
-
2. **PR Rules** — LLM-evaluated code invariants that **flag violations** in the diff
|
|
1389
|
-
|
|
1390
|
-
These are distinct systems. Make sure the user understands the difference before proceeding.
|
|
1391
|
-
|
|
1392
|
-
---
|
|
1393
|
-
|
|
1394
|
-
## The Two Systems
|
|
1395
|
-
|
|
1396
|
-
### Review Policies (\`.haystack/review-policy.md\`)
|
|
1397
|
-
|
|
1398
|
-
**"If a PR touches X → human review needed, period."**
|
|
1399
|
-
|
|
1400
|
-
Policies are simple glob patterns. When a PR changes files matching a policy's paths, Haystack will always recommend human review for that PR, regardless of what the code changes look like.
|
|
1401
|
-
|
|
1402
|
-
**Good candidates for policies:**
|
|
1403
|
-
- Infrastructure (Terraform, Kubernetes, CI/CD pipelines)
|
|
1404
|
-
- Authentication and authorization code
|
|
1405
|
-
- Database migrations and schemas
|
|
1406
|
-
- API contracts (OpenAPI specs, GraphQL schemas, protobuf)
|
|
1407
|
-
- Security-sensitive code (crypto, secrets handling)
|
|
1408
|
-
- Billing/payment logic
|
|
1409
|
-
- Code that bypasses feature flags or launches directly to production
|
|
1410
|
-
|
|
1411
|
-
**Format:**
|
|
1412
|
-
\`\`\`markdown
|
|
1413
|
-
## Policy name
|
|
1414
|
-
- **Paths**: \\\`glob/pattern/**\\\`, \\\`other/pattern/*.ts\\\`
|
|
1415
|
-
- **Severity**: critical | high | medium | low
|
|
1416
|
-
- **Reason**: Why this always needs human eyes
|
|
1417
|
-
\`\`\`
|
|
1418
|
-
|
|
1419
|
-
### PR Rules (\`.haystack/pr-rules.yml\`)
|
|
1420
|
-
|
|
1421
|
-
**"If this invariant is violated in the diff → flag it."**
|
|
1422
|
-
|
|
1423
|
-
Rules are checked by an LLM that reads the PR diff and looks for violations. They catch patterns that shouldn't appear in new code, regardless of which files are touched.
|
|
1424
|
-
|
|
1425
|
-
**Good candidates for rules:**
|
|
1426
|
-
- Error handling conventions (no silent catches, no swallowed errors)
|
|
1427
|
-
- Logging requirements (all API calls must be logged)
|
|
1428
|
-
- Security patterns (no hardcoded secrets, no SQL string concatenation)
|
|
1429
|
-
- Code style invariants (no TODOs without issue refs, no console.log in production)
|
|
1430
|
-
- Architecture boundaries (service A must not import from service B)
|
|
1431
|
-
- Naming conventions specific to your team
|
|
1432
|
-
|
|
1433
|
-
**Format:**
|
|
1434
|
-
\`\`\`yaml
|
|
1435
|
-
- id: PR001
|
|
1436
|
-
name: Short rule name
|
|
1437
|
-
type: llm
|
|
1438
|
-
severity: warning | error
|
|
1439
|
-
message: Human-readable description of the violation
|
|
1440
|
-
llm:
|
|
1441
|
-
prompt: >
|
|
1442
|
-
Detailed instructions for the LLM about what to look for
|
|
1443
|
-
in the PR diff. Be specific about what counts as a violation
|
|
1444
|
-
and what doesn't.
|
|
1445
|
-
files: "optional/glob/pattern/**/*.ts" # optional file scope
|
|
1446
|
-
\`\`\`
|
|
1447
|
-
|
|
1448
|
-
---
|
|
1449
|
-
|
|
1450
|
-
## Step 1: Explore the Codebase
|
|
1451
|
-
|
|
1452
|
-
Before suggesting anything, understand the repo structure:
|
|
1453
|
-
|
|
1454
|
-
\`\`\`bash
|
|
1455
|
-
# Understand repo layout
|
|
1456
|
-
ls -la
|
|
1457
|
-
find . -maxdepth 2 -type d -not -path '*/node_modules/*' -not -path '*/.git/*' | head -40
|
|
1458
|
-
|
|
1459
|
-
# Find infrastructure files
|
|
1460
|
-
find . -name "*.tf" -o -name "Dockerfile*" -o -name "docker-compose*" -o -name "*.yml" -path "*/.github/*" | head -20
|
|
1461
|
-
|
|
1462
|
-
# Find auth-related code
|
|
1463
|
-
grep -rl "auth\\|login\\|session\\|token\\|credential" --include="*.ts" --include="*.py" --include="*.go" --include="*.js" -l | grep -v node_modules | head -20
|
|
1464
|
-
|
|
1465
|
-
# Find database/migration files
|
|
1466
|
-
find . -path "*/migration*" -o -path "*/schema*" -o -name "*.sql" | grep -v node_modules | head -20
|
|
1467
|
-
|
|
1468
|
-
# Find API contract files
|
|
1469
|
-
find . -name "*.proto" -o -name "openapi*" -o -name "swagger*" -o -name "*.graphql" | head -20
|
|
1470
|
-
|
|
1471
|
-
# Find feature flag patterns
|
|
1472
|
-
grep -rl "feature.?flag\\|feature.?toggle\\|isEnabled\\|FEATURE_" --include="*.ts" --include="*.py" --include="*.go" --include="*.js" -l | grep -v node_modules | head -20
|
|
1473
|
-
|
|
1474
|
-
# Check for existing config
|
|
1475
|
-
cat .haystack/review-policy.md 2>/dev/null || echo "No review-policy.md yet"
|
|
1476
|
-
cat .haystack/pr-rules.yml 2>/dev/null || echo "No pr-rules.yml yet"
|
|
1477
|
-
\`\`\`
|
|
1478
|
-
|
|
1479
|
-
Also look for:
|
|
1480
|
-
- CI/CD pipeline files
|
|
1481
|
-
- Payment or billing directories
|
|
1482
|
-
- Secrets or credential management code
|
|
1483
|
-
- Environment configuration patterns
|
|
1484
|
-
- Key shared libraries or core utilities that many things depend on
|
|
1485
|
-
|
|
1486
|
-
---
|
|
1487
|
-
|
|
1488
|
-
## STOP — Propose Review Policies
|
|
1489
|
-
|
|
1490
|
-
Present your findings to the user. Be specific about what you found and why each area warrants a policy.
|
|
1491
|
-
|
|
1492
|
-
**Format your proposal like this:**
|
|
1493
|
-
|
|
1494
|
-
> Based on the codebase, here are the areas I'd recommend always requiring human review:
|
|
1495
|
-
>
|
|
1496
|
-
> | # | Policy | Paths | Severity | Why |
|
|
1497
|
-
> |---|--------|-------|----------|-----|
|
|
1498
|
-
> | 1 | Infrastructure | \\\`terraform/**\\\`, \\\`*.tf\\\` | high | Cost and security implications |
|
|
1499
|
-
> | 2 | Auth & secrets | \\\`src/auth/**\\\`, \\\`**/*.secret*\\\` | critical | Security-sensitive |
|
|
1500
|
-
> | 3 | ... | ... | ... | ... |
|
|
1501
|
-
>
|
|
1502
|
-
> **Notes:**
|
|
1503
|
-
> - I didn't find any database migrations in this repo, so I'm not including that
|
|
1504
|
-
> - The \`src/core/\` directory is imported by everything — should changes there always need review?
|
|
1505
|
-
>
|
|
1506
|
-
> Which of these should I include? Any to add, remove, or modify?
|
|
1507
|
-
|
|
1508
|
-
**Wait for the user to confirm before writing anything.**
|
|
1509
|
-
|
|
1510
|
-
---
|
|
1511
|
-
|
|
1512
|
-
## Step 2: Write review-policy.md
|
|
1513
|
-
|
|
1514
|
-
Once the user confirms, write the policies to \`.haystack/review-policy.md\`:
|
|
1515
|
-
|
|
1516
|
-
\`\`\`markdown
|
|
1517
|
-
# Review Policies
|
|
1518
|
-
|
|
1519
|
-
Review policies define file patterns that always require human attention. When a PR touches files matching these patterns, Haystack will flag it for human review.
|
|
1520
|
-
|
|
1521
|
-
## [Policy name]
|
|
1522
|
-
- **Paths**: \\\`pattern/**\\\`
|
|
1523
|
-
- **Severity**: [level]
|
|
1524
|
-
- **Reason**: [why]
|
|
1525
|
-
\`\`\`
|
|
1526
|
-
|
|
1527
|
-
Create the \`.haystack/\` directory if it doesn't exist.
|
|
1528
|
-
|
|
1529
|
-
---
|
|
1530
|
-
|
|
1531
|
-
## Step 3: Understand Team Conventions
|
|
1532
|
-
|
|
1533
|
-
Now shift to rules. Ask the user about their code quality invariants:
|
|
1534
|
-
|
|
1535
|
-
> Now let's set up **PR rules** — these are code invariants that Haystack checks in every diff.
|
|
1536
|
-
>
|
|
1537
|
-
> Unlike policies (which trigger on file paths), rules look at *what the code does*. For example:
|
|
1538
|
-
> - "Don't add empty catch blocks"
|
|
1539
|
-
> - "Don't add TODOs without issue references"
|
|
1540
|
-
> - "Don't hardcode API URLs"
|
|
1541
|
-
>
|
|
1542
|
-
> **What conventions does your team care about?** Some common categories:
|
|
1543
|
-
>
|
|
1544
|
-
> 1. **Error handling** — e.g., no swallowed errors, no silent fallbacks
|
|
1545
|
-
> 2. **Security** — e.g., no hardcoded secrets, no raw SQL strings
|
|
1546
|
-
> 3. **Logging** — e.g., all API calls must have logging
|
|
1547
|
-
> 4. **Code hygiene** — e.g., no TODOs without issues, no console.log in production
|
|
1548
|
-
> 5. **Architecture** — e.g., no cross-service imports, no direct DB access from UI layer
|
|
1549
|
-
> 6. **Testing** — e.g., no skipped tests without reason, no snapshot-only tests
|
|
1550
|
-
>
|
|
1551
|
-
> Tell me which categories matter to your team, or describe your own conventions.
|
|
1552
|
-
|
|
1553
|
-
Also explore the codebase for clues:
|
|
1554
|
-
|
|
1555
|
-
\`\`\`bash
|
|
1556
|
-
# Check for linter configs (reveals team conventions)
|
|
1557
|
-
cat .eslintrc* .eslintrc.json .eslintrc.js eslint.config.* 2>/dev/null | head -50
|
|
1558
|
-
|
|
1559
|
-
# Check for existing code review guidelines
|
|
1560
|
-
find . -name "CONTRIBUTING*" -o -name "REVIEW*" -o -name "STYLE*" | head -5
|
|
1561
|
-
|
|
1562
|
-
# Look for common anti-patterns
|
|
1563
|
-
grep -rn "catch.*{\\s*}" --include="*.ts" --include="*.js" | head -5
|
|
1564
|
-
grep -rn "console\\.log" --include="*.ts" --include="*.tsx" | grep -v test | head -5
|
|
1565
|
-
grep -rn "TODO\\|FIXME\\|HACK\\|XXX" --include="*.ts" --include="*.tsx" | head -10
|
|
1566
|
-
\`\`\`
|
|
1567
|
-
|
|
1568
|
-
---
|
|
1569
|
-
|
|
1570
|
-
## STOP — Propose PR Rules
|
|
1571
|
-
|
|
1572
|
-
Based on the user's input and your codebase exploration, propose 2-5 custom rules.
|
|
1573
|
-
|
|
1574
|
-
**Format your proposal like this:**
|
|
1575
|
-
|
|
1576
|
-
> Here are the PR rules I'd suggest:
|
|
1577
|
-
>
|
|
1578
|
-
> **1. PR001 — No silent error handling** (warning)
|
|
1579
|
-
> > Flag: Adding catch blocks that swallow errors without logging or re-throwing
|
|
1580
|
-
>
|
|
1581
|
-
> **2. PR002 — No TODOs without tracking issues** (warning)
|
|
1582
|
-
> > Flag: Adding TODO/FIXME comments without referencing an issue number
|
|
1583
|
-
>
|
|
1584
|
-
> **3. PR003 — No hardcoded API endpoints** (warning)
|
|
1585
|
-
> > Flag: Adding hardcoded URLs or API endpoints instead of using config/env vars
|
|
1586
|
-
>
|
|
1587
|
-
> Should I add, remove, or modify any of these?
|
|
1588
|
-
|
|
1589
|
-
**Wait for the user to confirm before writing.**
|
|
1590
|
-
|
|
1591
|
-
---
|
|
1592
|
-
|
|
1593
|
-
## Step 4: Write pr-rules.yml
|
|
1594
|
-
|
|
1595
|
-
Once confirmed, write to \`.haystack/pr-rules.yml\`:
|
|
1596
|
-
|
|
1597
|
-
\`\`\`yaml
|
|
1598
|
-
version: 1
|
|
1599
|
-
|
|
1600
|
-
rules:
|
|
1601
|
-
- id: PR001
|
|
1602
|
-
name: [Short name]
|
|
1603
|
-
type: llm
|
|
1604
|
-
severity: warning
|
|
1605
|
-
message: [Human-readable description]
|
|
1606
|
-
llm:
|
|
1607
|
-
prompt: >
|
|
1608
|
-
[Detailed LLM instructions — be specific about what counts as
|
|
1609
|
-
a violation and what doesn't. Include examples of violations
|
|
1610
|
-
and non-violations when helpful.]
|
|
1611
|
-
\`\`\`
|
|
1612
|
-
|
|
1613
|
-
**Tips for writing good rule prompts:**
|
|
1614
|
-
- Be specific about what counts as a violation
|
|
1615
|
-
- Mention common false positives to ignore (e.g., "test files are exempt")
|
|
1616
|
-
- Use the \`files\` field to scope rules to relevant file types
|
|
1617
|
-
- \`severity: error\` caps the Haystack rating at 3 (needs review); \`warning\` is advisory
|
|
1618
|
-
|
|
1619
|
-
---
|
|
1620
|
-
|
|
1621
|
-
## Step 5: Commit
|
|
1622
|
-
|
|
1623
|
-
\`\`\`bash
|
|
1624
|
-
git add .haystack/review-policy.md .haystack/pr-rules.yml
|
|
1625
|
-
git commit -m "Add Haystack review policies and PR rules"
|
|
1626
|
-
\`\`\`
|
|
1627
|
-
|
|
1628
|
-
Done! Haystack will now:
|
|
1629
|
-
- Flag PRs touching policy-matched files for human review
|
|
1630
|
-
- Check every PR diff against your custom rules and report violations
|
|
1631
|
-
`;
|
|
1632
|
-
const SETUP_RULES_COMMAND = `# Set Up Rules
|
|
1633
|
-
|
|
1634
|
-
Follow .agents/skills/setup-haystack-rules.md to configure review policies (what always needs human review) and PR rules (what code invariants to enforce).
|
|
1635
|
-
`;
|
|
1636
|
-
export async function createSkillFile() {
|
|
1637
|
-
const skillDir = path.join(process.cwd(), '.agents', 'skills');
|
|
1638
|
-
const setupPath = path.join(skillDir, 'setup-haystack.md');
|
|
1639
|
-
const refPath = path.join(skillDir, 'haystack-reference.md');
|
|
1640
|
-
const prepPath = path.join(skillDir, 'prepare-haystack.md');
|
|
1641
|
-
const secretsPath = path.join(skillDir, 'setup-haystack-secrets.md');
|
|
1642
|
-
const rulesPath = path.join(skillDir, 'setup-haystack-rules.md');
|
|
1643
|
-
// Create directory if needed
|
|
1644
|
-
await fs.mkdir(skillDir, { recursive: true });
|
|
1645
|
-
// Write all skill files
|
|
1646
|
-
await fs.writeFile(setupPath, SKILL_CONTENT, 'utf-8');
|
|
1647
|
-
await fs.writeFile(refPath, REFERENCE_CONTENT, 'utf-8');
|
|
1648
|
-
await fs.writeFile(prepPath, PREPARE_VERIFICATION_CONTENT, 'utf-8');
|
|
1649
|
-
await fs.writeFile(secretsPath, SECRETS_SKILL_CONTENT, 'utf-8');
|
|
1650
|
-
await fs.writeFile(rulesPath, SETUP_RULES_CONTENT, 'utf-8');
|
|
1651
|
-
return setupPath;
|
|
1652
|
-
}
|
|
1653
|
-
/**
|
|
1654
|
-
* Create the .claude/commands/ files for Claude Code slash commands
|
|
1655
|
-
* Users can invoke with /setup-haystack, /prepare-haystack, or /setup-haystack-secrets
|
|
1656
|
-
*/
|
|
1657
|
-
export async function createClaudeCommand() {
|
|
1658
|
-
const commandDir = path.join(process.cwd(), '.claude', 'commands');
|
|
1659
|
-
const setupPath = path.join(commandDir, 'setup-haystack.md');
|
|
1660
|
-
const prepPath = path.join(commandDir, 'prepare-haystack.md');
|
|
1661
|
-
const secretsPath = path.join(commandDir, 'setup-haystack-secrets.md');
|
|
1662
|
-
const rulesPath = path.join(commandDir, 'setup-haystack-rules.md');
|
|
1663
|
-
// Create directory if needed
|
|
1664
|
-
await fs.mkdir(commandDir, { recursive: true });
|
|
1665
|
-
// Write command files
|
|
1666
|
-
await fs.writeFile(setupPath, CLAUDE_COMMAND_CONTENT, 'utf-8');
|
|
1667
|
-
await fs.writeFile(prepPath, PREPARE_HAYSTACK_COMMAND, 'utf-8');
|
|
1668
|
-
await fs.writeFile(secretsPath, SECRETS_COMMAND_CONTENT, 'utf-8');
|
|
1669
|
-
await fs.writeFile(rulesPath, SETUP_RULES_COMMAND, 'utf-8');
|
|
1670
|
-
return setupPath;
|
|
1671
|
-
}
|