@haystackeditor/cli 0.7.2 → 0.8.1

package/dist/utils/skill.js

@@ -104,17 +104,32 @@ grep '"name":' .haystack.json
 }
 \`\`\`
 
-### Finding Golden Inputs
+---
 
-For analysis/processing pipelines:
-- Pick a **small, stable PR** that won't change
-- Document what the expected output should be
-- Add it as a comment in the flow description
+## STOP - Confirm Golden URLs and Golden Data
 
-\`\`\`bash
-# Find a good golden PR (small, merged, stable)
-gh pr list --state merged --limit 10 --json number,title,changedFiles | jq '.[] | select(.changedFiles < 5)'
-\`\`\`
+**If you identified backend services that need verification, you MUST ask the user with a proposed example:**
+
+First, explore the codebase to find:
+- Example inputs used in tests, scripts, or documentation
+- Default/demo values in config files or environment variables
+- IDs or URLs referenced in comments or READMEs
+
+Then propose a specific example:
+
+> I found these backend services that need verification:
+> - **[service name]**: [what it does]
+>
+> For backend verification, I need **golden test data** - stable inputs that produce predictable output.
+>
+> **Based on what I found in the codebase, here's my suggestion:**
+> - **Golden URL/Input**: \`[specific example you found, e.g., a test ID, demo endpoint, or sample file]\`
+> - **Command**: \`[the command to run with this input]\`
+> - **Expected Output**: \`[what success looks like based on the code]\`
+>
+> Does this look right? Or should I use different test data?
+
+**Wait for the user to confirm or provide alternatives before adding backend flows.**
 
 ---
 
@@ -153,6 +168,92 @@ grep -r "https://" src/ --include="*.ts" --include="*.tsx" | grep -v node_module
 
 ---
 
+## STOP - Confirm Frontend Golden URLs
+
+**Goldens are URLs with real staging data** - they let verification test pages that need actual content to render properly. You need MULTIPLE goldens to cover different scenarios.
+
+**Before writing flows for dynamic routes, explore the codebase to find example data:**
+
+1. Check test files (\`*.test.ts\`, \`cypress/\`, \`playwright/\`) for example IDs/URLs
+2. Check README/docs for demo links or example usage
+3. Check seed data, fixtures, or mock files
+4. Check \`.env.example\` or config for staging URLs
+5. Check existing routes to understand the URL format
+
+**Then propose MULTIPLE golden URLs (aim for 3-5) covering different scenarios:**
+
+> I found these dynamic routes that need staging data to render:
+>
+> **[Route Name]** (\`/path/:param1/:param2\`)
+>
+> | Golden | URL | Why |
+> |--------|-----|-----|
+> | Large dataset | \`/path/example/large-123\` | Tests performance with many items |
+> | Typical case | \`/path/example/medium-456\` | Common user scenario |
+> | Edge case | \`/path/example/empty-789\` | Tests empty/minimal state |
+> | Error state | \`/path/example/error-000\` | Tests error handling UI |
+>
+> **Evidence:**
+> - Found \`large-123\` in \`cypress/fixtures/test-data.json\`
+> - Found \`medium-456\` in README as demo example
+> - Found \`empty-789\` in \`src/tests/edge-cases.test.ts\`
+>
+> Does this look right?
+> 1. ✅ Yes, use these goldens
+> 2. 🔄 Use different URLs (I'll provide them)
+
+**Add goldens to \`verification.goldens\` in \`.haystack.json\` using this EXACT format:**
+
+\`\`\`json
+{
+  "verification": {
+    "goldens": {
+      "frontend": [
+        {
+          "id": "large-dataset",
+          "description": "100+ items - tests rendering performance",
+          "route": "/the/actual/url/with/data"
+        },
+        {
+          "id": "typical-case",
+          "description": "Standard 10-20 items",
+          "route": "/another/url/with/typical/data"
+        },
+        {
+          "id": "empty-state",
+          "description": "No items - tests empty state UI",
+          "route": "/url/that/shows/empty/state"
+        }
+      ],
+      "backend": [
+        {
+          "id": "golden-input-1",
+          "description": "Known-good input for pipeline testing",
+          "input": { "type": "github_pr", "value": "owner/repo#123" }
+        }
+      ]
+    },
+    "commands": [...]
+  }
+}
+\`\`\`
+
+**DO NOT:**
+- Ask the user what format to use - USE THIS FORMAT
+- Put goldens at the top level - they go inside \`verification.goldens\`
+- Use a flat object like \`{ "goldens": { "name": "url" } }\` - use the array format above
+- Embed golden URLs only in flow steps - also add them to the goldens section for discoverability
+
+**CRITICAL**:
+- Propose MULTIPLE goldens (3-5), not just one
+- Cover different scenarios: large, typical, empty, error states
+- Always propose SPECIFIC URLs, never ask the user to freestyle
+- Show WHERE you found the example data (file path, line if possible)
+- Make sure URLs match the app's actual route format (check \`<Route path=\`)
+- Wait for confirmation before writing flows with these URLs
+
+---
+
 ## Step 5: Write Flows
 
 Flows tell the Planner about your app's routes, UI elements, and user journeys.
@@ -596,6 +697,49 @@ The verification Planner needs to find UI elements by selectors. Generic selecto
 
 ---
 
+## ⚠️ CRITICAL RULE: Every Button MUST Have \`data-testid\` and \`aria-label\`
+
+**This is non-negotiable.** Every \`<button>\` element in the codebase MUST have:
+1. \`data-testid="descriptive-name"\` - For automated testing/verification
+2. \`aria-label="Action description"\` - For accessibility AND verification
+
+**Why?** The verification system clicks buttons by their \`data-testid\` or \`aria-label\`. Without these, buttons are invisible to automation. We learned this the hard way - a theme toggle button without these attributes broke our entire verification pipeline.
+
+\`\`\`tsx
+// ❌ WRONG - No identifiers, verification cannot click this
+<button onClick={toggleTheme}>
+  <SunIcon />
+</button>
+
+// ✅ CORRECT - Both data-testid AND aria-label
+<button
+  onClick={toggleTheme}
+  data-testid="theme-toggle"
+  aria-label="Toggle dark mode"
+>
+  <SunIcon />
+</button>
+\`\`\`
+
+**For button wrapper components**, pass through these props:
+\`\`\`tsx
+interface ButtonProps {
+  'data-testid'?: string;
+  'aria-label'?: string;
+  // ...other props
+}
+
+function IconButton({ 'data-testid': testId, 'aria-label': ariaLabel, ...props }: ButtonProps) {
+  return (
+    <button data-testid={testId} aria-label={ariaLabel} {...props}>
+      {props.children}
+    </button>
+  );
+}
+\`\`\`
+
+---
+
 ## What to Add
 
 ### 1. \`aria-label\` on Interactive Elements
@@ -727,13 +871,21 @@ Forms should have proper labels and descriptions:
 
 ## Step 1: Scan for Missing Identifiers
 
+**Start with buttons - these are the most critical:**
+
 \`\`\`bash
-# Find buttons without aria-label
-grep -rn "<button" src/ --include="*.tsx" | grep -v "aria-label" | head -20
+# CRITICAL: Find ALL buttons missing data-testid (fix ALL of these!)
+grep -rn "<button" src/ --include="*.tsx" | grep -v "data-testid" | head -30
+
+# CRITICAL: Find ALL buttons missing aria-label (fix ALL of these!)
+grep -rn "<button" src/ --include="*.tsx" | grep -v "aria-label" | head -30
 
-# Find icon-only buttons (likely missing labels)
+# Find icon-only buttons (MUST have aria-label since no visible text)
 grep -rn "<button.*Icon\\|<button.*>.*</.*Icon>" src/ --include="*.tsx" | head -20
 
+# Find button wrapper components that need to pass through data-testid/aria-label
+grep -rn "function.*Button\\|const.*Button.*=" src/ --include="*.tsx" | head -10
+
 # Find modals/dialogs without role
 grep -rn "modal\\|dialog\\|Modal\\|Dialog" src/ --include="*.tsx" | grep -v "role=" | head -20
 
@@ -744,6 +896,8 @@ grep -rn "<input\\|<select\\|<textarea" src/ --include="*.tsx" | grep -v "aria-l
 ls src/components/ src/pages/ 2>/dev/null
 \`\`\`
 
+**Every button found without \`data-testid\` or \`aria-label\` MUST be fixed.**
+
 ---
 
 ## Step 2: Prioritize by Impact
@@ -916,31 +1070,345 @@ Follow .agents/skills/prepare-haystack.md to add accessibility attributes that m
 
 Run this BEFORE /setup-haystack to ensure your codebase has good selectors.
 `;
+const SECRETS_COMMAND_CONTENT = `# Set Up Haystack Secrets
+
+Follow .agents/skills/setup-haystack-secrets.md to configure secrets (API keys, credentials) for Haystack verification.
+`;
+const SECRETS_SKILL_CONTENT = `# Set Up Haystack Secrets
+
+**Your job**: Help the user configure secrets needed for Haystack verification, especially LLM API keys for backend services.
+
+---
+
+## Step 1: Detect Services That Need Secrets
+
+Scan the codebase to identify services that might need API keys or credentials:
+
+\`\`\`bash
+# Find LLM/AI SDK usage
+grep -rn "openai\\|anthropic\\|bedrock\\|OPENAI_API_KEY\\|ANTHROPIC_API_KEY" --include="*.ts" --include="*.py" --include="*.js" | grep -v node_modules | head -20
+
+# Find environment variable references
+grep -rn "process.env\\.\\|os.environ\\|env\\." --include="*.ts" --include="*.py" --include="*.js" | grep -v node_modules | grep -iE "key|secret|token|api" | head -20
+
+# Check for .env.example or similar
+cat .env.example .env.sample 2>/dev/null || echo "No .env example found"
+
+# Check existing secrets in .haystack.json
+grep -A10 '"secrets"' .haystack.json 2>/dev/null || echo "No secrets configured yet"
+\`\`\`
+
+---
+
+## Step 2: Categorize the Secrets Found
+
+Group secrets by type and service:
+
+| Category | Examples | Storage Recommendation |
+|----------|----------|------------------------|
+| **LLM API Keys** | \`OPENAI_API_KEY\`, \`ANTHROPIC_API_KEY\` | Haystack Secrets (encrypted) |
+| **Cloud Credentials** | \`AWS_ACCESS_KEY_ID\`, \`GCP_SERVICE_ACCOUNT\` | Use OIDC in CI, not static keys |
+| **Database** | \`DATABASE_URL\`, \`REDIS_URL\` | Haystack Secrets or passthrough |
+| **Third-party APIs** | \`STRIPE_KEY\`, \`SENDGRID_KEY\` | Haystack Secrets |
+| **Internal Services** | \`API_TOKEN\`, \`WEBHOOK_SECRET\` | Haystack Secrets |
+
+---
+
+## STOP - Present Findings and Ask User
+
+**You MUST present what you found and ask the user before proceeding:**
+
+> I scanned the codebase and found these services that may need secrets for verification:
+>
+> **Services detected:**
+> - [Service 1]: Uses \`OPENAI_API_KEY\` for [purpose]
+> - [Service 2]: Uses \`DATABASE_URL\` for [purpose]
+> - ...
+>
+> **Questions:**
+>
+> 1. **LLM API Keys**: Which provider do you use?
+>    - OpenAI (\`OPENAI_API_KEY\`)
+>    - Anthropic (\`ANTHROPIC_API_KEY\`)
+>    - AWS Bedrock (use OIDC, no static keys needed)
+>    - Other: ___
+>
+> 2. **For each secret**, should verification use:
+>    - **Real credentials** (stored securely in Haystack Secrets)
+>    - **Test/sandbox credentials** (separate keys for verification only)
+>    - **Mock/skip** (service not needed for visual verification)
+>
+> 3. **Do you have separate test credentials?**
+>    - Yes, I have sandbox/test API keys
+>    - No, I'll use production keys (with usage limits)
+>    - I need to create test credentials first
+
+**Wait for the user's response before continuing.**
+
+---
+
+## Step 3: Configure Secrets in .haystack.json
+
+Based on user's answers, add secrets to the config:
+
+\`\`\`json
+{
+  "secrets": {
+    "OPENAI_API_KEY": {
+      "description": "OpenAI API key for analysis service",
+      "required": true,
+      "services": ["analysis"]
+    },
+    "DATABASE_URL": {
+      "description": "PostgreSQL connection string",
+      "required": false,
+      "services": ["api"]
+    }
+  }
+}
+\`\`\`
+
+### Secret Properties
+
+| Property | Description |
+|----------|-------------|
+| \`description\` | What this secret is for (shown to user) |
+| \`required\` | If \`true\`, verification fails without it |
+| \`services\` | Which services need this secret (for scoping) |
+
+---
+
+## Step 4: Guide User to Store Secrets
+
+### Option A: Haystack CLI (Recommended)
+
+\`\`\`bash
+# Store secrets securely (encrypted at rest)
+npx @haystackeditor/cli secrets set OPENAI_API_KEY
+# Prompts for value, never shown in terminal history
+
+# Verify it's stored
+npx @haystackeditor/cli secrets list
+\`\`\`
+
+### Option B: Environment Variables (Local Development)
+
+\`\`\`bash
+# Add to .env.local (gitignored)
+echo 'OPENAI_API_KEY=sk-...' >> .env.local
+
+# Make sure .env.local is in .gitignore
+grep -q '.env.local' .gitignore || echo '.env.local' >> .gitignore
+\`\`\`
+
+### Option C: CI/CD Secrets (GitHub Actions)
+
+\`\`\`yaml
+# .github/workflows/haystack.yml
+env:
+  OPENAI_API_KEY: \${{ secrets.OPENAI_API_KEY }}
+\`\`\`
+
+---
+
+## Step 5: Handle Cloud Provider Credentials
+
+**Never store long-lived cloud credentials.** Use OIDC instead:
+
+### AWS (Bedrock, S3, etc.)
+
+\`\`\`yaml
+# GitHub Actions with OIDC - no secrets needed!
+jobs:
+  verify:
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::123456789:role/haystack-verify
+          aws-region: us-west-2
+\`\`\`
+
+### GCP (Vertex AI, Cloud Storage, etc.)
+
+\`\`\`yaml
+jobs:
+  verify:
+    permissions:
+      id-token: write
+    steps:
+      - uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: projects/123/locations/global/...
+          service_account: haystack-verify@project.iam.gserviceaccount.com
+\`\`\`
+
+---
+
+## Step 6: Test Secret Access
+
+After storing secrets, verify they're accessible:
+
+\`\`\`bash
+# Test locally
+npx @haystackeditor/cli secrets test
+
+# Or check manually
+npx @haystackeditor/cli secrets get OPENAI_API_KEY --masked
+# Shows: sk-...xxxx (last 4 chars only)
+\`\`\`
+
+---
+
+## Step 7: Update .haystack.json
+
+Add the secrets configuration:
+
+\`\`\`bash
+# Show what to add
+cat << 'EOF'
+Add this to your .haystack.json:
+
+{
+  "secrets": {
+    "OPENAI_API_KEY": {
+      "description": "OpenAI API key for LLM analysis",
+      "required": true
+    }
+  }
+}
+EOF
+\`\`\`
+
+---
+
+## Common Patterns
+
+### Analysis Pipeline with LLM
+
+\`\`\`json
+{
+  "services": {
+    "analysis": {
+      "root": "packages/analysis",
+      "command": "pnpm start",
+      "env": {
+        "PR_IDENTIFIER": "$PR_IDENTIFIER"
+      }
+    }
+  },
+  "secrets": {
+    "OPENAI_API_KEY": {
+      "description": "OpenAI API for code analysis",
+      "required": true,
+      "services": ["analysis"]
+    }
+  }
+}
+\`\`\`
+
+### Multiple LLM Providers
+
+\`\`\`json
+{
+  "secrets": {
+    "OPENAI_API_KEY": {
+      "description": "OpenAI for embeddings",
+      "services": ["search"]
+    },
+    "ANTHROPIC_API_KEY": {
+      "description": "Claude for code review",
+      "services": ["review"]
+    }
+  }
+}
+\`\`\`
+
+### Database + API Keys
+
+\`\`\`json
+{
+  "secrets": {
+    "DATABASE_URL": {
+      "description": "PostgreSQL for test database",
+      "required": true
+    },
+    "STRIPE_TEST_KEY": {
+      "description": "Stripe test mode key",
+      "required": false
+    }
+  }
+}
+\`\`\`
+
+---
+
+## Security Best Practices
+
+1. **Use test/sandbox credentials** when available (OpenAI, Stripe, etc. offer these)
+2. **Set usage limits** on API keys used for verification
+3. **Never commit secrets** - use \`.gitignore\` for \`.env.local\`
+4. **Rotate regularly** - especially if verification runs on external PRs
+5. **Scope narrowly** - use \`services\` array to limit which services see which secrets
+6. **Prefer OIDC** for cloud providers instead of static credentials
+
+---
+
+## Troubleshooting
+
+### "Secret not found" errors
+
+\`\`\`bash
+# Check if secret is stored
+npx @haystackeditor/cli secrets list
+
+# Check if it's in the right scope
+npx @haystackeditor/cli secrets get SECRET_NAME --show-metadata
+\`\`\`
+
+### "Permission denied" for cloud resources
+
+- Verify OIDC trust policy includes your repo
+- Check IAM role has required permissions
+- Ensure GitHub Actions has \`id-token: write\` permission
+
+### Secrets work locally but not in CI
+
+- Secrets stored via CLI are user-scoped by default
+- For CI, use GitHub Secrets or organization-level Haystack secrets
+- Check \`npx @haystackeditor/cli secrets list --scope=org\`
+`;
 export async function createSkillFile() {
     const skillDir = path.join(process.cwd(), '.agents', 'skills');
     const setupPath = path.join(skillDir, 'setup-haystack.md');
     const refPath = path.join(skillDir, 'haystack-reference.md');
     const prepPath = path.join(skillDir, 'prepare-haystack.md');
+    const secretsPath = path.join(skillDir, 'setup-haystack-secrets.md');
     // Create directory if needed
     await fs.mkdir(skillDir, { recursive: true });
     // Write all skill files
     await fs.writeFile(setupPath, SKILL_CONTENT, 'utf-8');
     await fs.writeFile(refPath, REFERENCE_CONTENT, 'utf-8');
     await fs.writeFile(prepPath, PREPARE_VERIFICATION_CONTENT, 'utf-8');
+    await fs.writeFile(secretsPath, SECRETS_SKILL_CONTENT, 'utf-8');
     return setupPath;
 }
 /**
  * Create the .claude/commands/ files for Claude Code slash commands
- * Users can invoke with /setup-haystack or /prepare-haystack
+ * Users can invoke with /setup-haystack, /prepare-haystack, or /setup-haystack-secrets
  */
 export async function createClaudeCommand() {
     const commandDir = path.join(process.cwd(), '.claude', 'commands');
     const setupPath = path.join(commandDir, 'setup-haystack.md');
     const prepPath = path.join(commandDir, 'prepare-haystack.md');
+    const secretsPath = path.join(commandDir, 'setup-haystack-secrets.md');
     // Create directory if needed
     await fs.mkdir(commandDir, { recursive: true });
     // Write command files
     await fs.writeFile(setupPath, CLAUDE_COMMAND_CONTENT, 'utf-8');
     await fs.writeFile(prepPath, PREPARE_HAYSTACK_COMMAND, 'utf-8');
+    await fs.writeFile(secretsPath, SECRETS_COMMAND_CONTENT, 'utf-8');
     return setupPath;
 }

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@haystackeditor/cli",
-  "version": "0.7.2",
+  "version": "0.8.1",
   "description": "Set up Haystack verification for your project",
   "type": "module",
   "bin": {
     "haystack": "./dist/index.js"
   },
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && rm -rf dist/assets && cp -r src/assets dist/assets",
     "dev": "tsc --watch",
     "start": "node dist/index.js",
     "prepublishOnly": "npm run build"