@haystackeditor/cli 0.8.0 → 0.9.0

package/dist/utils/skill.js

@@ -168,6 +168,92 @@ grep -r "https://" src/ --include="*.ts" --include="*.tsx" | grep -v node_module
 
 ---
 
+## STOP - Confirm Frontend Golden URLs
+
+**Goldens are URLs with real staging data** - they let verification test pages that need actual content to render properly. You need MULTIPLE goldens to cover different scenarios.
+
+**Before writing flows for dynamic routes, explore the codebase to find example data:**
+
+1. Check test files (\`*.test.ts\`, \`cypress/\`, \`playwright/\`) for example IDs/URLs
+2. Check README/docs for demo links or example usage
+3. Check seed data, fixtures, or mock files
+4. Check \`.env.example\` or config for staging URLs
+5. Check existing routes to understand the URL format
+
+**Then propose MULTIPLE golden URLs (aim for 3-5) covering different scenarios:**
+
+> I found these dynamic routes that need staging data to render:
+>
+> **[Route Name]** (\`/path/:param1/:param2\`)
+>
+> | Golden | URL | Why |
+> |--------|-----|-----|
+> | Large dataset | \`/path/example/large-123\` | Tests performance with many items |
+> | Typical case | \`/path/example/medium-456\` | Common user scenario |
+> | Edge case | \`/path/example/empty-789\` | Tests empty/minimal state |
+> | Error state | \`/path/example/error-000\` | Tests error handling UI |
+>
+> **Evidence:**
+> - Found \`large-123\` in \`cypress/fixtures/test-data.json\`
+> - Found \`medium-456\` in README as demo example
+> - Found \`empty-789\` in \`src/tests/edge-cases.test.ts\`
+>
+> Does this look right?
+> 1. ✅ Yes, use these goldens
+> 2. 🔄 Use different URLs (I'll provide them)
+
+**Add goldens to \`verification.goldens\` in \`.haystack.json\` using this EXACT format:**
+
+\`\`\`json
+{
+  "verification": {
+    "goldens": {
+      "frontend": [
+        {
+          "id": "large-dataset",
+          "description": "100+ items - tests rendering performance",
+          "route": "/the/actual/url/with/data"
+        },
+        {
+          "id": "typical-case",
+          "description": "Standard 10-20 items",
+          "route": "/another/url/with/typical/data"
+        },
+        {
+          "id": "empty-state",
+          "description": "No items - tests empty state UI",
+          "route": "/url/that/shows/empty/state"
+        }
+      ],
+      "backend": [
+        {
+          "id": "golden-input-1",
+          "description": "Known-good input for pipeline testing",
+          "input": { "type": "github_pr", "value": "owner/repo#123" }
+        }
+      ]
+    },
+    "commands": [...]
+  }
+}
+\`\`\`
+
+**DO NOT:**
+- Ask the user what format to use - USE THIS FORMAT
+- Put goldens at the top level - they go inside \`verification.goldens\`
+- Use a flat object like \`{ "goldens": { "name": "url" } }\` - use the array format above
+- Embed golden URLs only in flow steps - also add them to the goldens section for discoverability
+
+**CRITICAL**:
+- Propose MULTIPLE goldens (3-5), not just one
+- Cover different scenarios: large, typical, empty, error states
+- Always propose SPECIFIC URLs, never ask the user to freestyle
+- Show WHERE you found the example data (file path, line if possible)
+- Make sure URLs match the app's actual route format (check \`<Route path=\`)
+- Wait for confirmation before writing flows with these URLs
+
+---
+
 ## Step 5: Write Flows
 
 Flows tell the Planner about your app's routes, UI elements, and user journeys.
@@ -984,31 +1070,602 @@ Follow .agents/skills/prepare-haystack.md to add accessibility attributes that m
 
 Run this BEFORE /setup-haystack to ensure your codebase has good selectors.
 `;
+const SECRETS_COMMAND_CONTENT = `# Set Up Haystack Secrets
+
+Follow .agents/skills/setup-haystack-secrets.md to configure secrets (API keys, credentials) for Haystack verification.
+`;
+const SECRETS_SKILL_CONTENT = `# Set Up Haystack Secrets
+
+**Your job**: Help the user configure secrets needed for Haystack verification, especially LLM API keys for backend services.
+
+---
+
+## Step 1: Detect Services That Need Secrets
+
+Scan the codebase to identify services that might need API keys or credentials:
+
+\`\`\`bash
+# Find LLM/AI SDK usage
+grep -rn "openai\\|anthropic\\|bedrock\\|OPENAI_API_KEY\\|ANTHROPIC_API_KEY" --include="*.ts" --include="*.py" --include="*.js" | grep -v node_modules | head -20
+
+# Find environment variable references
+grep -rn "process.env\\.\\|os.environ\\|env\\." --include="*.ts" --include="*.py" --include="*.js" | grep -v node_modules | grep -iE "key|secret|token|api" | head -20
+
+# Check for .env.example or similar
+cat .env.example .env.sample 2>/dev/null || echo "No .env example found"
+
+# Check existing secrets in .haystack.json
+grep -A10 '"secrets"' .haystack.json 2>/dev/null || echo "No secrets configured yet"
+\`\`\`
+
+---
+
+## Step 2: Categorize the Secrets Found
+
+Group secrets by type and service:
+
+| Category | Examples | Storage Recommendation |
+|----------|----------|------------------------|
+| **LLM API Keys** | \`OPENAI_API_KEY\`, \`ANTHROPIC_API_KEY\` | Haystack Secrets (encrypted) |
+| **Cloud Credentials** | \`AWS_ACCESS_KEY_ID\`, \`GCP_SERVICE_ACCOUNT\` | Use OIDC in CI, not static keys |
+| **Database** | \`DATABASE_URL\`, \`REDIS_URL\` | Haystack Secrets or passthrough |
+| **Third-party APIs** | \`STRIPE_KEY\`, \`SENDGRID_KEY\` | Haystack Secrets |
+| **Internal Services** | \`API_TOKEN\`, \`WEBHOOK_SECRET\` | Haystack Secrets |
+
+---
+
+## STOP - Present Findings and Ask User
+
+**You MUST present what you found and ask the user before proceeding:**
+
+> I scanned the codebase and found these services that may need secrets for verification:
+>
+> **Services detected:**
+> - [Service 1]: Uses \`OPENAI_API_KEY\` for [purpose]
+> - [Service 2]: Uses \`DATABASE_URL\` for [purpose]
+> - ...
+>
+> **Questions:**
+>
+> 1. **LLM API Keys**: Which provider do you use?
+>    - OpenAI (\`OPENAI_API_KEY\`)
+>    - Anthropic (\`ANTHROPIC_API_KEY\`)
+>    - AWS Bedrock (use OIDC, no static keys needed)
+>    - Other: ___
+>
+> 2. **For each secret**, should verification use:
+>    - **Real credentials** (stored securely in Haystack Secrets)
+>    - **Test/sandbox credentials** (separate keys for verification only)
+>    - **Mock/skip** (service not needed for visual verification)
+>
+> 3. **Do you have separate test credentials?**
+>    - Yes, I have sandbox/test API keys
+>    - No, I'll use production keys (with usage limits)
+>    - I need to create test credentials first
+
+**Wait for the user's response before continuing.**
+
+---
+
+## Step 3: Configure Secrets in .haystack.json
+
+Based on user's answers, add secrets to the config:
+
+\`\`\`json
+{
+  "secrets": {
+    "OPENAI_API_KEY": {
+      "description": "OpenAI API key for analysis service",
+      "required": true,
+      "services": ["analysis"]
+    },
+    "DATABASE_URL": {
+      "description": "PostgreSQL connection string",
+      "required": false,
+      "services": ["api"]
+    }
+  }
+}
+\`\`\`
+
+### Secret Properties
+
+| Property | Description |
+|----------|-------------|
+| \`description\` | What this secret is for (shown to user) |
+| \`required\` | If \`true\`, verification fails without it |
+| \`services\` | Which services need this secret (for scoping) |
+
+---
+
+## Step 4: Guide User to Store Secrets
+
+### Option A: Haystack CLI (Recommended)
+
+\`\`\`bash
+# Store secrets securely (encrypted at rest)
+npx @haystackeditor/cli secrets set OPENAI_API_KEY
+# Prompts for value, never shown in terminal history
+
+# Verify it's stored
+npx @haystackeditor/cli secrets list
+\`\`\`
+
+### Option B: Environment Variables (Local Development)
+
+\`\`\`bash
+# Add to .env.local (gitignored)
+echo 'OPENAI_API_KEY=sk-...' >> .env.local
+
+# Make sure .env.local is in .gitignore
+grep -q '.env.local' .gitignore || echo '.env.local' >> .gitignore
+\`\`\`
+
+### Option C: CI/CD Secrets (GitHub Actions)
+
+\`\`\`yaml
+# .github/workflows/haystack.yml
+env:
+  OPENAI_API_KEY: \${{ secrets.OPENAI_API_KEY }}
+\`\`\`
+
+---
+
+## Step 5: Handle Cloud Provider Credentials
+
+**Never store long-lived cloud credentials.** Use OIDC instead:
+
+### AWS (Bedrock, S3, etc.)
+
+\`\`\`yaml
+# GitHub Actions with OIDC - no secrets needed!
+jobs:
+  verify:
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::123456789:role/haystack-verify
+          aws-region: us-west-2
+\`\`\`
+
+### GCP (Vertex AI, Cloud Storage, etc.)
+
+\`\`\`yaml
+jobs:
+  verify:
+    permissions:
+      id-token: write
+    steps:
+      - uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: projects/123/locations/global/...
+          service_account: haystack-verify@project.iam.gserviceaccount.com
+\`\`\`
+
+---
+
+## Step 6: Test Secret Access
+
+After storing secrets, verify they're accessible:
+
+\`\`\`bash
+# Test locally
+npx @haystackeditor/cli secrets test
+
+# Or check manually
+npx @haystackeditor/cli secrets get OPENAI_API_KEY --masked
+# Shows: sk-...xxxx (last 4 chars only)
+\`\`\`
+
+---
+
+## Step 7: Update .haystack.json
+
+Add the secrets configuration:
+
+\`\`\`bash
+# Show what to add
+cat << 'EOF'
+Add this to your .haystack.json:
+
+{
+  "secrets": {
+    "OPENAI_API_KEY": {
+      "description": "OpenAI API key for LLM analysis",
+      "required": true
+    }
+  }
+}
+EOF
+\`\`\`
+
+---
+
+## Common Patterns
+
+### Analysis Pipeline with LLM
+
+\`\`\`json
+{
+  "services": {
+    "analysis": {
+      "root": "packages/analysis",
+      "command": "pnpm start",
+      "env": {
+        "PR_IDENTIFIER": "$PR_IDENTIFIER"
+      }
+    }
+  },
+  "secrets": {
+    "OPENAI_API_KEY": {
+      "description": "OpenAI API for code analysis",
+      "required": true,
+      "services": ["analysis"]
+    }
+  }
+}
+\`\`\`
+
+### Multiple LLM Providers
+
+\`\`\`json
+{
+  "secrets": {
+    "OPENAI_API_KEY": {
+      "description": "OpenAI for embeddings",
+      "services": ["search"]
+    },
+    "ANTHROPIC_API_KEY": {
+      "description": "Claude for code review",
+      "services": ["review"]
+    }
+  }
+}
+\`\`\`
+
+### Database + API Keys
+
+\`\`\`json
+{
+  "secrets": {
+    "DATABASE_URL": {
+      "description": "PostgreSQL for test database",
+      "required": true
+    },
+    "STRIPE_TEST_KEY": {
+      "description": "Stripe test mode key",
+      "required": false
+    }
+  }
+}
+\`\`\`
+
+---
+
+## Security Best Practices
+
+1. **Use test/sandbox credentials** when available (OpenAI, Stripe, etc. offer these)
+2. **Set usage limits** on API keys used for verification
+3. **Never commit secrets** - use \`.gitignore\` for \`.env.local\`
+4. **Rotate regularly** - especially if verification runs on external PRs
+5. **Scope narrowly** - use \`services\` array to limit which services see which secrets
+6. **Prefer OIDC** for cloud providers instead of static credentials
+
+---
+
+## Troubleshooting
+
+### "Secret not found" errors
+
+\`\`\`bash
+# Check if secret is stored
+npx @haystackeditor/cli secrets list
+
+# Check if it's in the right scope
+npx @haystackeditor/cli secrets get SECRET_NAME --show-metadata
+\`\`\`
+
+### "Permission denied" for cloud resources
+
+- Verify OIDC trust policy includes your repo
+- Check IAM role has required permissions
+- Ensure GitHub Actions has \`id-token: write\` permission
+
+### Secrets work locally but not in CI
+
+- Secrets stored via CLI are user-scoped by default
+- For CI, use GitHub Secrets or organization-level Haystack secrets
+- Check \`npx @haystackeditor/cli secrets list --scope=org\`
+`;
+const SETUP_RULES_CONTENT = `# Set Up Review Policies & Rules
+
+**Your job**: Help the user configure two systems that control how Haystack evaluates their PRs:
+
+1. **Review Policies** — deterministic, path-based triggers that **always** require human review
+2. **PR Rules** — LLM-evaluated code invariants that **flag violations** in the diff
+
+These are distinct systems. Make sure the user understands the difference before proceeding.
+
+---
+
+## The Two Systems
+
+### Review Policies (\`.haystack/review-policy.md\`)
+
+**"If a PR touches X → human review needed, period."**
+
+Policies are simple glob patterns. When a PR changes files matching a policy's paths, Haystack will always recommend human review for that PR, regardless of what the code changes look like.
+
+**Good candidates for policies:**
+- Infrastructure (Terraform, Kubernetes, CI/CD pipelines)
+- Authentication and authorization code
+- Database migrations and schemas
+- API contracts (OpenAPI specs, GraphQL schemas, protobuf)
+- Security-sensitive code (crypto, secrets handling)
+- Billing/payment logic
+- Code that bypasses feature flags or launches directly to production
+
+**Format:**
+\`\`\`markdown
+## Policy name
+- **Paths**: \\\`glob/pattern/**\\\`, \\\`other/pattern/*.ts\\\`
+- **Severity**: critical | high | medium | low
+- **Reason**: Why this always needs human eyes
+\`\`\`
+
+### PR Rules (\`.haystack/pr-rules.yml\`)
+
+**"If this invariant is violated in the diff → flag it."**
+
+Rules are checked by an LLM that reads the PR diff and looks for violations. They catch patterns that shouldn't appear in new code, regardless of which files are touched.
+
+**Good candidates for rules:**
+- Error handling conventions (no silent catches, no swallowed errors)
+- Logging requirements (all API calls must be logged)
+- Security patterns (no hardcoded secrets, no SQL string concatenation)
+- Code style invariants (no TODOs without issue refs, no console.log in production)
+- Architecture boundaries (service A must not import from service B)
+- Naming conventions specific to your team
+
+**Format:**
+\`\`\`yaml
+- id: PR001
+  name: Short rule name
+  type: llm
+  severity: warning | error
+  message: Human-readable description of the violation
+  llm:
+    prompt: >
+      Detailed instructions for the LLM about what to look for
+      in the PR diff. Be specific about what counts as a violation
+      and what doesn't.
+    files: "optional/glob/pattern/**/*.ts"  # optional file scope
+\`\`\`
+
+---
+
+## Step 1: Explore the Codebase
+
+Before suggesting anything, understand the repo structure:
+
+\`\`\`bash
+# Understand repo layout
+ls -la
+find . -maxdepth 2 -type d -not -path '*/node_modules/*' -not -path '*/.git/*' | head -40
+
+# Find infrastructure files
+find . -name "*.tf" -o -name "Dockerfile*" -o -name "docker-compose*" -o -name "*.yml" -path "*/.github/*" | head -20
+
+# Find auth-related code
+grep -rl "auth\\|login\\|session\\|token\\|credential" --include="*.ts" --include="*.py" --include="*.go" --include="*.js" -l | grep -v node_modules | head -20
+
+# Find database/migration files
+find . -path "*/migration*" -o -path "*/schema*" -o -name "*.sql" | grep -v node_modules | head -20
+
+# Find API contract files
+find . -name "*.proto" -o -name "openapi*" -o -name "swagger*" -o -name "*.graphql" | head -20
+
+# Find feature flag patterns
+grep -rl "feature.?flag\\|feature.?toggle\\|isEnabled\\|FEATURE_" --include="*.ts" --include="*.py" --include="*.go" --include="*.js" -l | grep -v node_modules | head -20
+
+# Check for existing config
+cat .haystack/review-policy.md 2>/dev/null || echo "No review-policy.md yet"
+cat .haystack/pr-rules.yml 2>/dev/null || echo "No pr-rules.yml yet"
+\`\`\`
+
+Also look for:
+- CI/CD pipeline files
+- Payment or billing directories
+- Secrets or credential management code
+- Environment configuration patterns
+- Key shared libraries or core utilities that many things depend on
+
+---
+
+## STOP — Propose Review Policies
+
+Present your findings to the user. Be specific about what you found and why each area warrants a policy.
+
+**Format your proposal like this:**
+
+> Based on the codebase, here are the areas I'd recommend always requiring human review:
+>
+> | # | Policy | Paths | Severity | Why |
+> |---|--------|-------|----------|-----|
+> | 1 | Infrastructure | \\\`terraform/**\\\`, \\\`*.tf\\\` | high | Cost and security implications |
+> | 2 | Auth & secrets | \\\`src/auth/**\\\`, \\\`**/*.secret*\\\` | critical | Security-sensitive |
+> | 3 | ... | ... | ... | ... |
+>
+> **Notes:**
+> - I didn't find any database migrations in this repo, so I'm not including that
+> - The \`src/core/\` directory is imported by everything — should changes there always need review?
+>
+> Which of these should I include? Any to add, remove, or modify?
+
+**Wait for the user to confirm before writing anything.**
+
+---
+
+## Step 2: Write review-policy.md
+
+Once the user confirms, write the policies to \`.haystack/review-policy.md\`:
+
+\`\`\`markdown
+# Review Policies
+
+Review policies define file patterns that always require human attention. When a PR touches files matching these patterns, Haystack will flag it for human review.
+
+## [Policy name]
+- **Paths**: \\\`pattern/**\\\`
+- **Severity**: [level]
+- **Reason**: [why]
+\`\`\`
+
+Create the \`.haystack/\` directory if it doesn't exist.
+
+---
+
+## Step 3: Understand Team Conventions
+
+Now shift to rules. Ask the user about their code quality invariants:
+
+> Now let's set up **PR rules** — these are code invariants that Haystack checks in every diff.
+>
+> Unlike policies (which trigger on file paths), rules look at *what the code does*. For example:
+> - "Don't add empty catch blocks"
+> - "Don't add TODOs without issue references"
+> - "Don't hardcode API URLs"
+>
+> **What conventions does your team care about?** Some common categories:
+>
+> 1. **Error handling** — e.g., no swallowed errors, no silent fallbacks
+> 2. **Security** — e.g., no hardcoded secrets, no raw SQL strings
+> 3. **Logging** — e.g., all API calls must have logging
+> 4. **Code hygiene** — e.g., no TODOs without issues, no console.log in production
+> 5. **Architecture** — e.g., no cross-service imports, no direct DB access from UI layer
+> 6. **Testing** — e.g., no skipped tests without reason, no snapshot-only tests
+>
+> Tell me which categories matter to your team, or describe your own conventions.
+
+Also explore the codebase for clues:
+
+\`\`\`bash
+# Check for linter configs (reveals team conventions)
+cat .eslintrc* .eslintrc.json .eslintrc.js eslint.config.* 2>/dev/null | head -50
+
+# Check for existing code review guidelines
+find . -name "CONTRIBUTING*" -o -name "REVIEW*" -o -name "STYLE*" | head -5
+
+# Look for common anti-patterns
+grep -rn "catch.*{\\s*}" --include="*.ts" --include="*.js" | head -5
+grep -rn "console\\.log" --include="*.ts" --include="*.tsx" | grep -v test | head -5
+grep -rn "TODO\\|FIXME\\|HACK\\|XXX" --include="*.ts" --include="*.tsx" | head -10
+\`\`\`
+
+---
+
+## STOP — Propose PR Rules
+
+Based on the user's input and your codebase exploration, propose 2-5 custom rules.
+
+**Format your proposal like this:**
+
+> Here are the PR rules I'd suggest:
+>
+> **1. PR001 — No silent error handling** (warning)
+> > Flag: Adding catch blocks that swallow errors without logging or re-throwing
+>
+> **2. PR002 — No TODOs without tracking issues** (warning)
+> > Flag: Adding TODO/FIXME comments without referencing an issue number
+>
+> **3. PR003 — No hardcoded API endpoints** (warning)
+> > Flag: Adding hardcoded URLs or API endpoints instead of using config/env vars
+>
+> Should I add, remove, or modify any of these?
+
+**Wait for the user to confirm before writing.**
+
+---
+
+## Step 4: Write pr-rules.yml
+
+Once confirmed, write to \`.haystack/pr-rules.yml\`:
+
+\`\`\`yaml
+version: 1
+
+rules:
+  - id: PR001
+    name: [Short name]
+    type: llm
+    severity: warning
+    message: [Human-readable description]
+    llm:
+      prompt: >
+        [Detailed LLM instructions — be specific about what counts as
+        a violation and what doesn't. Include examples of violations
+        and non-violations when helpful.]
+\`\`\`
+
+**Tips for writing good rule prompts:**
+- Be specific about what counts as a violation
+- Mention common false positives to ignore (e.g., "test files are exempt")
+- Use the \`files\` field to scope rules to relevant file types
+- \`severity: error\` caps the Haystack rating at 3 (needs review); \`warning\` is advisory
+
+---
+
+## Step 5: Commit
+
+\`\`\`bash
+git add .haystack/review-policy.md .haystack/pr-rules.yml
+git commit -m "Add Haystack review policies and PR rules"
+\`\`\`
+
+Done! Haystack will now:
+- Flag PRs touching policy-matched files for human review
+- Check every PR diff against your custom rules and report violations
+`;
+const SETUP_RULES_COMMAND = `# Set Up Rules
+
+Follow .agents/skills/setup-haystack-rules.md to configure review policies (what always needs human review) and PR rules (what code invariants to enforce).
+`;
 export async function createSkillFile() {
     const skillDir = path.join(process.cwd(), '.agents', 'skills');
     const setupPath = path.join(skillDir, 'setup-haystack.md');
     const refPath = path.join(skillDir, 'haystack-reference.md');
     const prepPath = path.join(skillDir, 'prepare-haystack.md');
+    const secretsPath = path.join(skillDir, 'setup-haystack-secrets.md');
+    const rulesPath = path.join(skillDir, 'setup-haystack-rules.md');
     // Create directory if needed
     await fs.mkdir(skillDir, { recursive: true });
     // Write all skill files
     await fs.writeFile(setupPath, SKILL_CONTENT, 'utf-8');
     await fs.writeFile(refPath, REFERENCE_CONTENT, 'utf-8');
     await fs.writeFile(prepPath, PREPARE_VERIFICATION_CONTENT, 'utf-8');
+    await fs.writeFile(secretsPath, SECRETS_SKILL_CONTENT, 'utf-8');
+    await fs.writeFile(rulesPath, SETUP_RULES_CONTENT, 'utf-8');
     return setupPath;
 }
 /**
  * Create the .claude/commands/ files for Claude Code slash commands
- * Users can invoke with /setup-haystack or /prepare-haystack
+ * Users can invoke with /setup-haystack, /prepare-haystack, or /setup-haystack-secrets
  */
 export async function createClaudeCommand() {
     const commandDir = path.join(process.cwd(), '.claude', 'commands');
     const setupPath = path.join(commandDir, 'setup-haystack.md');
     const prepPath = path.join(commandDir, 'prepare-haystack.md');
+    const secretsPath = path.join(commandDir, 'setup-haystack-secrets.md');
+    const rulesPath = path.join(commandDir, 'setup-haystack-rules.md');
     // Create directory if needed
     await fs.mkdir(commandDir, { recursive: true });
     // Write command files
     await fs.writeFile(setupPath, CLAUDE_COMMAND_CONTENT, 'utf-8');
     await fs.writeFile(prepPath, PREPARE_HAYSTACK_COMMAND, 'utf-8');
+    await fs.writeFile(secretsPath, SECRETS_COMMAND_CONTENT, 'utf-8');
+    await fs.writeFile(rulesPath, SETUP_RULES_COMMAND, 'utf-8');
     return setupPath;
 }