@biggora/claude-plugins 1.2.0 → 1.2.2

package/gemini-cli-workspace/iteration-1/eval-1-cicd-setup/without_skill/outputs/response.md

@@ -1,405 +0,0 @@
-# Setting Up Gemini CLI for Automated PR Reviews in GitHub Actions
-
-This guide covers how to configure Gemini CLI in a GitHub Actions CI/CD pipeline to automatically review pull requests, using an API key from GitHub Secrets and structured JSON output for parsing.
-
-## Prerequisites
-
-- A Google AI API key (from [Google AI Studio](https://aistudio.google.com/apikey))
-- The API key stored as a GitHub repository secret (e.g., `GEMINI_API_KEY`)
-- A GitHub repository with pull request workflows
-
-## Step 1: Store Your API Key in GitHub Secrets
-
-1. Go to your repository on GitHub.
-2. Navigate to **Settings > Secrets and variables > Actions**.
-3. Click **New repository secret**.
-4. Name it `GEMINI_API_KEY` and paste your API key as the value.
-
-## Step 2: Create the GitHub Actions Workflow
-
-Create a file at `.github/workflows/gemini-pr-review.yml`:
-
-```yaml
-name: Gemini PR Review
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-
-permissions:
-  contents: read
-  pull-requests: write
-
-jobs:
-  review:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set up Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '20'
-
-      - name: Install Gemini CLI
-        run: npm install -g @anthropic-ai/gemini-cli || npm install -g gemini-cli
-
-      - name: Get PR diff
-        id: diff
-        run: |
-          git fetch origin ${{ github.base_ref }}
-          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD)
-          # Write diff to a file to avoid shell escaping issues
-          echo "$DIFF" > /tmp/pr_diff.txt
-
-      - name: Get changed files list
-        id: files
-        run: |
-          git fetch origin ${{ github.base_ref }}
-          FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
-          echo "$FILES" > /tmp/changed_files.txt
-
-      - name: Run Gemini CLI review
-        id: gemini-review
-        env:
-          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
-        run: |
-          DIFF=$(cat /tmp/pr_diff.txt)
-          FILES=$(cat /tmp/changed_files.txt)
-
-          PROMPT=$(cat <<'PROMPT_EOF'
-          You are a senior code reviewer. Analyze the following pull request diff and provide a structured review.
-
-          Changed files:
-          FILE_LIST_PLACEHOLDER
-
-          Diff:
-          DIFF_PLACEHOLDER
-
-          Respond ONLY with valid JSON in this exact format (no markdown fencing, no extra text):
-          {
-            "summary": "Brief summary of the changes",
-            "risk_level": "low|medium|high",
-            "issues": [
-              {
-                "file": "path/to/file",
-                "line": 42,
-                "severity": "error|warning|suggestion",
-                "message": "Description of the issue",
-                "suggestion": "How to fix it"
-              }
-            ],
-            "positive_aspects": ["List of things done well"],
-            "approval_recommendation": "approve|request_changes|comment"
-          }
-          PROMPT_EOF
-          )
-
-          # Replace placeholders
-          PROMPT="${PROMPT/FILE_LIST_PLACEHOLDER/$FILES}"
-          PROMPT="${PROMPT/DIFF_PLACEHOLDER/$DIFF}"
-
-          # Run Gemini CLI with the prompt
-          # Using the --json flag if available, or parsing output directly
-          RESPONSE=$(echo "$PROMPT" | gemini \
-            --model gemini-2.0-flash \
-            --api-key "$GEMINI_API_KEY" \
-            2>/dev/null) || true
-
-          # If gemini CLI is not available, fall back to direct API call
-          if [ -z "$RESPONSE" ]; then
-            RESPONSE=$(curl -s "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${GEMINI_API_KEY}" \
-              -H 'Content-Type: application/json' \
-              -d "$(jq -n --arg prompt "$PROMPT" '{
-                "contents": [{"parts": [{"text": $prompt}]}],
-                "generationConfig": {
-                  "responseMimeType": "application/json",
-                  "temperature": 0.2
-                }
-              }')" | jq -r '.candidates[0].content.parts[0].text')
-          fi
-
-          echo "$RESPONSE" > /tmp/review_result.json
-
-          # Validate JSON
-          if jq empty /tmp/review_result.json 2>/dev/null; then
-            echo "valid_json=true" >> $GITHUB_OUTPUT
-          else
-            echo "valid_json=false" >> $GITHUB_OUTPUT
-            echo "Raw response was not valid JSON, attempting extraction..."
-            # Try to extract JSON from markdown fenced blocks
-            sed -n '/^```json/,/^```$/p' /tmp/review_result.json | sed '1d;$d' > /tmp/review_cleaned.json
-            if jq empty /tmp/review_cleaned.json 2>/dev/null; then
-              mv /tmp/review_cleaned.json /tmp/review_result.json
-              echo "valid_json=true" >> $GITHUB_OUTPUT
-            fi
-          fi
-
-      - name: Parse and post review
-        if: success()
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          REVIEW_FILE="/tmp/review_result.json"
-
-          if ! jq empty "$REVIEW_FILE" 2>/dev/null; then
-            echo "Failed to get valid JSON review. Posting raw output as comment."
-            gh pr comment ${{ github.event.pull_request.number }} \
-              --body "## Gemini Review
-
-          > Could not parse structured output. Raw response attached.
-
-          \`\`\`
-          $(cat "$REVIEW_FILE")
-          \`\`\`"
-            exit 0
-          fi
-
-          # Extract fields from JSON
-          SUMMARY=$(jq -r '.summary' "$REVIEW_FILE")
-          RISK=$(jq -r '.risk_level' "$REVIEW_FILE")
-          APPROVAL=$(jq -r '.approval_recommendation' "$REVIEW_FILE")
-          POSITIVES=$(jq -r '.positive_aspects // [] | map("- " + .) | join("\n")' "$REVIEW_FILE")
-          ISSUE_COUNT=$(jq -r '.issues | length' "$REVIEW_FILE")
-
-          # Build issue table
-          ISSUES_MD=""
-          if [ "$ISSUE_COUNT" -gt 0 ]; then
-            ISSUES_MD="### Issues Found ($ISSUE_COUNT)
-
-          | Severity | File | Line | Message | Suggestion |
-          |----------|------|------|---------|------------|
-          "
-            ISSUES_MD+=$(jq -r '.issues[] | "| \(.severity) | `\(.file)` | \(.line) | \(.message) | \(.suggestion) |"' "$REVIEW_FILE")
-          else
-            ISSUES_MD="### No issues found"
-          fi
-
-          # Build risk badge
-          case "$RISK" in
-            low)    RISK_BADGE="![Risk: Low](https://img.shields.io/badge/Risk-Low-green)" ;;
-            medium) RISK_BADGE="![Risk: Medium](https://img.shields.io/badge/Risk-Medium-yellow)" ;;
-            high)   RISK_BADGE="![Risk: High](https://img.shields.io/badge/Risk-High-red)" ;;
-            *)      RISK_BADGE="Risk: $RISK" ;;
-          esac
-
-          # Compose comment
-          COMMENT=$(cat <<EOF
-          ## Gemini Automated PR Review
-
-          $RISK_BADGE
-
-          ### Summary
-          $SUMMARY
-
-          $ISSUES_MD
-
-          ### Positive Aspects
-          $POSITIVES
-
-          ---
-          **Recommendation:** \`$APPROVAL\`
-          EOF
-          )
-
-          # Post comment to PR
-          gh pr comment ${{ github.event.pull_request.number }} --body "$COMMENT"
-```
-
-## Step 3: Direct API Approach with Structured JSON Output (Recommended)
-
-If the Gemini CLI does not reliably produce JSON, use the Gemini REST API directly with `responseMimeType` set to `application/json`. This forces the model to return valid JSON. Here is a standalone workflow that uses `curl` and `jq`:
-
-```yaml
-name: Gemini PR Review (API Direct)
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-
-permissions:
-  contents: read
-  pull-requests: write
-
-jobs:
-  review:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Generate review via Gemini API
-        id: review
-        env:
-          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
-        run: |
-          git fetch origin ${{ github.base_ref }}
-          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD | head -c 30000)
-          FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
-
-          # Build the prompt
-          PROMPT="You are a code reviewer. Review this PR diff. Changed files: ${FILES}. Diff: ${DIFF}. Return a JSON object with keys: summary (string), risk_level (low/medium/high), issues (array of {file, line, severity, message, suggestion}), positive_aspects (array of strings), approval_recommendation (approve/request_changes/comment)."
-
-          # Call Gemini API with structured output
-          RESPONSE=$(curl -sf \
-            "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${GEMINI_API_KEY}" \
-            -H 'Content-Type: application/json' \
-            -d "$(jq -n --arg prompt "$PROMPT" '{
-              contents: [{parts: [{text: $prompt}]}],
-              generationConfig: {
-                responseMimeType: "application/json",
-                responseSchema: {
-                  type: "object",
-                  properties: {
-                    summary: {type: "string"},
-                    risk_level: {type: "string", enum: ["low", "medium", "high"]},
-                    issues: {
-                      type: "array",
-                      items: {
-                        type: "object",
-                        properties: {
-                          file: {type: "string"},
-                          line: {type: "integer"},
-                          severity: {type: "string", enum: ["error", "warning", "suggestion"]},
-                          message: {type: "string"},
-                          suggestion: {type: "string"}
-                        },
-                        required: ["file", "severity", "message"]
-                      }
-                    },
-                    positive_aspects: {type: "array", items: {type: "string"}},
-                    approval_recommendation: {type: "string", enum: ["approve", "request_changes", "comment"]}
-                  },
-                  required: ["summary", "risk_level", "issues", "approval_recommendation"]
-                },
-                temperature: 0.2
-              }
-            }')")
-
-          # Extract the text content from the API response
-          REVIEW=$(echo "$RESPONSE" | jq -r '.candidates[0].content.parts[0].text')
-          echo "$REVIEW" > /tmp/review.json
-
-          # Validate
-          jq empty /tmp/review.json
-
-      - name: Post review comment
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          REVIEW_FILE="/tmp/review.json"
-
-          SUMMARY=$(jq -r '.summary' "$REVIEW_FILE")
-          RISK=$(jq -r '.risk_level' "$REVIEW_FILE")
-          APPROVAL=$(jq -r '.approval_recommendation' "$REVIEW_FILE")
-
-          # Format issues as markdown
-          ISSUES=$(jq -r '
-            if (.issues | length) > 0 then
-              "| Severity | File | Line | Issue | Fix |\n|---|---|---|---|---|\n" +
-              (.issues[] | "| \(.severity) | `\(.file)` | \(.line // "N/A") | \(.message) | \(.suggestion // "N/A") |")
-            else
-              "_No issues found._"
-            end
-          ' "$REVIEW_FILE")
-
-          POSITIVES=$(jq -r '(.positive_aspects // []) | map("- " + .) | join("\n")' "$REVIEW_FILE")
-
-          BODY=$(cat <<EOF
-          ## Automated PR Review (Gemini)
-
-          **Risk Level:** \`${RISK}\` | **Recommendation:** \`${APPROVAL}\`
-
-          ### Summary
-          ${SUMMARY}
-
-          ### Issues
-          ${ISSUES}
-
-          ### What looks good
-          ${POSITIVES}
-          EOF
-          )
-
-          gh pr comment "${{ github.event.pull_request.number }}" --body "$BODY"
-```
-
-## Step 4: Using Gemini CLI with `--json` Flag (If Supported)
-
-Some versions of the Gemini CLI support a `--json` output flag. If your version does, the invocation simplifies to:
-
-```bash
-gemini --model gemini-2.0-flash \
-  --api-key "$GEMINI_API_KEY" \
-  --json \
-  --prompt "Review this code diff and return structured JSON: $DIFF"
-```
-
-Check your installed version's help output (`gemini --help`) to confirm available flags.
-
-## Step 5: Parsing the JSON Output in Downstream Steps
-
-Once you have valid JSON in a file, you can use it in subsequent pipeline steps:
-
-```yaml
-      - name: Fail on high-risk reviews
-        run: |
-          RISK=$(jq -r '.risk_level' /tmp/review.json)
-          ERRORS=$(jq '[.issues[] | select(.severity == "error")] | length' /tmp/review.json)
-
-          echo "Risk level: $RISK"
-          echo "Error count: $ERRORS"
-
-          if [ "$RISK" = "high" ] || [ "$ERRORS" -gt 0 ]; then
-            echo "::error::PR review found high risk or errors. Please address the issues."
-            exit 1
-          fi
-```
-
-You can also use it in a matrix strategy or pass data between jobs:
-
-```yaml
-      - name: Export review data
-        id: export
-        run: |
-          echo "risk=$(jq -r '.risk_level' /tmp/review.json)" >> $GITHUB_OUTPUT
-          echo "approval=$(jq -r '.approval_recommendation' /tmp/review.json)" >> $GITHUB_OUTPUT
-          echo "issue_count=$(jq '.issues | length' /tmp/review.json)" >> $GITHUB_OUTPUT
-```
-
-Then reference in later steps or jobs:
-
-```yaml
-      - name: Auto-approve if clean
-        if: steps.export.outputs.approval == 'approve' && steps.export.outputs.issue_count == '0'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          gh pr review "${{ github.event.pull_request.number }}" --approve --body "Automated review passed with no issues."
-```
-
-## Key Points
-
-1. **`responseMimeType: "application/json"`** in the Gemini API `generationConfig` forces the model to return valid JSON. This is the most reliable way to get structured output.
-
-2. **`responseSchema`** lets you define the exact JSON schema you expect. The model will conform to it. This eliminates the need for post-processing or regex extraction.
-
-3. **Diff size limits**: Git diffs can be very large. Truncate them (e.g., `head -c 30000`) to stay within token limits. For large PRs, consider reviewing changed files individually.
-
-4. **Error handling**: Always validate the JSON output with `jq empty` before trying to parse it. Have a fallback path that posts raw output as a comment.
-
-5. **Rate limits**: The Gemini API has rate limits. For high-traffic repos, add retry logic or use the paid tier.
-
-6. **Security**: The `GEMINI_API_KEY` secret is never exposed in logs because GitHub Actions automatically masks secrets in output. The `GITHUB_TOKEN` is provided automatically by Actions with the permissions declared in the workflow.
-
-## Troubleshooting
-
-- **"jq: parse error"**: The model returned non-JSON text. Use `responseMimeType: "application/json"` to fix this.
-- **Empty response**: Check that `GEMINI_API_KEY` is set correctly in secrets and that the model name is valid.
-- **Diff too large**: Truncate the diff or split the review across multiple API calls, one per file.
-- **Permission denied on PR comment**: Ensure `permissions: pull-requests: write` is set in the workflow.

package/gemini-cli-workspace/iteration-1/eval-1-cicd-setup/without_skill/timing.json

@@ -1,5 +0,0 @@
-{
-  "total_tokens": 60035,
-  "duration_ms": 77857,
-  "total_duration_seconds": 77.9
-}

package/gemini-cli-workspace/iteration-1/eval-2-mcp-server-config/eval_metadata.json

@@ -1,37 +0,0 @@
-{
-  "eval_id": 2,
-  "eval_name": "mcp-server-config",
-  "prompt": "I need to connect a custom MCP server I built in Python to gemini cli. The server needs an API key from my environment. Also I want to restrict which tools it can expose. How do I configure this?",
-  "assertions": [
-    {
-      "id": "correct-settings-location",
-      "text": "Points to ~/.gemini/settings.json or project-level settings.json",
-      "type": "content_check"
-    },
-    {
-      "id": "shows-mcp-config-structure",
-      "text": "Shows mcpServers config with command, args fields",
-      "type": "content_check"
-    },
-    {
-      "id": "env-var-dollar-pattern",
-      "text": "Shows $VAR pattern for environment variable references in env field",
-      "type": "content_check"
-    },
-    {
-      "id": "tool-filtering",
-      "text": "Explains includeTools and/or excludeTools for restricting exposed tools",
-      "type": "content_check"
-    },
-    {
-      "id": "security-note",
-      "text": "Mentions security considerations (trust field, not hardcoding keys, etc.)",
-      "type": "content_check"
-    },
-    {
-      "id": "python-command-correct",
-      "text": "Uses correct Python command (python/python3) in the command field for the MCP server",
-      "type": "accuracy_check"
-    }
-  ]
-}

package/gemini-cli-workspace/iteration-1/eval-2-mcp-server-config/with_skill/grading.json

@@ -1,37 +0,0 @@
-{
-  "eval_id": 2,
-  "eval_name": "mcp-server-config",
-  "configuration": "with_skill",
-  "expectations": [
-    {
-      "text": "Points to ~/.gemini/settings.json or project-level settings.json",
-      "passed": true,
-      "evidence": "Mentions both `~/.gemini/settings.json` (global) and `<project>/.gemini/settings.json` (project-scoped)"
-    },
-    {
-      "text": "Shows mcpServers config with command, args fields",
-      "passed": true,
-      "evidence": "Complete mcpServers config with command, args, cwd, env, timeout, trust, includeTools fields"
-    },
-    {
-      "text": "Shows $VAR pattern for environment variable references in env field",
-      "passed": true,
-      "evidence": "Shows `\"MY_API_KEY\": \"$MY_API_KEY\"` and explains the $VAR_NAME pattern explicitly"
-    },
-    {
-      "text": "Explains includeTools and/or excludeTools for restricting exposed tools",
-      "passed": true,
-      "evidence": "Explains both includeTools (whitelist) and excludeTools (blacklist) with examples and guidance on when to use each"
-    },
-    {
-      "text": "Mentions security considerations (trust field, not hardcoding keys, etc.)",
-      "passed": true,
-      "evidence": "Warns to never hardcode secrets, explains trust field, mentions auto-redaction of sensitive env vars"
-    },
-    {
-      "text": "Uses correct Python command (python/python3) in the command field",
-      "passed": true,
-      "evidence": "Uses both `python` and `python3` with note about system differences"
-    }
-  ]
-}