@biggora/claude-plugins 1.2.0 → 1.2.2

package/gemini-cli-workspace/iteration-2/eval-1-cicd-setup/without_skill/outputs/response.md

@@ -1,377 +0,0 @@
-# Setting Up Gemini CLI for Automated PR Reviews in GitHub Actions
-
-## Overview
-
-This guide covers how to configure Gemini CLI in a GitHub Actions CI/CD pipeline to automatically review pull requests, using an API key from GitHub Secrets and structured JSON output for parsing.
-
-## Prerequisites
-
-1. A Google AI API key (from Google AI Studio)
-2. The API key stored as a GitHub Actions secret (e.g., `GEMINI_API_KEY`)
-3. A GitHub repository with pull requests to review
-
-## Step 1: Store Your API Key in GitHub Secrets
-
-1. Go to your repository on GitHub
-2. Navigate to **Settings > Secrets and variables > Actions**
-3. Click **New repository secret**
-4. Name it `GEMINI_API_KEY` and paste your API key
-5. Click **Add secret**
-
-## Step 2: Create the GitHub Actions Workflow
-
-Create a file at `.github/workflows/gemini-pr-review.yml`:
-
-```yaml
-name: Gemini PR Review
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-
-permissions:
-  contents: read
-  pull-requests: write
-
-jobs:
-  review:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '20'
-
-      - name: Install Gemini CLI
-        run: npm install -g @google/gemini-cli
-
-      - name: Get PR diff
-        id: diff
-        run: |
-          git diff origin/${{ github.base_ref }}...HEAD > pr_diff.txt
-          echo "diff_size=$(wc -c < pr_diff.txt)" >> $GITHUB_OUTPUT
-
-      - name: Run Gemini Review
-        id: review
-        env:
-          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
-        run: |
-          DIFF_CONTENT=$(cat pr_diff.txt)
-
-          # Create the prompt requesting structured JSON output
-          cat > review_prompt.txt << 'PROMPT_EOF'
-          You are a code reviewer. Analyze the following PR diff and provide your review as a JSON object with this exact structure:
-
-          {
-            "summary": "Brief overall summary of the changes",
-            "risk_level": "low|medium|high",
-            "issues": [
-              {
-                "severity": "critical|warning|suggestion",
-                "file": "path/to/file",
-                "line": 0,
-                "description": "Description of the issue",
-                "suggestion": "Suggested fix"
-              }
-            ],
-            "positives": ["List of good practices observed"],
-            "approved": true
-          }
-
-          Respond ONLY with valid JSON, no markdown fences, no extra text.
-
-          Here is the diff:
-          PROMPT_EOF
-
-          # Append the diff content
-          cat pr_diff.txt >> review_prompt.txt
-
-          # Run Gemini CLI with the prompt
-          # Use --json flag if available, otherwise parse text output
-          REVIEW_OUTPUT=$(gemini -p "$(cat review_prompt.txt)" 2>/dev/null || true)
-
-          # Save raw output for debugging
-          echo "$REVIEW_OUTPUT" > review_output.json
-
-          # Validate JSON
-          if echo "$REVIEW_OUTPUT" | jq . > /dev/null 2>&1; then
-            echo "valid_json=true" >> $GITHUB_OUTPUT
-          else
-            echo "valid_json=false" >> $GITHUB_OUTPUT
-            # Try to extract JSON from the response (in case of markdown fences)
-            REVIEW_OUTPUT=$(echo "$REVIEW_OUTPUT" | sed -n '/^{/,/^}/p' | head -1)
-            echo "$REVIEW_OUTPUT" > review_output.json
-          fi
-
-      - name: Parse and Post Review
-        if: always()
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const fs = require('fs');
-
-            let review;
-            try {
-              const rawOutput = fs.readFileSync('review_output.json', 'utf8');
-              // Strip markdown code fences if present
-              const cleaned = rawOutput
-                .replace(/```json\n?/g, '')
-                .replace(/```\n?/g, '')
-                .trim();
-              review = JSON.parse(cleaned);
-            } catch (e) {
-              console.log('Failed to parse review JSON:', e.message);
-              // Post a fallback comment
-              await github.rest.issues.createComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: context.issue.number,
-                body: '## Gemini Review\n\n_Review output could not be parsed as JSON. Raw output saved as artifact._'
-              });
-              return;
-            }
-
-            // Build the review comment
-            let body = `## Gemini PR Review\n\n`;
-            body += `**Summary:** ${review.summary}\n\n`;
-            body += `**Risk Level:** ${review.risk_level}\n\n`;
-
-            if (review.issues && review.issues.length > 0) {
-              body += `### Issues Found (${review.issues.length})\n\n`;
-              for (const issue of review.issues) {
-                const emoji = issue.severity === 'critical' ? '🔴' :
-                              issue.severity === 'warning' ? '🟡' : '🔵';
-                body += `${emoji} **${issue.severity.toUpperCase()}** - \`${issue.file}\``;
-                if (issue.line) body += ` (line ${issue.line})`;
-                body += `\n`;
-                body += `> ${issue.description}\n`;
-                if (issue.suggestion) {
-                  body += `> **Suggestion:** ${issue.suggestion}\n`;
-                }
-                body += `\n`;
-              }
-            } else {
-              body += `### No Issues Found\n\n`;
-            }
-
-            if (review.positives && review.positives.length > 0) {
-              body += `### Positives\n\n`;
-              for (const positive of review.positives) {
-                body += `- ${positive}\n`;
-              }
-              body += `\n`;
-            }
-
-            body += `---\n_Automated review by Gemini CLI_`;
-
-            await github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-              body: body
-            });
-
-      - name: Upload Review Artifact
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: gemini-review
-          path: review_output.json
-          retention-days: 7
-```
-
-## Step 3: Alternative Approach Using the Gemini API Directly (More Reliable for JSON)
-
-If you want more control over structured output, you can call the Gemini API directly with `curl` instead of relying on the CLI. This gives you access to the `response_mime_type` parameter for guaranteed JSON:
-
-```yaml
-      - name: Run Gemini Review via API
-        id: review
-        env:
-          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
-        run: |
-          DIFF_CONTENT=$(cat pr_diff.txt | jq -Rs .)
-
-          # Build the API request with structured output
-          cat > request.json << EOF
-          {
-            "contents": [{
-              "parts": [{
-                "text": "You are a code reviewer. Analyze this PR diff and provide a review.\n\nDiff:\n${DIFF_CONTENT}"
-              }]
-            }],
-            "generationConfig": {
-              "responseMimeType": "application/json",
-              "responseSchema": {
-                "type": "object",
-                "properties": {
-                  "summary": { "type": "string" },
-                  "risk_level": { "type": "string", "enum": ["low", "medium", "high"] },
-                  "issues": {
-                    "type": "array",
-                    "items": {
-                      "type": "object",
-                      "properties": {
-                        "severity": { "type": "string", "enum": ["critical", "warning", "suggestion"] },
-                        "file": { "type": "string" },
-                        "line": { "type": "integer" },
-                        "description": { "type": "string" },
-                        "suggestion": { "type": "string" }
-                      },
-                      "required": ["severity", "file", "description"]
-                    }
-                  },
-                  "positives": {
-                    "type": "array",
-                    "items": { "type": "string" }
-                  },
-                  "approved": { "type": "boolean" }
-                },
-                "required": ["summary", "risk_level", "issues", "approved"]
-              }
-            }
-          }
-          EOF
-
-          # Call the Gemini API
-          curl -s "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${GEMINI_API_KEY}" \
-            -H 'Content-Type: application/json' \
-            -d @request.json \
-            -o api_response.json
-
-          # Extract the text content from the API response
-          jq -r '.candidates[0].content.parts[0].text' api_response.json > review_output.json
-```
-
-## Step 4: Handling Large Diffs
-
-For large PRs, you may hit token limits. Add a truncation step:
-
-```yaml
-      - name: Prepare diff (truncated if needed)
-        run: |
-          # Limit diff to ~100KB to stay within token limits
-          if [ $(wc -c < pr_diff.txt) -gt 100000 ]; then
-            head -c 100000 pr_diff.txt > pr_diff_truncated.txt
-            echo -e "\n\n[DIFF TRUNCATED - showing first 100KB]" >> pr_diff_truncated.txt
-            mv pr_diff_truncated.txt pr_diff.txt
-          fi
-```
-
-## Key Configuration Notes
-
-### Authentication
-
-- Gemini CLI uses the `GEMINI_API_KEY` environment variable for authentication
-- In GitHub Actions, pass it via `env:` from secrets -- never hardcode it
-
-### Structured JSON Output
-
-There are two approaches for getting structured JSON:
-
-1. **Prompt-based** (CLI approach): Ask the model to return JSON in your prompt. Less reliable but simpler. Include "Respond ONLY with valid JSON" in your prompt.
-
-2. **Schema-based** (API approach): Use `responseMimeType: "application/json"` with a `responseSchema` in the API request. This guarantees valid JSON output conforming to your schema. This is the more reliable approach for CI/CD pipelines.
-
-### Parsing the Output
-
-Always include fallback handling when parsing JSON from LLM output:
-
-```bash
-# Strip potential markdown code fences
-CLEAN_JSON=$(echo "$RAW_OUTPUT" | sed 's/^```json//; s/^```//; /^$/d')
-
-# Validate with jq
-if echo "$CLEAN_JSON" | jq . > /dev/null 2>&1; then
-  echo "Valid JSON"
-else
-  echo "Invalid JSON - falling back"
-fi
-```
-
-### Cost and Rate Limiting
-
-- Consider adding rate limiting or caching to avoid excessive API calls
-- Use `concurrency` in your workflow to prevent parallel runs:
-
-```yaml
-concurrency:
-  group: gemini-review-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
-```
-
-### Security Considerations
-
-- Never log or echo the API key in workflow output
-- Use `pull_requests: write` permission (minimum required)
-- Consider limiting the workflow to specific file types or paths:
-
-```yaml
-on:
-  pull_request:
-    paths:
-      - '**.js'
-      - '**.ts'
-      - '**.py'
-```
-
-## Complete Minimal Example
-
-For a quick start, here is the simplest possible setup:
-
-```yaml
-name: Gemini Review
-on:
-  pull_request:
-    types: [opened, synchronize]
-
-permissions:
-  contents: read
-  pull-requests: write
-
-jobs:
-  review:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Setup and Review
-        env:
-          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
-        run: |
-          npm install -g @google/gemini-cli
-          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD)
-          REVIEW=$(gemini -p "Review this code diff. Return JSON with keys: summary (string), issues (array of {severity, file, description}), approved (boolean). Only output JSON. Diff: $DIFF")
-          echo "$REVIEW" > review.json
-
-      - name: Post Comment
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const fs = require('fs');
-            const raw = fs.readFileSync('review.json', 'utf8');
-            const clean = raw.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
-            let review;
-            try { review = JSON.parse(clean); } catch(e) {
-              return github.rest.issues.createComment({
-                ...context.repo, issue_number: context.issue.number,
-                body: '## Gemini Review\nFailed to parse output.'
-              });
-            }
-            let body = `## Gemini Review\n**Summary:** ${review.summary}\n\n`;
-            if (review.issues?.length) {
-              body += review.issues.map(i => `- **${i.severity}** \`${i.file}\`: ${i.description}`).join('\n');
-            } else { body += 'No issues found.'; }
-            await github.rest.issues.createComment({
-              ...context.repo, issue_number: context.issue.number, body
-            });
-```
-
-This setup will automatically run Gemini CLI on every PR, parse the structured JSON review, and post the results as a PR comment.

package/gemini-cli-workspace/iteration-2/eval-1-cicd-setup/without_skill/timing.json

@@ -1,5 +0,0 @@
-{
-  "total_tokens": 59322,
-  "duration_ms": 67990,
-  "total_duration_seconds": 68.0
-}

package/gemini-cli-workspace/iteration-2/eval-2-mcp-server-config/eval_metadata.json

@@ -1,37 +0,0 @@
-{
-  "eval_id": 2,
-  "eval_name": "mcp-server-config",
-  "prompt": "I need to connect a custom MCP server I built in Python to gemini cli. The server needs an API key from my environment. Also I want to restrict which tools it can expose. How do I configure this?",
-  "assertions": [
-    {
-      "id": "correct-settings-location",
-      "text": "Points to ~/.gemini/settings.json or project-level settings.json",
-      "type": "content_check"
-    },
-    {
-      "id": "shows-mcp-config-structure",
-      "text": "Shows mcpServers config with command, args fields",
-      "type": "content_check"
-    },
-    {
-      "id": "env-var-dollar-pattern",
-      "text": "Shows $VAR pattern for environment variable references in env field",
-      "type": "content_check"
-    },
-    {
-      "id": "tool-filtering",
-      "text": "Explains includeTools and/or excludeTools for restricting exposed tools",
-      "type": "content_check"
-    },
-    {
-      "id": "security-note",
-      "text": "Mentions security considerations (trust field, not hardcoding keys, etc.)",
-      "type": "content_check"
-    },
-    {
-      "id": "python-command-correct",
-      "text": "Uses correct Python command (python/python3) in the command field for the MCP server",
-      "type": "accuracy_check"
-    }
-  ]
-}

package/gemini-cli-workspace/iteration-2/eval-2-mcp-server-config/with_skill/grading.json

@@ -1,37 +0,0 @@
-{
-  "eval_id": 2,
-  "eval_name": "mcp-server-config",
-  "config": "with_skill",
-  "expectations": [
-    {
-      "text": "Points to ~/.gemini/settings.json or project-level settings.json",
-      "passed": true,
-      "evidence": "Opens with: 'Add your server to ~/.gemini/settings.json (global) or <project>/.gemini/settings.json (project-scoped)'"
-    },
-    {
-      "text": "Shows mcpServers config with command, args fields",
-      "passed": true,
-      "evidence": "Full JSON config block with mcpServers containing command, args, cwd, env, timeout, trust, includeTools"
-    },
-    {
-      "text": "Shows $VAR pattern for environment variable references in env field",
-      "passed": true,
-      "evidence": "Shows \"API_KEY\": \"$MY_API_KEY\" pattern and explains the dollar-sign prefix syntax"
-    },
-    {
-      "text": "Explains includeTools and/or excludeTools for restricting exposed tools",
-      "passed": true,
-      "evidence": "Dedicated section explaining both includeTools (whitelist) and excludeTools (blacklist) with examples"
-    },
-    {
-      "text": "Mentions security considerations (trust field, not hardcoding keys, etc.)",
-      "passed": true,
-      "evidence": "Explains trust field (default false, requires confirmation), warns never to hardcode secrets, mentions automatic redaction of env vars"
-    },
-    {
-      "text": "Uses correct Python command (python/python3) in the command field for the MCP server",
-      "passed": true,
-      "evidence": "Uses \"command\": \"python\" consistently across examples"
-    }
-  ]
-}

package/gemini-cli-workspace/iteration-2/eval-2-mcp-server-config/with_skill/outputs/response.md

@@ -1,127 +0,0 @@
-# Connecting a Custom Python MCP Server to Gemini CLI
-
-## Configuration
-
-Add your server to `~/.gemini/settings.json` (global) or `<project>/.gemini/settings.json` (project-scoped) under the `mcpServers` key:
-
-```json
-{
-  "mcpServers": {
-    "myPythonServer": {
-      "command": "python",
-      "args": ["-m", "my_mcp_server"],
-      "cwd": "./mcp_tools",
-      "env": {
-        "API_KEY": "$MY_API_KEY"
-      },
-      "timeout": 15000,
-      "trust": false,
-      "includeTools": ["tool_a", "tool_b"]
-    }
-  }
-}
-```
-
-## Breakdown of each field
-
-| Field | Purpose |
-|---|---|
-| `command` | The executable to run. Use `"python"` (or `"python3"` depending on your system). |
-| `args` | Arguments passed to the command. Use `["-m", "my_mcp_server"]` to run as a module, or `["./path/to/server.py"]` to run a script directly. |
-| `cwd` | Optional working directory for the server process. |
-| `env` | Environment variables passed to the server. Use the `"$VAR_NAME"` syntax to reference variables from your shell environment rather than hardcoding secrets. |
-| `timeout` | Connection timeout in milliseconds. Defaults to 600000 (10 minutes). Set lower (e.g., 15000) for servers that should start quickly. |
-| `trust` | When `false` (the default), Gemini CLI will prompt you for confirmation before executing tools from this server. Set to `true` only for servers you fully control and trust. |
-| `includeTools` | A whitelist array. Only the tools listed here will be exposed to Gemini. All other tools from the server are hidden. |
-| `excludeTools` | A blacklist array. The listed tools are hidden; everything else is exposed. |
-
-Use `includeTools` or `excludeTools`, not both. `includeTools` is the safer choice when you want to tightly restrict what the model can call.
-
-## Passing the API key securely
-
-Never hardcode secrets in `settings.json`. Instead, set the variable in your shell environment:
-
-```bash
-export MY_API_KEY="sk-your-actual-key"
-```
-
-Then reference it with the dollar-sign prefix in the config:
-
-```json
-"env": {
-  "API_KEY": "$MY_API_KEY"
-}
-```
-
-Gemini CLI automatically redacts environment variable values that match patterns like `*TOKEN*`, `*SECRET*`, `*KEY*` from MCP process output, so your key will not leak into logs.
-
-You can also store the variable in `~/.gemini/.env` or `<project>/.gemini/.env` and it will be loaded automatically.
-
-## Restricting exposed tools
-
-To allow only specific tools (whitelist approach):
-
-```json
-"includeTools": ["query_data", "list_items"]
-```
-
-To block specific tools while allowing everything else (blacklist approach):
-
-```json
-"excludeTools": ["delete_all", "admin_reset"]
-```
-
-## Verifying the connection
-
-After configuring, start Gemini CLI and check the server status:
-
-```bash
-gemini
-```
-
-Then inside the interactive session:
-
-```
-/mcp list
-/mcp status
-```
-
-You can also manage servers from the command line:
-
-```bash
-gemini mcp list
-gemini mcp remove myPythonServer
-```
-
-## Invoking your server's tools
-
-In the interactive session, reference the server by name with the `@` prefix:
-
-```
-@myPythonServer query all active users
-```
-
-Gemini CLI will route the request to the appropriate tool exposed by your MCP server.
-
-## Complete example
-
-Assuming your Python MCP server is at `./tools/my_server.py` and exposes tools named `search`, `fetch_details`, and `delete_record`, and you only want to allow `search` and `fetch_details`:
-
-```json
-{
-  "mcpServers": {
-    "myPythonServer": {
-      "command": "python",
-      "args": ["./tools/my_server.py"],
-      "env": {
-        "API_KEY": "$MY_API_KEY"
-      },
-      "timeout": 15000,
-      "trust": false,
-      "includeTools": ["search", "fetch_details"]
-    }
-  }
-}
-```
-
-This configuration starts the Python server, passes the API key from your environment, sets a 15-second connection timeout, requires confirmation prompts for tool calls, and only exposes the `search` and `fetch_details` tools to Gemini.

package/gemini-cli-workspace/iteration-2/eval-2-mcp-server-config/with_skill/timing.json

@@ -1,5 +0,0 @@
-{
-  "total_tokens": 66522,
-  "duration_ms": 47343,
-  "total_duration_seconds": 47.3
-}

package/gemini-cli-workspace/iteration-2/eval-2-mcp-server-config/without_skill/grading.json

@@ -1,37 +0,0 @@
-{
-  "eval_id": 2,
-  "eval_name": "mcp-server-config",
-  "config": "without_skill",
-  "expectations": [
-    {
-      "text": "Points to ~/.gemini/settings.json or project-level settings.json",
-      "passed": true,
-      "evidence": "States '~/.gemini/settings.json' (global) and '.gemini/settings.json' (project-level)"
-    },
-    {
-      "text": "Shows mcpServers config with command, args fields",
-      "passed": true,
-      "evidence": "Full JSON config with mcpServers, command, args, env, cwd, timeout, includeTools"
-    },
-    {
-      "text": "Shows $VAR pattern for environment variable references in env field",
-      "passed": true,
-      "evidence": "Shows both $MY_API_KEY and ${ANOTHER_ENV_VAR} syntax with explanation"
-    },
-    {
-      "text": "Explains includeTools and/or excludeTools for restricting exposed tools",
-      "passed": true,
-      "evidence": "Dedicated sections for both includeTools (allowlist) and excludeTools (blocklist) with examples and precedence rules"
-    },
-    {
-      "text": "Mentions security considerations (trust field, not hardcoding keys, etc.)",
-      "passed": true,
-      "evidence": "Mentions trust field, automatic redaction, env variable security note about trusted declarations"
-    },
-    {
-      "text": "Uses correct Python command (python/python3) in the command field for the MCP server",
-      "passed": true,
-      "evidence": "Uses \"command\": \"python\" consistently, also shows uv-based FastMCP alternative"
-    }
-  ]
-}