@biggora/claude-plugins 1.2.0 → 1.2.2

package/gemini-cli-workspace/iteration-2/benchmark.json

@@ -1,173 +0,0 @@
-{
-  "metadata": {
-    "skill_name": "gemini-cli",
-    "skill_path": "src/skills/gemini-cli/",
-    "executor_model": "claude-opus-4-6",
-    "analyzer_model": "claude-opus-4-6",
-    "timestamp": "2026-03-11T18:15:00Z",
-    "evals_run": [1, 2, 3],
-    "runs_per_configuration": 3
-  },
-  "runs": [
-    {
-      "eval_id": 1,
-      "eval_name": "cicd-setup",
-      "configuration": "with_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 6,
-        "failed": 0,
-        "total": 6,
-        "time_seconds": 53.6,
-        "tokens": 64216
-      },
-      "expectations": [
-        {"text": "Uses -p or --prompt flag for headless/non-interactive mode", "passed": true, "evidence": "Line 53: gemini -p with --output-format json"},
-        {"text": "Includes --output-format json flag for structured output", "passed": true, "evidence": "Line 62: --output-format json flag used correctly"},
-        {"text": "Shows how to set GEMINI_API_KEY from GitHub secrets", "passed": true, "evidence": "Line 42: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}"},
-        {"text": "Provides a GitHub Actions workflow YAML snippet", "passed": true, "evidence": "Complete workflow YAML with checkout, node setup, install, review, and comment steps"},
-        {"text": "Shows how to parse JSON response (e.g., with jq .response)", "passed": true, "evidence": "Line 65: jq -r '.response' and line 68: token stats extraction"},
-        {"text": "Does not invent non-existent CLI flags or options", "passed": true, "evidence": "Only uses verified flags: -p and --output-format json. Exit codes match corrected values."}
-      ],
-      "notes": []
-    },
-    {
-      "eval_id": 1,
-      "eval_name": "cicd-setup",
-      "configuration": "without_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 0.6667,
-        "passed": 4,
-        "failed": 2,
-        "total": 6,
-        "time_seconds": 68.0,
-        "tokens": 59322
-      },
-      "expectations": [
-        {"text": "Uses -p or --prompt flag for headless/non-interactive mode", "passed": true, "evidence": "Line 96: gemini -p"},
-        {"text": "Includes --output-format json flag for structured output", "passed": false, "evidence": "Does not use --output-format json. Line 95 comment mentions '--json flag if available' which is not a real flag."},
-        {"text": "Shows how to set GEMINI_API_KEY from GitHub secrets", "passed": true, "evidence": "Line 62: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}"},
-        {"text": "Provides a GitHub Actions workflow YAML snippet", "passed": true, "evidence": "Complete workflow YAML provided"},
-        {"text": "Shows how to parse JSON response (e.g., with jq .response)", "passed": true, "evidence": "Uses jq for JSON validation and parsing"},
-        {"text": "Does not invent non-existent CLI flags or options", "passed": false, "evidence": "Line 95: references '--json flag if available' which is not a real Gemini CLI flag"}
-      ],
-      "notes": []
-    },
-    {
-      "eval_id": 2,
-      "eval_name": "mcp-server-config",
-      "configuration": "with_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 6,
-        "failed": 0,
-        "total": 6,
-        "time_seconds": 47.3,
-        "tokens": 66522
-      },
-      "expectations": [
-        {"text": "Points to ~/.gemini/settings.json or project-level settings.json", "passed": true, "evidence": "Opens with ~/.gemini/settings.json (global) or <project>/.gemini/settings.json (project-scoped)"},
-        {"text": "Shows mcpServers config with command, args fields", "passed": true, "evidence": "Full JSON config with mcpServers, command, args, env, timeout, trust, includeTools"},
-        {"text": "Shows $VAR pattern for environment variable references in env field", "passed": true, "evidence": "Shows \"API_KEY\": \"$MY_API_KEY\" pattern"},
-        {"text": "Explains includeTools and/or excludeTools for restricting exposed tools", "passed": true, "evidence": "Dedicated section for both includeTools and excludeTools with examples"},
-        {"text": "Mentions security considerations (trust field, not hardcoding keys, etc.)", "passed": true, "evidence": "Explains trust field, warns never to hardcode secrets, mentions env var redaction"},
-        {"text": "Uses correct Python command (python/python3) in the command field", "passed": true, "evidence": "Uses \"command\": \"python\" consistently"}
-      ],
-      "notes": []
-    },
-    {
-      "eval_id": 2,
-      "eval_name": "mcp-server-config",
-      "configuration": "without_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 6,
-        "failed": 0,
-        "total": 6,
-        "time_seconds": 86.7,
-        "tokens": 61417
-      },
-      "expectations": [
-        {"text": "Points to ~/.gemini/settings.json or project-level settings.json", "passed": true, "evidence": "States ~/.gemini/settings.json (global) and .gemini/settings.json (project-level)"},
-        {"text": "Shows mcpServers config with command, args fields", "passed": true, "evidence": "Full JSON config with mcpServers, command, args, env"},
-        {"text": "Shows $VAR pattern for environment variable references in env field", "passed": true, "evidence": "Shows both $MY_API_KEY and ${ANOTHER_ENV_VAR} syntax"},
-        {"text": "Explains includeTools and/or excludeTools for restricting exposed tools", "passed": true, "evidence": "Dedicated sections for includeTools and excludeTools with examples"},
-        {"text": "Mentions security considerations (trust field, not hardcoding keys, etc.)", "passed": true, "evidence": "Mentions trust field, automatic redaction, env variable security"},
-        {"text": "Uses correct Python command (python/python3) in the command field", "passed": true, "evidence": "Uses \"command\": \"python\" consistently"}
-      ],
-      "notes": []
-    },
-    {
-      "eval_id": 3,
-      "eval_name": "custom-slash-command",
-      "configuration": "with_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 5,
-        "failed": 0,
-        "total": 5,
-        "time_seconds": 43.7,
-        "tokens": 63047
-      },
-      "expectations": [
-        {"text": "Points to ~/.gemini/commands/ for global user-scoped commands", "passed": true, "evidence": "States ~/.gemini/commands/commit.toml for global commands"},
-        {"text": "Shows .toml file format with prompt field", "passed": true, "evidence": "Complete .toml example with description and prompt fields"},
-        {"text": "Uses !{git diff --cached} shell execution syntax in the prompt", "passed": true, "evidence": "Uses !{git diff --cached} and explains the syntax"},
-        {"text": "Warns that custom slash commands don't work in headless/non-interactive mode", "passed": true, "evidence": "States 'Custom slash commands work in interactive mode only. They are not available in headless mode (-p flag).'"},
-        {"text": "Provides a complete, copy-pasteable .toml file example", "passed": true, "evidence": "Complete commit.toml ready to copy-paste"}
-      ],
-      "notes": []
-    },
-    {
-      "eval_id": 3,
-      "eval_name": "custom-slash-command",
-      "configuration": "without_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 0.8,
-        "passed": 4,
-        "failed": 1,
-        "total": 5,
-        "time_seconds": 73.6,
-        "tokens": 59908
-      },
-      "expectations": [
-        {"text": "Points to ~/.gemini/commands/ for global user-scoped commands", "passed": true, "evidence": "States ~/.gemini/commands/ for global commands"},
-        {"text": "Shows .toml file format with prompt field", "passed": true, "evidence": "Complete .toml example with description and prompt fields"},
-        {"text": "Uses !{git diff --cached} shell execution syntax in the prompt", "passed": true, "evidence": "Uses !{git diff --cached} in the prompt field"},
-        {"text": "Warns that custom slash commands don't work in headless/non-interactive mode", "passed": false, "evidence": "No mention of headless mode limitations anywhere in the response"},
-        {"text": "Provides a complete, copy-pasteable .toml file example", "passed": true, "evidence": "Complete commit.toml file provided"}
-      ],
-      "notes": []
-    }
-  ],
-  "run_summary": {
-    "with_skill": {
-      "pass_rate": {"mean": 1.0, "stddev": 0.0, "min": 1.0, "max": 1.0},
-      "time_seconds": {"mean": 48.2, "stddev": 5.0, "min": 43.7, "max": 53.6},
-      "tokens": {"mean": 64595, "stddev": 1741, "min": 63047, "max": 66522}
-    },
-    "without_skill": {
-      "pass_rate": {"mean": 0.822, "stddev": 0.167, "min": 0.667, "max": 1.0},
-      "time_seconds": {"mean": 76.1, "stddev": 9.7, "min": 68.0, "max": 86.7},
-      "tokens": {"mean": 60216, "stddev": 1372, "min": 59322, "max": 61417}
-    },
-    "delta": {
-      "pass_rate": "+0.18",
-      "time_seconds": "-27.9",
-      "tokens": "+4379"
-    }
-  },
-  "notes": [
-    "With-skill achieves 100% pass rate across all 3 evals (17/17 assertions) vs 82.2% baseline (14/17)",
-    "Eval 2 (MCP server config) is non-discriminating — both configs pass all 6 assertions. This eval could be made harder.",
-    "With-skill is ~28s faster on average despite using ~4k more tokens, suggesting more focused responses",
-    "Baseline failures are accuracy-related: hallucinated --json flag (eval 1) and missing headless limitation warning (eval 3)",
-    "The skill's value is strongest on accuracy assertions — it prevents hallucination of non-existent flags and ensures important caveats are mentioned"
-  ]
-}

package/gemini-cli-workspace/iteration-2/benchmark.md

@@ -1,28 +0,0 @@
-# Skill Benchmark: gemini-cli
-
-**Model**: claude-opus-4-6
-**Date**: 2026-03-11T18:15:00Z
-**Evals**: 1, 2, 3 (1 run each per configuration)
-
-## Summary
-
-| Metric | With Skill | Without Skill | Delta |
-|--------|------------|---------------|-------|
-| Pass Rate | 100% ± 0% | 82% ± 17% | +0.18 |
-| Time | 48.2s ± 5.0s | 76.1s ± 9.7s | -27.9s |
-| Tokens | 64595 ± 1741 | 60216 ± 1372 | +4379 |
-
-## Per-Eval Breakdown
-
-| Eval | With Skill | Without Skill |
-|------|-----------|---------------|
-| 1: CI/CD Setup | 6/6 (100%) | 4/6 (67%) |
-| 2: MCP Server Config | 6/6 (100%) | 6/6 (100%) |
-| 3: Custom Slash Command | 5/5 (100%) | 4/5 (80%) |
-
-## Analyst Notes
-
-- **Eval 2 is non-discriminating**: Both configs pass all 6 assertions. Consider adding harder assertions (e.g., correct timeout default of 600000ms, httpUrl transport option).
-- **Baseline failures are accuracy-related**: The without-skill run hallucinated a `--json` flag (eval 1) and omitted the headless limitation warning (eval 3). These are exactly the kinds of errors the skill prevents.
-- **Speed advantage**: With-skill runs are ~28s faster on average despite using ~4k more tokens, suggesting the skill helps produce more focused, direct responses.
-- **The skill's value is strongest on accuracy**: It prevents hallucination of non-existent flags and ensures important caveats are mentioned.

package/gemini-cli-workspace/iteration-2/eval-1-cicd-setup/eval_metadata.json

@@ -1,37 +0,0 @@
-{
-  "eval_id": 1,
-  "eval_name": "cicd-setup",
-  "prompt": "I want to set up gemini cli in my CI/CD pipeline on GitHub Actions so it can automatically review PRs. We use an API key stored in GitHub secrets. How do I set this up with structured JSON output so I can parse the review?",
-  "assertions": [
-    {
-      "id": "mentions-p-flag",
-      "text": "Uses -p or --prompt flag for headless/non-interactive mode",
-      "type": "content_check"
-    },
-    {
-      "id": "mentions-json-output",
-      "text": "Includes --output-format json flag for structured output",
-      "type": "content_check"
-    },
-    {
-      "id": "mentions-api-key-env",
-      "text": "Shows how to set GEMINI_API_KEY from GitHub secrets",
-      "type": "content_check"
-    },
-    {
-      "id": "provides-workflow-yaml",
-      "text": "Provides a GitHub Actions workflow YAML snippet",
-      "type": "content_check"
-    },
-    {
-      "id": "mentions-jq-parsing",
-      "text": "Shows how to parse JSON response (e.g., with jq .response)",
-      "type": "content_check"
-    },
-    {
-      "id": "no-hallucinated-flags",
-      "text": "Does not invent non-existent CLI flags or options",
-      "type": "accuracy_check"
-    }
-  ]
-}

package/gemini-cli-workspace/iteration-2/eval-1-cicd-setup/with_skill/grading.json

@@ -1,37 +0,0 @@
-{
-  "eval_id": 1,
-  "eval_name": "cicd-setup",
-  "config": "with_skill",
-  "expectations": [
-    {
-      "text": "Uses -p or --prompt flag for headless/non-interactive mode",
-      "passed": true,
-      "evidence": "Line 53: gemini -p \"You are a senior code reviewer...\" with --output-format json"
-    },
-    {
-      "text": "Includes --output-format json flag for structured output",
-      "passed": true,
-      "evidence": "Line 62: --output-format json flag used correctly"
-    },
-    {
-      "text": "Shows how to set GEMINI_API_KEY from GitHub secrets",
-      "passed": true,
-      "evidence": "Line 42: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} in env block"
-    },
-    {
-      "text": "Provides a GitHub Actions workflow YAML snippet",
-      "passed": true,
-      "evidence": "Complete workflow YAML at .github/workflows/gemini-pr-review.yml with checkout, node setup, install, review, and comment steps"
-    },
-    {
-      "text": "Shows how to parse JSON response (e.g., with jq .response)",
-      "passed": true,
-      "evidence": "Line 65: REVIEW=$(echo \"$RESULT\" | jq -r '.response') and line 68: token stats extraction with jq"
-    },
-    {
-      "text": "Does not invent non-existent CLI flags or options",
-      "passed": true,
-      "evidence": "Only uses verified flags: -p and --output-format json. Exit codes match corrected values (0, 1, 42, 53)."
-    }
-  ]
-}

package/gemini-cli-workspace/iteration-2/eval-1-cicd-setup/with_skill/outputs/response.md

@@ -1,195 +0,0 @@
-# Setting Up Gemini CLI for Automated PR Reviews in GitHub Actions
-
-## Prerequisites
-
-1. A Gemini API key (get one at https://aistudio.google.com/apikey)
-2. The API key stored as a GitHub Actions secret named `GEMINI_API_KEY`
-
-## GitHub Actions Workflow
-
-Create a file at `.github/workflows/gemini-pr-review.yml`:
-
-```yaml
-name: Gemini PR Review
-
-on:
-  pull_request:
-    types: [opened, synchronize]
-
-permissions:
-  pull-requests: write
-  contents: read
-
-jobs:
-  review:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0  # Full history so we can diff against base
-
-      - name: Set up Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '20'
-
-      - name: Install Gemini CLI
-        run: npm install -g @google/gemini-cli
-
-      - name: Run AI Code Review
-        env:
-          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
-        run: |
-          # Get the diff between the PR branch and the base branch
-          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD)
-
-          if [ -z "$DIFF" ]; then
-            echo "No changes to review."
-            exit 0
-          fi
-
-          # Run Gemini CLI with structured JSON output
-          RESULT=$(echo "$DIFF" | gemini -p "You are a senior code reviewer. Review the following pull request diff for:
-          1. Bugs and logical errors
-          2. Security vulnerabilities
-          3. Performance issues
-          4. Code style and best practices
-
-          Provide your review in a structured format with sections for each category. For each issue found, include the file name, line context, severity (critical/warning/info), and a suggested fix.
-
-          If no issues are found in a category, say so explicitly." \
-            --output-format json)
-
-          # Extract just the response text
-          REVIEW=$(echo "$RESULT" | jq -r '.response')
-
-          # Extract token usage stats for logging
-          TOKENS=$(echo "$RESULT" | jq '.stats.models | to_entries[0].value.tokens.total // "unknown"')
-          echo "Tokens used: $TOKENS"
-
-          # Write the review to the GitHub Step Summary
-          echo "## Gemini Code Review" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "$REVIEW" >> $GITHUB_STEP_SUMMARY
-
-          # Save the full JSON result for downstream steps
-          echo "$RESULT" > review-result.json
-
-      - name: Post review as PR comment
-        if: success()
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          REVIEW=$(jq -r '.response' review-result.json)
-          TOKENS=$(jq '.stats.models | to_entries[0].value.tokens.total // "unknown"' review-result.json)
-
-          COMMENT_BODY=$(cat <<EOF
-          ## Gemini Automated Code Review
-
-          $REVIEW
-
-          ---
-          *Reviewed by Gemini CLI | Tokens used: $TOKENS*
-          EOF
-          )
-
-          gh pr comment ${{ github.event.pull_request.number }} --body "$COMMENT_BODY"
-```
-
-## How It Works
-
-### Authentication
-
-The workflow uses the `GEMINI_API_KEY` environment variable, which is pulled from GitHub Secrets. Gemini CLI automatically detects this variable -- no additional configuration is needed. The free tier allows 1,000 requests per day, which is sufficient for most PR review workflows.
-
-### Structured JSON Output
-
-The `--output-format json` flag causes Gemini CLI to return a single JSON object once all processing is complete. The schema looks like this:
-
-```json
-{
-  "response": "The review text content...",
-  "stats": {
-    "models": {
-      "gemini-2.5-pro": {
-        "tokens": {
-          "prompt": 24939,
-          "candidates": 20,
-          "total": 25113
-        },
-        "api": {
-          "totalRequests": 2,
-          "totalErrors": 0,
-          "totalLatencyMs": 5053
-        }
-      }
-    },
-    "tools": {
-      "totalCalls": 1,
-      "totalSuccess": 1,
-      "totalFail": 0
-    },
-    "files": {
-      "totalLinesAdded": 0,
-      "totalLinesRemoved": 0
-    }
-  }
-}
-```
-
-You extract the review text with `jq -r '.response'` and can inspect token usage or tool call stats from the `stats` object.
-
-### Exit Codes
-
-Gemini CLI uses these exit codes, which you can use for conditional logic:
-
-| Code | Meaning |
-|------|---------|
-| 0    | Success |
-| 1    | General error |
-| 42   | Input error (invalid arguments, missing config) |
-| 53   | Turn limit exceeded |
-
-## Parsing the JSON Output in a Downstream Step
-
-If you need to parse the review in a separate step (for example, to fail the build on critical findings), you can do it like this:
-
-```yaml
-      - name: Check for critical issues
-        run: |
-          REVIEW=$(jq -r '.response' review-result.json)
-
-          # Simple check: fail if the review mentions critical severity
-          if echo "$REVIEW" | grep -qi "critical"; then
-            echo "::error::Critical issues found in code review"
-            exit 1
-          fi
-```
-
-## Advanced: Using a GEMINI.md for Consistent Review Context
-
-Create a `.gemini/GEMINI.md` file in your repository root to give Gemini persistent context about your project's standards:
-
-```markdown
-# Project Review Guidelines
-
-## Code Standards
-- All functions must have JSDoc comments
-- Use 2-space indentation
-- No console.log in production code
-
-## Security Requirements
-- Never hardcode secrets or API keys
-- All user input must be validated
-- SQL queries must use parameterized statements
-```
-
-Gemini CLI automatically loads this file when running in your repo directory, so the reviewer will apply your project-specific rules during every PR review.
-
-## Known Limitations
-
-- Custom slash commands (`.toml` files) do not work in headless/non-interactive mode. You must embed the full prompt text directly in your script, as shown above.
-- Extensions are also unavailable in headless mode.
-- The free tier is rate-limited to 60 requests/minute and 1,000 requests/day. For high-volume repos, consider using a paid API key.
-- Very large diffs may exceed the model's context window. Consider filtering the diff to only changed files that matter (e.g., exclude lockfiles and generated code).

package/gemini-cli-workspace/iteration-2/eval-1-cicd-setup/with_skill/timing.json

@@ -1,5 +0,0 @@
-{
-  "total_tokens": 64216,
-  "duration_ms": 53586,
-  "total_duration_seconds": 53.6
-}

package/gemini-cli-workspace/iteration-2/eval-1-cicd-setup/without_skill/grading.json

@@ -1,37 +0,0 @@
-{
-  "eval_id": 1,
-  "eval_name": "cicd-setup",
-  "config": "without_skill",
-  "expectations": [
-    {
-      "text": "Uses -p or --prompt flag for headless/non-interactive mode",
-      "passed": true,
-      "evidence": "Line 96: gemini -p \"$(cat review_prompt.txt)\" and line 351: gemini -p \"Review this code diff...\""
-    },
-    {
-      "text": "Includes --output-format json flag for structured output",
-      "passed": false,
-      "evidence": "Does not use --output-format json. Instead relies on prompt-based JSON (asking model to return JSON in prompt text). Line 95 comment mentions '--json flag if available' which is not a real Gemini CLI flag."
-    },
-    {
-      "text": "Shows how to set GEMINI_API_KEY from GitHub secrets",
-      "passed": true,
-      "evidence": "Line 62: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} in env block"
-    },
-    {
-      "text": "Provides a GitHub Actions workflow YAML snippet",
-      "passed": true,
-      "evidence": "Complete workflow YAML provided with checkout, node setup, install, review, and post-comment steps"
-    },
-    {
-      "text": "Shows how to parse JSON response (e.g., with jq .response)",
-      "passed": true,
-      "evidence": "Uses jq for JSON validation (line 102) and parsing (line 248: jq -r '.candidates[0].content.parts[0].text'). Different schema than CLI --output-format json but still demonstrates jq parsing."
-    },
-    {
-      "text": "Does not invent non-existent CLI flags or options",
-      "passed": false,
-      "evidence": "Line 95: comment references '--json flag if available' which is not a real Gemini CLI flag. Should use --output-format json instead."
-    }
-  ]
-}