npm - @biggora/claude-plugins - Versions diffs - 1.2.0 → 1.2.2 - Mend

@biggora/claude-plugins 1.2.0 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (185) hide show

package/gemini-cli-workspace/iteration-2/eval-viewer.html DELETED Viewed

@@ -1,1325 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Eval Review</title>
-  <link rel="preconnect" href="https://fonts.googleapis.com">
-  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-  <link href="https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" rel="stylesheet">
-  <script src="https://cdn.sheetjs.com/xlsx-0.20.3/package/dist/xlsx.full.min.js" integrity="sha384-EnyY0/GSHQGSxSgMwaIPzSESbqoOLSexfnSMN2AP+39Ckmn92stwABZynq1JyzdT" crossorigin="anonymous"></script>
-  <style>
-    :root {
-      --bg: #faf9f5;
-      --surface: #ffffff;
-      --border: #e8e6dc;
-      --text: #141413;
-      --text-muted: #b0aea5;
-      --accent: #d97757;
-      --accent-hover: #c4613f;
-      --green: #788c5d;
-      --green-bg: #eef2e8;
-      --red: #c44;
-      --red-bg: #fceaea;
-      --header-bg: #141413;
-      --header-text: #faf9f5;
-      --radius: 6px;
-    }
-    * { box-sizing: border-box; margin: 0; padding: 0; }
-    body {
-      font-family: 'Lora', Georgia, serif;
-      background: var(--bg);
-      color: var(--text);
-      height: 100vh;
-      display: flex;
-      flex-direction: column;
-    }
-    /* ---- Header ---- */
-    .header {
-      background: var(--header-bg);
-      color: var(--header-text);
-      padding: 1rem 2rem;
-      display: flex;
-      justify-content: space-between;
-      align-items: center;
-      flex-shrink: 0;
-    }
-    .header h1 {
-      font-family: 'Poppins', sans-serif;
-      font-size: 1.25rem;
-      font-weight: 600;
-    }
-    .header .instructions {
-      font-size: 0.8rem;
-      opacity: 0.7;
-      margin-top: 0.25rem;
-    }
-    .header .progress {
-      font-size: 0.875rem;
-      opacity: 0.8;
-      text-align: right;
-    }
-    /* ---- Main content ---- */
-    .main {
-      flex: 1;
-      overflow-y: auto;
-      padding: 1.5rem 2rem;
-      display: flex;
-      flex-direction: column;
-      gap: 1.25rem;
-    }
-    /* ---- Sections ---- */
-    .section {
-      background: var(--surface);
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      flex-shrink: 0;
-    }
-    .section-header {
-      font-family: 'Poppins', sans-serif;
-      padding: 0.75rem 1rem;
-      font-size: 0.75rem;
-      font-weight: 500;
-      text-transform: uppercase;
-      letter-spacing: 0.05em;
-      color: var(--text-muted);
-      border-bottom: 1px solid var(--border);
-      background: var(--bg);
-    }
-    .section-body {
-      padding: 1rem;
-    }
-    /* ---- Config badge ---- */
-    .config-badge {
-      display: inline-block;
-      padding: 0.2rem 0.625rem;
-      border-radius: 9999px;
-      font-family: 'Poppins', sans-serif;
-      font-size: 0.6875rem;
-      font-weight: 600;
-      text-transform: uppercase;
-      letter-spacing: 0.03em;
-      margin-left: 0.75rem;
-      vertical-align: middle;
-    }
-    .config-badge.config-primary {
-      background: rgba(33, 150, 243, 0.12);
-      color: #1976d2;
-    }
-    .config-badge.config-baseline {
-      background: rgba(255, 193, 7, 0.15);
-      color: #f57f17;
-    }
-    /* ---- Prompt ---- */
-    .prompt-text {
-      white-space: pre-wrap;
-      font-size: 0.9375rem;
-      line-height: 1.6;
-    }
-    /* ---- Outputs ---- */
-    .output-file {
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      overflow: hidden;
-    }
-    .output-file + .output-file {
-      margin-top: 1rem;
-    }
-    .output-file-header {
-      padding: 0.5rem 0.75rem;
-      font-size: 0.8rem;
-      font-weight: 600;
-      color: var(--text-muted);
-      background: var(--bg);
-      border-bottom: 1px solid var(--border);
-      font-family: 'SF Mono', SFMono-Regular, Consolas, 'Liberation Mono', Menlo, monospace;
-      display: flex;
-      justify-content: space-between;
-      align-items: center;
-    }
-    .output-file-header .dl-btn {
-      font-size: 0.7rem;
-      color: var(--accent);
-      text-decoration: none;
-      cursor: pointer;
-      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
-      font-weight: 500;
-      opacity: 0.8;
-    }
-    .output-file-header .dl-btn:hover {
-      opacity: 1;
-      text-decoration: underline;
-    }
-    .output-file-content {
-      padding: 0.75rem;
-      overflow-x: auto;
-    }
-    .output-file-content pre {
-      font-size: 0.8125rem;
-      line-height: 1.5;
-      white-space: pre-wrap;
-      word-break: break-word;
-      font-family: 'SF Mono', SFMono-Regular, Consolas, 'Liberation Mono', Menlo, monospace;
-    }
-    .output-file-content img {
-      max-width: 100%;
-      height: auto;
-      border-radius: 4px;
-    }
-    .output-file-content iframe {
-      width: 100%;
-      height: 600px;
-      border: none;
-    }
-    .output-file-content table {
-      border-collapse: collapse;
-      font-size: 0.8125rem;
-      width: 100%;
-    }
-    .output-file-content table td,
-    .output-file-content table th {
-      border: 1px solid var(--border);
-      padding: 0.375rem 0.5rem;
-      text-align: left;
-    }
-    .output-file-content table th {
-      background: var(--bg);
-      font-weight: 600;
-    }
-    .output-file-content .download-link {
-      display: inline-flex;
-      align-items: center;
-      gap: 0.5rem;
-      padding: 0.5rem 1rem;
-      background: var(--bg);
-      border: 1px solid var(--border);
-      border-radius: 4px;
-      color: var(--accent);
-      text-decoration: none;
-      font-size: 0.875rem;
-      cursor: pointer;
-    }
-    .output-file-content .download-link:hover {
-      background: var(--border);
-    }
-    .empty-state {
-      color: var(--text-muted);
-      font-style: italic;
-      padding: 2rem;
-      text-align: center;
-    }
-    /* ---- Feedback ---- */
-    .prev-feedback {
-      background: var(--bg);
-      border: 1px solid var(--border);
-      border-radius: 4px;
-      padding: 0.625rem 0.75rem;
-      margin-top: 0.75rem;
-      font-size: 0.8125rem;
-      color: var(--text-muted);
-      line-height: 1.5;
-    }
-    .prev-feedback-label {
-      font-size: 0.7rem;
-      font-weight: 600;
-      text-transform: uppercase;
-      letter-spacing: 0.04em;
-      margin-bottom: 0.25rem;
-      color: var(--text-muted);
-    }
-    .feedback-textarea {
-      width: 100%;
-      min-height: 100px;
-      padding: 0.75rem;
-      border: 1px solid var(--border);
-      border-radius: 4px;
-      font-family: inherit;
-      font-size: 0.9375rem;
-      line-height: 1.5;
-      resize: vertical;
-      color: var(--text);
-    }
-    .feedback-textarea:focus {
-      outline: none;
-      border-color: var(--accent);
-      box-shadow: 0 0 0 3px rgba(37, 99, 235, 0.1);
-    }
-    .feedback-status {
-      font-size: 0.75rem;
-      color: var(--text-muted);
-      margin-top: 0.5rem;
-      min-height: 1.1em;
-    }
-    /* ---- Grades (collapsible) ---- */
-    .grades-toggle {
-      display: flex;
-      align-items: center;
-      cursor: pointer;
-      user-select: none;
-    }
-    .grades-toggle:hover {
-      color: var(--accent);
-    }
-    .grades-toggle .arrow {
-      margin-right: 0.5rem;
-      transition: transform 0.15s;
-      font-size: 0.75rem;
-    }
-    .grades-toggle .arrow.open {
-      transform: rotate(90deg);
-    }
-    .grades-content {
-      display: none;
-      margin-top: 0.75rem;
-    }
-    .grades-content.open {
-      display: block;
-    }
-    .grades-summary {
-      font-size: 0.875rem;
-      margin-bottom: 0.75rem;
-      display: flex;
-      align-items: center;
-      gap: 0.5rem;
-    }
-    .grade-badge {
-      display: inline-block;
-      padding: 0.125rem 0.5rem;
-      border-radius: 9999px;
-      font-size: 0.75rem;
-      font-weight: 600;
-    }
-    .grade-pass { background: var(--green-bg); color: var(--green); }
-    .grade-fail { background: var(--red-bg); color: var(--red); }
-    .assertion-list {
-      list-style: none;
-    }
-    .assertion-item {
-      padding: 0.625rem 0;
-      border-bottom: 1px solid var(--border);
-      font-size: 0.8125rem;
-    }
-    .assertion-item:last-child { border-bottom: none; }
-    .assertion-status {
-      font-weight: 600;
-      margin-right: 0.5rem;
-    }
-    .assertion-status.pass { color: var(--green); }
-    .assertion-status.fail { color: var(--red); }
-    .assertion-evidence {
-      color: var(--text-muted);
-      font-size: 0.75rem;
-      margin-top: 0.25rem;
-      padding-left: 1.5rem;
-    }
-    /* ---- View tabs ---- */
-    .view-tabs {
-      display: flex;
-      gap: 0;
-      padding: 0 2rem;
-      background: var(--bg);
-      border-bottom: 1px solid var(--border);
-      flex-shrink: 0;
-    }
-    .view-tab {
-      font-family: 'Poppins', sans-serif;
-      padding: 0.625rem 1.25rem;
-      font-size: 0.8125rem;
-      font-weight: 500;
-      cursor: pointer;
-      border: none;
-      background: none;
-      color: var(--text-muted);
-      border-bottom: 2px solid transparent;
-      transition: all 0.15s;
-    }
-    .view-tab:hover { color: var(--text); }
-    .view-tab.active {
-      color: var(--accent);
-      border-bottom-color: var(--accent);
-    }
-    .view-panel { display: none; }
-    .view-panel.active { display: flex; flex-direction: column; flex: 1; overflow: hidden; }
-    /* ---- Benchmark view ---- */
-    .benchmark-view {
-      padding: 1.5rem 2rem;
-      overflow-y: auto;
-      flex: 1;
-    }
-    .benchmark-table {
-      border-collapse: collapse;
-      background: var(--surface);
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      font-size: 0.8125rem;
-      width: 100%;
-      margin-bottom: 1.5rem;
-    }
-    .benchmark-table th, .benchmark-table td {
-      padding: 0.625rem 0.75rem;
-      text-align: left;
-      border: 1px solid var(--border);
-    }
-    .benchmark-table th {
-      font-family: 'Poppins', sans-serif;
-      background: var(--header-bg);
-      color: var(--header-text);
-      font-weight: 500;
-      font-size: 0.75rem;
-      text-transform: uppercase;
-      letter-spacing: 0.04em;
-    }
-    .benchmark-table tr:hover { background: var(--bg); }
-    .benchmark-table tr.benchmark-row-with { background: rgba(33, 150, 243, 0.06); }
-    .benchmark-table tr.benchmark-row-without { background: rgba(255, 193, 7, 0.06); }
-    .benchmark-table tr.benchmark-row-with:hover { background: rgba(33, 150, 243, 0.12); }
-    .benchmark-table tr.benchmark-row-without:hover { background: rgba(255, 193, 7, 0.12); }
-    .benchmark-table tr.benchmark-row-avg { font-weight: 600; border-top: 2px solid var(--border); }
-    .benchmark-table tr.benchmark-row-avg.benchmark-row-with { background: rgba(33, 150, 243, 0.12); }
-    .benchmark-table tr.benchmark-row-avg.benchmark-row-without { background: rgba(255, 193, 7, 0.12); }
-    .benchmark-delta-positive { color: var(--green); font-weight: 600; }
-    .benchmark-delta-negative { color: var(--red); font-weight: 600; }
-    .benchmark-notes {
-      background: var(--surface);
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      padding: 1rem;
-    }
-    .benchmark-notes h3 {
-      font-family: 'Poppins', sans-serif;
-      font-size: 0.875rem;
-      margin-bottom: 0.75rem;
-    }
-    .benchmark-notes ul {
-      list-style: disc;
-      padding-left: 1.25rem;
-    }
-    .benchmark-notes li {
-      font-size: 0.8125rem;
-      line-height: 1.6;
-      margin-bottom: 0.375rem;
-    }
-    .benchmark-empty {
-      color: var(--text-muted);
-      font-style: italic;
-      text-align: center;
-      padding: 3rem;
-    }
-    /* ---- Navigation ---- */
-    .nav {
-      display: flex;
-      justify-content: space-between;
-      align-items: center;
-      padding: 1rem 2rem;
-      border-top: 1px solid var(--border);
-      background: var(--surface);
-      flex-shrink: 0;
-    }
-    .nav-btn {
-      font-family: 'Poppins', sans-serif;
-      padding: 0.5rem 1.25rem;
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      background: var(--surface);
-      cursor: pointer;
-      font-size: 0.875rem;
-      font-weight: 500;
-      color: var(--text);
-      transition: all 0.15s;
-    }
-    .nav-btn:hover:not(:disabled) {
-      background: var(--bg);
-      border-color: var(--text-muted);
-    }
-    .nav-btn:disabled {
-      opacity: 0.4;
-      cursor: not-allowed;
-    }
-    .done-btn {
-      font-family: 'Poppins', sans-serif;
-      padding: 0.5rem 1.5rem;
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      background: var(--surface);
-      color: var(--text);
-      cursor: pointer;
-      font-size: 0.875rem;
-      font-weight: 500;
-      transition: all 0.15s;
-    }
-    .done-btn:hover {
-      background: var(--bg);
-      border-color: var(--text-muted);
-    }
-    .done-btn.ready {
-      border: none;
-      background: var(--accent);
-      color: white;
-      font-weight: 600;
-    }
-    .done-btn.ready:hover {
-      background: var(--accent-hover);
-    }
-    /* ---- Done overlay ---- */
-    .done-overlay {
-      display: none;
-      position: fixed;
-      inset: 0;
-      background: rgba(0, 0, 0, 0.5);
-      z-index: 100;
-      justify-content: center;
-      align-items: center;
-    }
-    .done-overlay.visible {
-      display: flex;
-    }
-    .done-card {
-      background: var(--surface);
-      border-radius: 12px;
-      padding: 2rem 3rem;
-      text-align: center;
-      box-shadow: 0 20px 60px rgba(0, 0, 0, 0.3);
-      max-width: 500px;
-    }
-    .done-card h2 {
-      font-size: 1.5rem;
-      margin-bottom: 0.5rem;
-    }
-    .done-card p {
-      color: var(--text-muted);
-      margin-bottom: 1.5rem;
-      line-height: 1.5;
-    }
-    .done-card .btn-row {
-      display: flex;
-      gap: 0.5rem;
-      justify-content: center;
-    }
-    .done-card button {
-      padding: 0.5rem 1.25rem;
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      background: var(--surface);
-      cursor: pointer;
-      font-size: 0.875rem;
-    }
-    .done-card button:hover {
-      background: var(--bg);
-    }
-    /* ---- Toast ---- */
-    .toast {
-      position: fixed;
-      bottom: 5rem;
-      left: 50%;
-      transform: translateX(-50%);
-      background: var(--header-bg);
-      color: var(--header-text);
-      padding: 0.625rem 1.25rem;
-      border-radius: var(--radius);
-      font-size: 0.875rem;
-      opacity: 0;
-      transition: opacity 0.3s;
-      pointer-events: none;
-      z-index: 200;
-    }
-    .toast.visible {
-      opacity: 1;
-    }
-  </style>
-</head>
-<body>
-  <div id="app" style="height:100vh; display:flex; flex-direction:column;">
-    <div class="header">
-      <div>
-        <h1>Eval Review: <span id="skill-name"></span></h1>
-        <div class="instructions">Review each output and leave feedback below. Navigate with arrow keys or buttons. When done, copy feedback and paste into Claude Code.</div>
-      </div>
-      <div class="progress" id="progress"></div>
-    </div>
-    <!-- View tabs (only shown when benchmark data exists) -->
-    <div class="view-tabs" id="view-tabs" style="display:none;">
-      <button class="view-tab active" onclick="switchView('outputs')">Outputs</button>
-      <button class="view-tab" onclick="switchView('benchmark')">Benchmark</button>
-    </div>
-    <!-- Outputs panel (qualitative review) -->
-    <div class="view-panel active" id="panel-outputs">
-    <div class="main">
-      <!-- Prompt -->
-      <div class="section">
-        <div class="section-header">Prompt <span class="config-badge" id="config-badge" style="display:none;"></span></div>
-        <div class="section-body">
-          <div class="prompt-text" id="prompt-text"></div>
-        </div>
-      </div>
-      <!-- Outputs -->
-      <div class="section">
-        <div class="section-header">Output</div>
-        <div class="section-body" id="outputs-body">
-          <div class="empty-state">No output files found</div>
-        </div>
-      </div>
-      <!-- Previous Output (collapsible) -->
-      <div class="section" id="prev-outputs-section" style="display:none;">
-        <div class="section-header">
-          <div class="grades-toggle" onclick="togglePrevOutputs()">
-            <span class="arrow" id="prev-outputs-arrow">&#9654;</span>
-            Previous Output
-          </div>
-        </div>
-        <div class="grades-content" id="prev-outputs-content"></div>
-      </div>
-      <!-- Grades (collapsible) -->
-      <div class="section" id="grades-section" style="display:none;">
-        <div class="section-header">
-          <div class="grades-toggle" onclick="toggleGrades()">
-            <span class="arrow" id="grades-arrow">&#9654;</span>
-            Formal Grades
-          </div>
-        </div>
-        <div class="grades-content" id="grades-content"></div>
-      </div>
-      <!-- Feedback -->
-      <div class="section">
-        <div class="section-header">Your Feedback</div>
-        <div class="section-body">
-          <textarea
-            class="feedback-textarea"
-            id="feedback"
-            placeholder="What do you think of this output? Any issues, suggestions, or things that look great?"
-          ></textarea>
-          <div class="feedback-status" id="feedback-status"></div>
-          <div class="prev-feedback" id="prev-feedback" style="display:none;">
-            <div class="prev-feedback-label">Previous feedback</div>
-            <div id="prev-feedback-text"></div>
-          </div>
-        </div>
-      </div>
-    </div>
-    <div class="nav" id="outputs-nav">
-      <button class="nav-btn" id="prev-btn" onclick="navigate(-1)">&#8592; Previous</button>
-      <button class="done-btn" id="done-btn" onclick="showDoneDialog()">Submit All Reviews</button>
-      <button class="nav-btn" id="next-btn" onclick="navigate(1)">Next &#8594;</button>
-    </div>
-    </div><!-- end panel-outputs -->
-    <!-- Benchmark panel (quantitative stats) -->
-    <div class="view-panel" id="panel-benchmark">
-      <div class="benchmark-view" id="benchmark-content">
-        <div class="benchmark-empty">No benchmark data available. Run a benchmark to see quantitative results here.</div>
-      </div>
-    </div>
-  </div>
-  <!-- Done overlay -->
-  <div class="done-overlay" id="done-overlay">
-    <div class="done-card">
-      <h2>Review Complete</h2>
-      <p>Your feedback has been saved. Go back to your Claude Code session and tell Claude you're done reviewing.</p>
-      <div class="btn-row">
-        <button onclick="closeDoneDialog()">OK</button>
-      </div>
-    </div>
-  </div>
-  <!-- Toast -->
-  <div class="toast" id="toast"></div>
-  <script>
-    // ---- Embedded data (injected by generate_review.py) ----
-    const EMBEDDED_DATA = {"skill_name": "gemini-cli", "runs": [{"id": "eval-1-cicd-setup-with_skill", "prompt": "I want to set up gemini cli in my CI/CD pipeline on GitHub Actions so it can automatically review PRs. We use an API key stored in GitHub secrets. How do I set this up with structured JSON output so I can parse the review?", "eval_id": 1, "outputs": [{"name": "response.md", "type": "text", "content": "# Setting Up Gemini CLI for Automated PR Reviews in GitHub Actions\n\n## Prerequisites\n\n1. A Gemini API key (get one at https://aistudio.google.com/apikey)\n2. The API key stored as a GitHub Actions secret named `GEMINI_API_KEY`\n\n## GitHub Actions Workflow\n\nCreate a file at `.github/workflows/gemini-pr-review.yml`:\n\n```yaml\nname: Gemini PR Review\n\non:\n  pull_request:\n    types: [opened, synchronize]\n\npermissions:\n  pull-requests: write\n  contents: read\n\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout code\n        uses: actions/checkout@v4\n        with:\n          fetch-depth: 0  # Full history so we can diff against base\n\n      - name: Set up Node.js\n        uses: actions/setup-node@v4\n        with:\n          node-version: '20'\n\n      - name: Install Gemini CLI\n        run: npm install -g @google/gemini-cli\n\n      - name: Run AI Code Review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          # Get the diff between the PR branch and the base branch\n          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD)\n\n          if [ -z \"$DIFF\" ]; then\n            echo \"No changes to review.\"\n            exit 0\n          fi\n\n          # Run Gemini CLI with structured JSON output\n          RESULT=$(echo \"$DIFF\" | gemini -p \"You are a senior code reviewer. Review the following pull request diff for:\n          1. Bugs and logical errors\n          2. Security vulnerabilities\n          3. Performance issues\n          4. Code style and best practices\n\n          Provide your review in a structured format with sections for each category. For each issue found, include the file name, line context, severity (critical/warning/info), and a suggested fix.\n\n          If no issues are found in a category, say so explicitly.\" \\\n            --output-format json)\n\n          # Extract just the response text\n          REVIEW=$(echo \"$RESULT\" | jq -r '.response')\n\n          # Extract token usage stats for logging\n          TOKENS=$(echo \"$RESULT\" | jq '.stats.models | to_entries[0].value.tokens.total // \"unknown\"')\n          echo \"Tokens used: $TOKENS\"\n\n          # Write the review to the GitHub Step Summary\n          echo \"## Gemini Code Review\" >> $GITHUB_STEP_SUMMARY\n          echo \"\" >> $GITHUB_STEP_SUMMARY\n          echo \"$REVIEW\" >> $GITHUB_STEP_SUMMARY\n\n          # Save the full JSON result for downstream steps\n          echo \"$RESULT\" > review-result.json\n\n      - name: Post review as PR comment\n        if: success()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: |\n          REVIEW=$(jq -r '.response' review-result.json)\n          TOKENS=$(jq '.stats.models | to_entries[0].value.tokens.total // \"unknown\"' review-result.json)\n\n          COMMENT_BODY=$(cat <<EOF\n          ## Gemini Automated Code Review\n\n          $REVIEW\n\n          ---\n          *Reviewed by Gemini CLI | Tokens used: $TOKENS*\n          EOF\n          )\n\n          gh pr comment ${{ github.event.pull_request.number }} --body \"$COMMENT_BODY\"\n```\n\n## How It Works\n\n### Authentication\n\nThe workflow uses the `GEMINI_API_KEY` environment variable, which is pulled from GitHub Secrets. Gemini CLI automatically detects this variable -- no additional configuration is needed. The free tier allows 1,000 requests per day, which is sufficient for most PR review workflows.\n\n### Structured JSON Output\n\nThe `--output-format json` flag causes Gemini CLI to return a single JSON object once all processing is complete. The schema looks like this:\n\n```json\n{\n  \"response\": \"The review text content...\",\n  \"stats\": {\n    \"models\": {\n      \"gemini-2.5-pro\": {\n        \"tokens\": {\n          \"prompt\": 24939,\n          \"candidates\": 20,\n          \"total\": 25113\n        },\n        \"api\": {\n          \"totalRequests\": 2,\n          \"totalErrors\": 0,\n          \"totalLatencyMs\": 5053\n        }\n      }\n    },\n    \"tools\": {\n      \"totalCalls\": 1,\n      \"totalSuccess\": 1,\n      \"totalFail\": 0\n    },\n    \"files\": {\n      \"totalLinesAdded\": 0,\n      \"totalLinesRemoved\": 0\n    }\n  }\n}\n```\n\nYou extract the review text with `jq -r '.response'` and can inspect token usage or tool call stats from the `stats` object.\n\n### Exit Codes\n\nGemini CLI uses these exit codes, which you can use for conditional logic:\n\n| Code | Meaning |\n|------|---------|\n| 0    | Success |\n| 1    | General error |\n| 42   | Input error (invalid arguments, missing config) |\n| 53   | Turn limit exceeded |\n\n## Parsing the JSON Output in a Downstream Step\n\nIf you need to parse the review in a separate step (for example, to fail the build on critical findings), you can do it like this:\n\n```yaml\n      - name: Check for critical issues\n        run: |\n          REVIEW=$(jq -r '.response' review-result.json)\n\n          # Simple check: fail if the review mentions critical severity\n          if echo \"$REVIEW\" | grep -qi \"critical\"; then\n            echo \"::error::Critical issues found in code review\"\n            exit 1\n          fi\n```\n\n## Advanced: Using a GEMINI.md for Consistent Review Context\n\nCreate a `.gemini/GEMINI.md` file in your repository root to give Gemini persistent context about your project's standards:\n\n```markdown\n# Project Review Guidelines\n\n## Code Standards\n- All functions must have JSDoc comments\n- Use 2-space indentation\n- No console.log in production code\n\n## Security Requirements\n- Never hardcode secrets or API keys\n- All user input must be validated\n- SQL queries must use parameterized statements\n```\n\nGemini CLI automatically loads this file when running in your repo directory, so the reviewer will apply your project-specific rules during every PR review.\n\n## Known Limitations\n\n- Custom slash commands (`.toml` files) do not work in headless/non-interactive mode. You must embed the full prompt text directly in your script, as shown above.\n- Extensions are also unavailable in headless mode.\n- The free tier is rate-limited to 60 requests/minute and 1,000 requests/day. For high-volume repos, consider using a paid API key.\n- Very large diffs may exceed the model's context window. Consider filtering the diff to only changed files that matter (e.g., exclude lockfiles and generated code).\n"}], "grading": {"eval_id": 1, "eval_name": "cicd-setup", "config": "with_skill", "expectations": [{"text": "Uses -p or --prompt flag for headless/non-interactive mode", "passed": true, "evidence": "Line 53: gemini -p \"You are a senior code reviewer...\" with --output-format json"}, {"text": "Includes --output-format json flag for structured output", "passed": true, "evidence": "Line 62: --output-format json flag used correctly"}, {"text": "Shows how to set GEMINI_API_KEY from GitHub secrets", "passed": true, "evidence": "Line 42: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} in env block"}, {"text": "Provides a GitHub Actions workflow YAML snippet", "passed": true, "evidence": "Complete workflow YAML at .github/workflows/gemini-pr-review.yml with checkout, node setup, install, review, and comment steps"}, {"text": "Shows how to parse JSON response (e.g., with jq .response)", "passed": true, "evidence": "Line 65: REVIEW=$(echo \"$RESULT\" | jq -r '.response') and line 68: token stats extraction with jq"}, {"text": "Does not invent non-existent CLI flags or options", "passed": true, "evidence": "Only uses verified flags: -p and --output-format json. Exit codes match corrected values (0, 1, 42, 53)."}]}}, {"id": "eval-1-cicd-setup-without_skill", "prompt": "I want to set up gemini cli in my CI/CD pipeline on GitHub Actions so it can automatically review PRs. We use an API key stored in GitHub secrets. How do I set this up with structured JSON output so I can parse the review?", "eval_id": 1, "outputs": [{"name": "response.md", "type": "text", "content": "# Setting Up Gemini CLI for Automated PR Reviews in GitHub Actions\n\n## Overview\n\nThis guide covers how to configure Gemini CLI in a GitHub Actions CI/CD pipeline to automatically review pull requests, using an API key from GitHub Secrets and structured JSON output for parsing.\n\n## Prerequisites\n\n1. A Google AI API key (from Google AI Studio)\n2. The API key stored as a GitHub Actions secret (e.g., `GEMINI_API_KEY`)\n3. A GitHub repository with pull requests to review\n\n## Step 1: Store Your API Key in GitHub Secrets\n\n1. Go to your repository on GitHub\n2. Navigate to **Settings > Secrets and variables > Actions**\n3. Click **New repository secret**\n4. Name it `GEMINI_API_KEY` and paste your API key\n5. Click **Add secret**\n\n## Step 2: Create the GitHub Actions Workflow\n\nCreate a file at `.github/workflows/gemini-pr-review.yml`:\n\n```yaml\nname: Gemini PR Review\n\non:\n  pull_request:\n    types: [opened, synchronize, reopened]\n\npermissions:\n  contents: read\n  pull-requests: write\n\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout code\n        uses: actions/checkout@v4\n        with:\n          fetch-depth: 0\n\n      - name: Setup Node.js\n        uses: actions/setup-node@v4\n        with:\n          node-version: '20'\n\n      - name: Install Gemini CLI\n        run: npm install -g @google/gemini-cli\n\n      - name: Get PR diff\n        id: diff\n        run: |\n          git diff origin/${{ github.base_ref }}...HEAD > pr_diff.txt\n          echo \"diff_size=$(wc -c < pr_diff.txt)\" >> $GITHUB_OUTPUT\n\n      - name: Run Gemini Review\n        id: review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          DIFF_CONTENT=$(cat pr_diff.txt)\n\n          # Create the prompt requesting structured JSON output\n          cat > review_prompt.txt << 'PROMPT_EOF'\n          You are a code reviewer. Analyze the following PR diff and provide your review as a JSON object with this exact structure:\n\n          {\n            \"summary\": \"Brief overall summary of the changes\",\n            \"risk_level\": \"low|medium|high\",\n            \"issues\": [\n              {\n                \"severity\": \"critical|warning|suggestion\",\n                \"file\": \"path/to/file\",\n                \"line\": 0,\n                \"description\": \"Description of the issue\",\n                \"suggestion\": \"Suggested fix\"\n              }\n            ],\n            \"positives\": [\"List of good practices observed\"],\n            \"approved\": true\n          }\n\n          Respond ONLY with valid JSON, no markdown fences, no extra text.\n\n          Here is the diff:\n          PROMPT_EOF\n\n          # Append the diff content\n          cat pr_diff.txt >> review_prompt.txt\n\n          # Run Gemini CLI with the prompt\n          # Use --json flag if available, otherwise parse text output\n          REVIEW_OUTPUT=$(gemini -p \"$(cat review_prompt.txt)\" 2>/dev/null || true)\n\n          # Save raw output for debugging\n          echo \"$REVIEW_OUTPUT\" > review_output.json\n\n          # Validate JSON\n          if echo \"$REVIEW_OUTPUT\" | jq . > /dev/null 2>&1; then\n            echo \"valid_json=true\" >> $GITHUB_OUTPUT\n          else\n            echo \"valid_json=false\" >> $GITHUB_OUTPUT\n            # Try to extract JSON from the response (in case of markdown fences)\n            REVIEW_OUTPUT=$(echo \"$REVIEW_OUTPUT\" | sed -n '/^{/,/^}/p' | head -1)\n            echo \"$REVIEW_OUTPUT\" > review_output.json\n          fi\n\n      - name: Parse and Post Review\n        if: always()\n        uses: actions/github-script@v7\n        with:\n          script: |\n            const fs = require('fs');\n\n            let review;\n            try {\n              const rawOutput = fs.readFileSync('review_output.json', 'utf8');\n              // Strip markdown code fences if present\n              const cleaned = rawOutput\n                .replace(/```json\\n?/g, '')\n                .replace(/```\\n?/g, '')\n                .trim();\n              review = JSON.parse(cleaned);\n            } catch (e) {\n              console.log('Failed to parse review JSON:', e.message);\n              // Post a fallback comment\n              await github.rest.issues.createComment({\n                owner: context.repo.owner,\n                repo: context.repo.repo,\n                issue_number: context.issue.number,\n                body: '## Gemini Review\\n\\n_Review output could not be parsed as JSON. Raw output saved as artifact._'\n              });\n              return;\n            }\n\n            // Build the review comment\n            let body = `## Gemini PR Review\\n\\n`;\n            body += `**Summary:** ${review.summary}\\n\\n`;\n            body += `**Risk Level:** ${review.risk_level}\\n\\n`;\n\n            if (review.issues && review.issues.length > 0) {\n              body += `### Issues Found (${review.issues.length})\\n\\n`;\n              for (const issue of review.issues) {\n                const emoji = issue.severity === 'critical' ? '\u00f0\u0178\u201d\u00b4' :\n                              issue.severity === 'warning' ? '\u00f0\u0178\u0178\u00a1' : '\u00f0\u0178\u201d\u00b5';\n                body += `${emoji} **${issue.severity.toUpperCase()}** - \\`${issue.file}\\``;\n                if (issue.line) body += ` (line ${issue.line})`;\n                body += `\\n`;\n                body += `> ${issue.description}\\n`;\n                if (issue.suggestion) {\n                  body += `> **Suggestion:** ${issue.suggestion}\\n`;\n                }\n                body += `\\n`;\n              }\n            } else {\n              body += `### No Issues Found\\n\\n`;\n            }\n\n            if (review.positives && review.positives.length > 0) {\n              body += `### Positives\\n\\n`;\n              for (const positive of review.positives) {\n                body += `- ${positive}\\n`;\n              }\n              body += `\\n`;\n            }\n\n            body += `---\\n_Automated review by Gemini CLI_`;\n\n            await github.rest.issues.createComment({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              issue_number: context.issue.number,\n              body: body\n            });\n\n      - name: Upload Review Artifact\n        if: always()\n        uses: actions/upload-artifact@v4\n        with:\n          name: gemini-review\n          path: review_output.json\n          retention-days: 7\n```\n\n## Step 3: Alternative Approach Using the Gemini API Directly (More Reliable for JSON)\n\nIf you want more control over structured output, you can call the Gemini API directly with `curl` instead of relying on the CLI. This gives you access to the `response_mime_type` parameter for guaranteed JSON:\n\n```yaml\n      - name: Run Gemini Review via API\n        id: review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          DIFF_CONTENT=$(cat pr_diff.txt | jq -Rs .)\n\n          # Build the API request with structured output\n          cat > request.json << EOF\n          {\n            \"contents\": [{\n              \"parts\": [{\n                \"text\": \"You are a code reviewer. Analyze this PR diff and provide a review.\\n\\nDiff:\\n${DIFF_CONTENT}\"\n              }]\n            }],\n            \"generationConfig\": {\n              \"responseMimeType\": \"application/json\",\n              \"responseSchema\": {\n                \"type\": \"object\",\n                \"properties\": {\n                  \"summary\": { \"type\": \"string\" },\n                  \"risk_level\": { \"type\": \"string\", \"enum\": [\"low\", \"medium\", \"high\"] },\n                  \"issues\": {\n                    \"type\": \"array\",\n                    \"items\": {\n                      \"type\": \"object\",\n                      \"properties\": {\n                        \"severity\": { \"type\": \"string\", \"enum\": [\"critical\", \"warning\", \"suggestion\"] },\n                        \"file\": { \"type\": \"string\" },\n                        \"line\": { \"type\": \"integer\" },\n                        \"description\": { \"type\": \"string\" },\n                        \"suggestion\": { \"type\": \"string\" }\n                      },\n                      \"required\": [\"severity\", \"file\", \"description\"]\n                    }\n                  },\n                  \"positives\": {\n                    \"type\": \"array\",\n                    \"items\": { \"type\": \"string\" }\n                  },\n                  \"approved\": { \"type\": \"boolean\" }\n                },\n                \"required\": [\"summary\", \"risk_level\", \"issues\", \"approved\"]\n              }\n            }\n          }\n          EOF\n\n          # Call the Gemini API\n          curl -s \"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${GEMINI_API_KEY}\" \\\n            -H 'Content-Type: application/json' \\\n            -d @request.json \\\n            -o api_response.json\n\n          # Extract the text content from the API response\n          jq -r '.candidates[0].content.parts[0].text' api_response.json > review_output.json\n```\n\n## Step 4: Handling Large Diffs\n\nFor large PRs, you may hit token limits. Add a truncation step:\n\n```yaml\n      - name: Prepare diff (truncated if needed)\n        run: |\n          # Limit diff to ~100KB to stay within token limits\n          if [ $(wc -c < pr_diff.txt) -gt 100000 ]; then\n            head -c 100000 pr_diff.txt > pr_diff_truncated.txt\n            echo -e \"\\n\\n[DIFF TRUNCATED - showing first 100KB]\" >> pr_diff_truncated.txt\n            mv pr_diff_truncated.txt pr_diff.txt\n          fi\n```\n\n## Key Configuration Notes\n\n### Authentication\n\n- Gemini CLI uses the `GEMINI_API_KEY` environment variable for authentication\n- In GitHub Actions, pass it via `env:` from secrets -- never hardcode it\n\n### Structured JSON Output\n\nThere are two approaches for getting structured JSON:\n\n1. **Prompt-based** (CLI approach): Ask the model to return JSON in your prompt. Less reliable but simpler. Include \"Respond ONLY with valid JSON\" in your prompt.\n\n2. **Schema-based** (API approach): Use `responseMimeType: \"application/json\"` with a `responseSchema` in the API request. This guarantees valid JSON output conforming to your schema. This is the more reliable approach for CI/CD pipelines.\n\n### Parsing the Output\n\nAlways include fallback handling when parsing JSON from LLM output:\n\n```bash\n# Strip potential markdown code fences\nCLEAN_JSON=$(echo \"$RAW_OUTPUT\" | sed 's/^```json//; s/^```//; /^$/d')\n\n# Validate with jq\nif echo \"$CLEAN_JSON\" | jq . > /dev/null 2>&1; then\n  echo \"Valid JSON\"\nelse\n  echo \"Invalid JSON - falling back\"\nfi\n```\n\n### Cost and Rate Limiting\n\n- Consider adding rate limiting or caching to avoid excessive API calls\n- Use `concurrency` in your workflow to prevent parallel runs:\n\n```yaml\nconcurrency:\n  group: gemini-review-${{ github.event.pull_request.number }}\n  cancel-in-progress: true\n```\n\n### Security Considerations\n\n- Never log or echo the API key in workflow output\n- Use `pull_requests: write` permission (minimum required)\n- Consider limiting the workflow to specific file types or paths:\n\n```yaml\non:\n  pull_request:\n    paths:\n      - '**.js'\n      - '**.ts'\n      - '**.py'\n```\n\n## Complete Minimal Example\n\nFor a quick start, here is the simplest possible setup:\n\n```yaml\nname: Gemini Review\non:\n  pull_request:\n    types: [opened, synchronize]\n\npermissions:\n  contents: read\n  pull-requests: write\n\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v4\n        with:\n          fetch-depth: 0\n\n      - name: Setup and Review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          npm install -g @google/gemini-cli\n          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD)\n          REVIEW=$(gemini -p \"Review this code diff. Return JSON with keys: summary (string), issues (array of {severity, file, description}), approved (boolean). Only output JSON. Diff: $DIFF\")\n          echo \"$REVIEW\" > review.json\n\n      - name: Post Comment\n        uses: actions/github-script@v7\n        with:\n          script: |\n            const fs = require('fs');\n            const raw = fs.readFileSync('review.json', 'utf8');\n            const clean = raw.replace(/```json\\n?/g, '').replace(/```\\n?/g, '').trim();\n            let review;\n            try { review = JSON.parse(clean); } catch(e) {\n              return github.rest.issues.createComment({\n                ...context.repo, issue_number: context.issue.number,\n                body: '## Gemini Review\\nFailed to parse output.'\n              });\n            }\n            let body = `## Gemini Review\\n**Summary:** ${review.summary}\\n\\n`;\n            if (review.issues?.length) {\n              body += review.issues.map(i => `- **${i.severity}** \\`${i.file}\\`: ${i.description}`).join('\\n');\n            } else { body += 'No issues found.'; }\n            await github.rest.issues.createComment({\n              ...context.repo, issue_number: context.issue.number, body\n            });\n```\n\nThis setup will automatically run Gemini CLI on every PR, parse the structured JSON review, and post the results as a PR comment.\n"}], "grading": {"eval_id": 1, "eval_name": "cicd-setup", "config": "without_skill", "expectations": [{"text": "Uses -p or --prompt flag for headless/non-interactive mode", "passed": true, "evidence": "Line 96: gemini -p \"$(cat review_prompt.txt)\" and line 351: gemini -p \"Review this code diff...\""}, {"text": "Includes --output-format json flag for structured output", "passed": false, "evidence": "Does not use --output-format json. Instead relies on prompt-based JSON (asking model to return JSON in prompt text). Line 95 comment mentions '--json flag if available' which is not a real Gemini CLI flag."}, {"text": "Shows how to set GEMINI_API_KEY from GitHub secrets", "passed": true, "evidence": "Line 62: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} in env block"}, {"text": "Provides a GitHub Actions workflow YAML snippet", "passed": true, "evidence": "Complete workflow YAML provided with checkout, node setup, install, review, and post-comment steps"}, {"text": "Shows how to parse JSON response (e.g., with jq .response)", "passed": true, "evidence": "Uses jq for JSON validation (line 102) and parsing (line 248: jq -r '.candidates[0].content.parts[0].text'). Different schema than CLI --output-format json but still demonstrates jq parsing."}, {"text": "Does not invent non-existent CLI flags or options", "passed": false, "evidence": "Line 95: comment references '--json flag if available' which is not a real Gemini CLI flag. Should use --output-format json instead."}]}}, {"id": "eval-2-mcp-server-config-with_skill", "prompt": "I need to connect a custom MCP server I built in Python to gemini cli. The server needs an API key from my environment. Also I want to restrict which tools it can expose. How do I configure this?", "eval_id": 2, "outputs": [{"name": "response.md", "type": "text", "content": "# Connecting a Custom Python MCP Server to Gemini CLI\n\n## Configuration\n\nAdd your server to `~/.gemini/settings.json` (global) or `<project>/.gemini/settings.json` (project-scoped) under the `mcpServers` key:\n\n```json\n{\n  \"mcpServers\": {\n    \"myPythonServer\": {\n      \"command\": \"python\",\n      \"args\": [\"-m\", \"my_mcp_server\"],\n      \"cwd\": \"./mcp_tools\",\n      \"env\": {\n        \"API_KEY\": \"$MY_API_KEY\"\n      },\n      \"timeout\": 15000,\n      \"trust\": false,\n      \"includeTools\": [\"tool_a\", \"tool_b\"]\n    }\n  }\n}\n```\n\n## Breakdown of each field\n\n| Field | Purpose |\n|---|---|\n| `command` | The executable to run. Use `\"python\"` (or `\"python3\"` depending on your system). |\n| `args` | Arguments passed to the command. Use `[\"-m\", \"my_mcp_server\"]` to run as a module, or `[\"./path/to/server.py\"]` to run a script directly. |\n| `cwd` | Optional working directory for the server process. |\n| `env` | Environment variables passed to the server. Use the `\"$VAR_NAME\"` syntax to reference variables from your shell environment rather than hardcoding secrets. |\n| `timeout` | Connection timeout in milliseconds. Defaults to 600000 (10 minutes). Set lower (e.g., 15000) for servers that should start quickly. |\n| `trust` | When `false` (the default), Gemini CLI will prompt you for confirmation before executing tools from this server. Set to `true` only for servers you fully control and trust. |\n| `includeTools` | A whitelist array. Only the tools listed here will be exposed to Gemini. All other tools from the server are hidden. |\n| `excludeTools` | A blacklist array. The listed tools are hidden; everything else is exposed. |\n\nUse `includeTools` or `excludeTools`, not both. `includeTools` is the safer choice when you want to tightly restrict what the model can call.\n\n## Passing the API key securely\n\nNever hardcode secrets in `settings.json`. Instead, set the variable in your shell environment:\n\n```bash\nexport MY_API_KEY=\"sk-your-actual-key\"\n```\n\nThen reference it with the dollar-sign prefix in the config:\n\n```json\n\"env\": {\n  \"API_KEY\": \"$MY_API_KEY\"\n}\n```\n\nGemini CLI automatically redacts environment variable values that match patterns like `*TOKEN*`, `*SECRET*`, `*KEY*` from MCP process output, so your key will not leak into logs.\n\nYou can also store the variable in `~/.gemini/.env` or `<project>/.gemini/.env` and it will be loaded automatically.\n\n## Restricting exposed tools\n\nTo allow only specific tools (whitelist approach):\n\n```json\n\"includeTools\": [\"query_data\", \"list_items\"]\n```\n\nTo block specific tools while allowing everything else (blacklist approach):\n\n```json\n\"excludeTools\": [\"delete_all\", \"admin_reset\"]\n```\n\n## Verifying the connection\n\nAfter configuring, start Gemini CLI and check the server status:\n\n```bash\ngemini\n```\n\nThen inside the interactive session:\n\n```\n/mcp list\n/mcp status\n```\n\nYou can also manage servers from the command line:\n\n```bash\ngemini mcp list\ngemini mcp remove myPythonServer\n```\n\n## Invoking your server's tools\n\nIn the interactive session, reference the server by name with the `@` prefix:\n\n```\n@myPythonServer query all active users\n```\n\nGemini CLI will route the request to the appropriate tool exposed by your MCP server.\n\n## Complete example\n\nAssuming your Python MCP server is at `./tools/my_server.py` and exposes tools named `search`, `fetch_details`, and `delete_record`, and you only want to allow `search` and `fetch_details`:\n\n```json\n{\n  \"mcpServers\": {\n    \"myPythonServer\": {\n      \"command\": \"python\",\n      \"args\": [\"./tools/my_server.py\"],\n      \"env\": {\n        \"API_KEY\": \"$MY_API_KEY\"\n      },\n      \"timeout\": 15000,\n      \"trust\": false,\n      \"includeTools\": [\"search\", \"fetch_details\"]\n    }\n  }\n}\n```\n\nThis configuration starts the Python server, passes the API key from your environment, sets a 15-second connection timeout, requires confirmation prompts for tool calls, and only exposes the `search` and `fetch_details` tools to Gemini.\n"}], "grading": {"eval_id": 2, "eval_name": "mcp-server-config", "config": "with_skill", "expectations": [{"text": "Points to ~/.gemini/settings.json or project-level settings.json", "passed": true, "evidence": "Opens with: 'Add your server to ~/.gemini/settings.json (global) or <project>/.gemini/settings.json (project-scoped)'"}, {"text": "Shows mcpServers config with command, args fields", "passed": true, "evidence": "Full JSON config block with mcpServers containing command, args, cwd, env, timeout, trust, includeTools"}, {"text": "Shows $VAR pattern for environment variable references in env field", "passed": true, "evidence": "Shows \"API_KEY\": \"$MY_API_KEY\" pattern and explains the dollar-sign prefix syntax"}, {"text": "Explains includeTools and/or excludeTools for restricting exposed tools", "passed": true, "evidence": "Dedicated section explaining both includeTools (whitelist) and excludeTools (blacklist) with examples"}, {"text": "Mentions security considerations (trust field, not hardcoding keys, etc.)", "passed": true, "evidence": "Explains trust field (default false, requires confirmation), warns never to hardcode secrets, mentions automatic redaction of env vars"}, {"text": "Uses correct Python command (python/python3) in the command field for the MCP server", "passed": true, "evidence": "Uses \"command\": \"python\" consistently across examples"}]}}, {"id": "eval-2-mcp-server-config-without_skill", "prompt": "I need to connect a custom MCP server I built in Python to gemini cli. The server needs an API key from my environment. Also I want to restrict which tools it can expose. How do I configure this?", "eval_id": 2, "outputs": [{"name": "response.md", "type": "text", "content": "# Connecting a Custom Python MCP Server to Gemini CLI\n\n## Configuration File Location\n\nGemini CLI reads MCP server configuration from `settings.json` at two levels:\n\n- **Global (user-level):** `~/.gemini/settings.json` -- applies to all Gemini CLI sessions\n- **Project-level:** `.gemini/settings.json` in your project root -- applies only when running from that project\n\nProject-level settings override global ones.\n\n## Full Configuration Example\n\nHere is how to configure a custom Python MCP server with an API key from your environment and tool filtering:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-python-server\": {\n      \"command\": \"python\",\n      \"args\": [\"path/to/your/server.py\"],\n      \"env\": {\n        \"API_KEY\": \"$MY_API_KEY\"\n      },\n      \"cwd\": \"/path/to/server/directory\",\n      \"timeout\": 30000,\n      \"includeTools\": [\"allowed_tool_1\", \"allowed_tool_2\", \"allowed_tool_3\"]\n    }\n  }\n}\n```\n\n## Explanation of Each Property\n\n### Required (choose one transport)\n\n| Property | Description |\n|----------|-------------|\n| `command` | Path to the executable for stdio transport (e.g., `\"python\"`, `\"node\"`) |\n| `url` | SSE endpoint URL for remote servers (e.g., `\"http://localhost:8080/sse\"`) |\n| `httpUrl` | HTTP streaming endpoint URL |\n\n### Optional Properties\n\n| Property | Type | Description |\n|----------|------|-------------|\n| `args` | string[] | Command-line arguments passed to the server process |\n| `env` | object | Environment variables to pass to the server (supports expansion) |\n| `cwd` | string | Working directory for the server process |\n| `timeout` | number | Request timeout in milliseconds (default: 600,000ms / 10 minutes) |\n| `trust` | boolean | When `true`, bypasses tool confirmation prompts |\n| `includeTools` | string[] | Allowlist -- only these tools will be available |\n| `excludeTools` | string[] | Blocklist -- these tools will be hidden from the model |\n| `headers` | object | Custom HTTP headers (for remote transports) |\n\n## Passing the API Key via Environment Variables\n\nGemini CLI supports environment variable expansion in the `env` block using two syntaxes:\n\n```json\n\"env\": {\n  \"API_KEY\": \"$MY_API_KEY\",\n  \"OTHER_SECRET\": \"${ANOTHER_ENV_VAR}\"\n}\n```\n\nBoth `$VAR_NAME` and `${VAR_NAME}` are supported. At runtime, Gemini CLI reads the value of `MY_API_KEY` from your host shell environment and passes it to the MCP server process.\n\n**Security note:** Gemini CLI automatically redacts environment variables to prevent leakage. However, variables you explicitly declare in the `env` block are considered trusted (since you intentionally configured them) and will not be redacted when passed to that specific server. You must explicitly list any environment variable that the server needs -- variables are not inherited automatically.\n\n## Restricting Exposed Tools\n\nGemini CLI provides two complementary mechanisms for tool filtering:\n\n### Allowlist with `includeTools`\n\nOnly the tools listed here will be available from that server. All other tools the server exposes will be hidden:\n\n```json\n\"includeTools\": [\"query_data\", \"get_status\", \"run_analysis\"]\n```\n\n### Blocklist with `excludeTools`\n\nTools listed here will be hidden, even if the server exposes them:\n\n```json\n\"excludeTools\": [\"delete_records\", \"admin_reset\"]\n```\n\n### Precedence Rule\n\n`excludeTools` takes precedence over `includeTools`. If a tool appears in both lists, it will be excluded.\n\n### Choosing an Approach\n\n- Use `includeTools` when your server exposes many tools but you only want a specific subset available (allowlist approach).\n- Use `excludeTools` when you want most tools available but need to block a few dangerous ones (blocklist approach).\n- You can combine both for fine-grained control.\n\n## Complete Practical Example\n\nSuppose you built a Python MCP server at `~/mcp-servers/my_api_server.py` that exposes tools `fetch_data`, `analyze`, `delete_all`, and `export_csv`. You want to pass your API key and only allow `fetch_data` and `analyze`.\n\nCreate or edit `~/.gemini/settings.json`:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-api-server\": {\n      \"command\": \"python\",\n      \"args\": [\"/home/user/mcp-servers/my_api_server.py\"],\n      \"env\": {\n        \"API_KEY\": \"$MY_API_KEY\"\n      },\n      \"timeout\": 30000,\n      \"includeTools\": [\"fetch_data\", \"analyze\"]\n    }\n  }\n}\n```\n\nMake sure `MY_API_KEY` is set in your shell environment (e.g., in `~/.bashrc` or `~/.zshrc`):\n\n```bash\nexport MY_API_KEY=\"your-secret-key-here\"\n```\n\n### Using FastMCP\n\nIf your Python server uses FastMCP (the recommended approach for Python MCP servers), the configuration would look like:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-api-server\": {\n      \"command\": \"uv\",\n      \"args\": [\"run\", \"--with\", \"fastmcp\", \"fastmcp\", \"run\", \"my_server.py\"],\n      \"cwd\": \"/home/user/mcp-servers\",\n      \"env\": {\n        \"API_KEY\": \"$MY_API_KEY\"\n      },\n      \"includeTools\": [\"fetch_data\", \"analyze\"]\n    }\n  }\n}\n```\n\n### Using the CLI to Add a Server\n\nYou can also add a server via the command line instead of editing `settings.json` manually:\n\n```bash\ngemini mcp add my-api-server -e API_KEY=$MY_API_KEY python /home/user/mcp-servers/my_api_server.py\n```\n\nNote: Tool filtering (`includeTools`/`excludeTools`) must be configured by editing `settings.json` directly, as the CLI `add` command may not support these options as flags.\n\n## Sources\n\n- [MCP servers with the Gemini CLI (official docs)](https://google-gemini.github.io/gemini-cli/docs/tools/mcp-server.html)\n- [Gemini CLI configuration reference](https://google-gemini.github.io/gemini-cli/docs/get-started/configuration.html)\n- [Gemini CLI + FastMCP integration (Google Developers Blog)](https://developers.googleblog.com/en/gemini-cli-fastmcp-simplifying-mcp-server-development/)\n- [MCP server docs on GitHub](https://github.com/google-gemini/gemini-cli/blob/main/docs/tools/mcp-server.md)\n"}], "grading": {"eval_id": 2, "eval_name": "mcp-server-config", "config": "without_skill", "expectations": [{"text": "Points to ~/.gemini/settings.json or project-level settings.json", "passed": true, "evidence": "States '~/.gemini/settings.json' (global) and '.gemini/settings.json' (project-level)"}, {"text": "Shows mcpServers config with command, args fields", "passed": true, "evidence": "Full JSON config with mcpServers, command, args, env, cwd, timeout, includeTools"}, {"text": "Shows $VAR pattern for environment variable references in env field", "passed": true, "evidence": "Shows both $MY_API_KEY and ${ANOTHER_ENV_VAR} syntax with explanation"}, {"text": "Explains includeTools and/or excludeTools for restricting exposed tools", "passed": true, "evidence": "Dedicated sections for both includeTools (allowlist) and excludeTools (blocklist) with examples and precedence rules"}, {"text": "Mentions security considerations (trust field, not hardcoding keys, etc.)", "passed": true, "evidence": "Mentions trust field, automatic redaction, env variable security note about trusted declarations"}, {"text": "Uses correct Python command (python/python3) in the command field for the MCP server", "passed": true, "evidence": "Uses \"command\": \"python\" consistently, also shows uv-based FastMCP alternative"}]}}, {"id": "eval-3-custom-slash-command-with_skill", "prompt": "How do I create a custom slash command in gemini cli that runs git diff --cached and asks the AI to write a commit message? I want it available in all my projects.", "eval_id": 3, "outputs": [{"name": "response.md", "type": "text", "content": "# Creating a Global Custom Slash Command for Commit Messages in Gemini CLI\n\nTo create a custom slash command that runs `git diff --cached` and asks the AI to write a commit message, available across all your projects, create a single `.toml` file in your user-scoped commands directory.\n\n## Step-by-Step\n\n### 1. Create the commands directory (if it doesn't exist)\n\n```bash\nmkdir -p ~/.gemini/commands\n```\n\n### 2. Create the slash command file\n\nCreate the file `~/.gemini/commands/commit.toml`:\n\n```toml\ndescription = \"Generate a commit message from staged changes\"\nprompt = \"\"\"\nBased on the following staged git diff, write a concise and descriptive commit message following conventional commit format (e.g., feat:, fix:, refactor:, docs:, chore:).\n\nThe first line should be a short summary (50 chars or less). If needed, add a blank line followed by a more detailed explanation.\n\nStaged changes:\n!{git diff --cached}\n\"\"\"\n```\n\n### How it works\n\n- **File location**: `~/.gemini/commands/commit.toml` -- placing it under `~/.gemini/commands/` makes it user-scoped, meaning it is available in every project you open with Gemini CLI.\n- **`!{git diff --cached}`**: The `!{...}` syntax is shell execution interpolation. When the slash command runs, Gemini CLI executes `git diff --cached` and substitutes the output directly into the prompt before sending it to the model.\n- **`description`**: Optional but recommended. It shows up when you run `/help` to list available commands.\n- **`prompt`**: The full prompt template sent to the model, with the shell output injected.\n\n### 3. Use the command\n\nStart an interactive Gemini CLI session in any git repository:\n\n```bash\ngemini\n```\n\nThen stage your changes and invoke the command:\n\n```\n> /commit\n```\n\nGemini will run `git diff --cached`, inject the diff into the prompt, and return a suggested commit message.\n\n## Variations\n\n### Accept an argument for extra context\n\nIf you want to optionally pass additional context (e.g., a ticket number or description):\n\n```toml\ndescription = \"Generate a commit message from staged changes\"\nprompt = \"\"\"\nBased on the following staged git diff, write a concise commit message.\n\nAdditional context from the user: {{args}}\n\nStaged changes:\n!{git diff --cached}\n\"\"\"\n```\n\nUsage: `/commit JIRA-1234 add user validation`\n\nThe `{{args}}` placeholder captures everything typed after `/commit`.\n\n### Namespaced command\n\nIf you prefer organizing under a `git` namespace, place the file at:\n\n```\n~/.gemini/commands/git/commit.toml\n```\n\nThen invoke it as `/git:commit` in the interactive session.\n\n## Important notes\n\n- Custom slash commands work in **interactive mode only**. They are not available in headless mode (`-p` flag). For headless commit message generation, pipe the diff directly:\n  ```bash\n  git diff --cached | gemini -p \"Write a concise commit message\" --output-format json | jq -r '.response'\n  ```\n- The command runs in whatever directory you launched `gemini` from, so `git diff --cached` will pick up the correct repository context automatically.\n- If there are no staged changes, the diff output will be empty, and the model will tell you there is nothing to generate a message for.\n"}], "grading": {"eval_id": 3, "eval_name": "custom-slash-command", "config": "with_skill", "expectations": [{"text": "Points to ~/.gemini/commands/ for global user-scoped commands", "passed": true, "evidence": "States 'Create the file ~/.gemini/commands/commit.toml' and explains ~/.gemini/commands/ is user-scoped"}, {"text": "Shows .toml file format with prompt field", "passed": true, "evidence": "Complete .toml example with description and prompt fields using triple-quoted string"}, {"text": "Uses !{git diff --cached} shell execution syntax in the prompt", "passed": true, "evidence": "Uses !{git diff --cached} in the prompt field and explains the !{...} syntax is shell execution interpolation"}, {"text": "Warns that custom slash commands don't work in headless/non-interactive mode", "passed": true, "evidence": "Important notes section: 'Custom slash commands work in interactive mode only. They are not available in headless mode (-p flag).'"}, {"text": "Provides a complete, copy-pasteable .toml file example", "passed": true, "evidence": "Complete commit.toml file with description and prompt fields, ready to copy-paste"}]}}, {"id": "eval-3-custom-slash-command-without_skill", "prompt": "How do I create a custom slash command in gemini cli that runs git diff --cached and asks the AI to write a commit message? I want it available in all my projects.", "eval_id": 3, "outputs": [{"name": "response.md", "type": "text", "content": "# Creating a Custom Slash Command in Gemini CLI for Commit Messages\n\nGemini CLI supports custom slash commands defined as `.toml` files. To make a command available across all your projects, place the file in the **global** commands directory: `~/.gemini/commands/`.\n\n## Step-by-Step Setup\n\n### 1. Create the commands directory (if it does not exist)\n\n```bash\nmkdir -p ~/.gemini/commands\n```\n\n### 2. Create the command file\n\nCreate the file `~/.gemini/commands/commit.toml`:\n\n```toml\ndescription = \"Generate a commit message from staged changes\"\nprompt = \"\"\"Review the following staged git diff and write a clear, conventional commit message for it. Follow the Conventional Commits format (e.g., feat:, fix:, refactor:, docs:, chore:). Include a concise subject line (max 72 characters) and, if the changes are non-trivial, a body that explains the \"why\" behind the changes.\n\n```diff\n!{git diff --cached}\n```\"\"\"\n```\n\nThis creates the slash command `/commit`.\n\n### How It Works\n\n- **`description`** -- A one-line summary shown when you run `/help` inside Gemini CLI.\n- **`prompt`** -- The full prompt sent to the Gemini model. The `!{git diff --cached}` syntax is a **shell command injection block**. When the command runs, Gemini CLI executes `git diff --cached` in your shell, captures the output, and injects it directly into the prompt at that location.\n- **Security note** -- The first time a shell command inside `!{...}` runs, Gemini CLI will prompt you for confirmation before executing it.\n\n### 3. Reload commands (if Gemini CLI is already running)\n\nInside an active Gemini CLI session, run:\n\n```\n/commands reload\n```\n\nThis picks up new or modified `.toml` files without restarting the CLI.\n\n### 4. Usage\n\nStage your changes and invoke the command:\n\n```bash\ngit add -p                # stage your changes\ngemini                    # start Gemini CLI\n```\n\nThen inside the Gemini CLI session:\n\n```\n/commit\n```\n\nGemini will read your staged diff, analyze the changes, and generate a commit message for you.\n\n## Namespaced Variant\n\nIf you prefer organizing your commands by category, use subdirectories. For example, placing the file at:\n\n```\n~/.gemini/commands/git/commit.toml\n```\n\nmakes it available as `/git:commit` instead of `/commit`. Subdirectory separators are converted to colons in the command name.\n\n## Adding Arguments Support\n\nYou can extend the command to accept optional context using `{{args}}`:\n\n```toml\ndescription = \"Generate a commit message from staged changes with optional context\"\nprompt = \"\"\"Review the following staged git diff and write a clear, conventional commit message. Follow the Conventional Commits format (e.g., feat:, fix:, refactor:).\n\nAdditional context from the developer: {{args}}\n\n```diff\n!{git diff --cached}\n```\"\"\"\n```\n\nThis allows you to provide extra context when invoking the command:\n\n```\n/commit This change addresses the performance regression in the auth module\n```\n\nThe `{{args}}` placeholder gets replaced with whatever text you type after the command name. Inside `!{...}` blocks, arguments are automatically shell-escaped for safety.\n\n## Summary of Key Concepts\n\n| Concept | Detail |\n|---|---|\n| Global location | `~/.gemini/commands/` |\n| Project location | `<project-root>/.gemini/commands/` |\n| File format | TOML (`.toml` extension) |\n| Shell execution | `!{command}` syntax, requires confirmation |\n| Arguments | `{{args}}` placeholder |\n| File injection | `@{path/to/file}` syntax |\n| Namespacing | Subdirectories map to colon-separated names |\n| Precedence | Project commands override global commands with the same name |\n\n## Sources\n\n- [Custom commands - Gemini CLI Documentation](https://geminicli.com/docs/cli/custom-commands/)\n- [Gemini CLI: Custom slash commands - Google Cloud Blog](https://cloud.google.com/blog/topics/developers-practitioners/gemini-cli-custom-slash-commands)\n- [CLI Commands - Gemini CLI](https://google-gemini.github.io/gemini-cli/docs/cli/commands.html)\n- [Gemini CLI Tutorial Series Part 7: Custom slash commands - Medium](https://medium.com/google-cloud/gemini-cli-tutorial-series-part-7-custom-slash-commands-64c06195294b)\n"}], "grading": {"eval_id": 3, "eval_name": "custom-slash-command", "config": "without_skill", "expectations": [{"text": "Points to ~/.gemini/commands/ for global user-scoped commands", "passed": true, "evidence": "States '~/.gemini/commands/' as global location in both instructions and summary table"}, {"text": "Shows .toml file format with prompt field", "passed": true, "evidence": "Complete .toml example with description and prompt fields"}, {"text": "Uses !{git diff --cached} shell execution syntax in the prompt", "passed": true, "evidence": "Uses !{git diff --cached} in the prompt field and explains the shell command injection block syntax"}, {"text": "Warns that custom slash commands don't work in headless/non-interactive mode", "passed": false, "evidence": "No mention of headless mode limitations anywhere in the response. Missing this critical caveat."}, {"text": "Provides a complete, copy-pasteable .toml file example", "passed": true, "evidence": "Complete commit.toml file at ~/.gemini/commands/commit.toml with description and prompt fields"}]}}], "previous_feedback": {}, "previous_outputs": {"eval-1-cicd-setup-with_skill": [{"name": "response.md", "type": "text", "content": "# Setting Up Gemini CLI for Automated PR Reviews in GitHub Actions\n\nHere is a complete guide to configuring Gemini CLI in your CI/CD pipeline to automatically review pull requests with structured JSON output.\n\n---\n\n## Prerequisites\n\n1. A Gemini API key (get one at https://aistudio.google.com/apikey).\n2. The API key stored as a GitHub Actions secret named `GEMINI_API_KEY`.\n\n---\n\n## Step 1: Basic GitHub Actions Workflow\n\nCreate `.github/workflows/gemini-pr-review.yml`:\n\n```yaml\nname: Gemini PR Review\n\non:\n  pull_request:\n    types: [opened, synchronize, reopened]\n\npermissions:\n  pull-requests: write\n  contents: read\n\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout code\n        uses: actions/checkout@v4\n        with:\n          fetch-depth: 0  # Full history so we can diff against base\n\n      - name: Set up Node.js\n        uses: actions/setup-node@v4\n        with:\n          node-version: '20'\n\n      - name: Install Gemini CLI\n        run: npm install -g @google/gemini-cli\n\n      - name: Get PR diff\n        id: diff\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          git diff origin/${{ github.base_ref }}...HEAD > /tmp/pr-diff.txt\n          echo \"diff_size=$(wc -c < /tmp/pr-diff.txt)\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Run Gemini PR Review\n        id: review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          REVIEW_JSON=$(cat /tmp/pr-diff.txt | gemini -p \"You are a senior code reviewer. Review the following pull request diff for:\n          1. Bugs and logical errors\n          2. Security vulnerabilities\n          3. Performance issues\n          4. Code style and best practices\n\n          Provide your review as structured text with sections for each category.\n          If no issues are found in a category, say 'No issues found.'\n          End with an overall summary and a verdict: APPROVE, REQUEST_CHANGES, or COMMENT.\" \\\n            --output-format json)\n\n          # Extract the response text from the JSON envelope\n          REVIEW_TEXT=$(echo \"$REVIEW_JSON\" | jq -r '.response')\n\n          # Extract token stats for logging\n          TOTAL_TOKENS=$(echo \"$REVIEW_JSON\" | jq '.stats.models | to_entries[0].value.tokens.total // 0')\n          echo \"Tokens used: $TOTAL_TOKENS\"\n\n          # Save review to file (handles multi-line safely)\n          echo \"$REVIEW_TEXT\" > /tmp/review-output.txt\n\n          # Save for the summary\n          echo \"$REVIEW_TEXT\" >> \"$GITHUB_STEP_SUMMARY\"\n\n      - name: Post review as PR comment\n        uses: actions/github-script@v7\n        with:\n          script: |\n            const fs = require('fs');\n            const review = fs.readFileSync('/tmp/review-output.txt', 'utf8');\n            await github.rest.issues.createComment({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              issue_number: context.issue.number,\n              body: `## Gemini Code Review\\n\\n${review}\\n\\n---\\n*Automated review by Gemini CLI*`\n            });\n```\n\nThis workflow triggers on every PR event, runs the diff through Gemini CLI with `--output-format json`, extracts the response with `jq`, and posts it as a PR comment.\n\n---\n\n## Step 2: Advanced Workflow with Full JSON Parsing\n\nIf you want to parse the structured JSON output more thoroughly -- for example, to extract token usage, check for errors, or conditionally fail the build -- use this expanded version:\n\n```yaml\nname: Gemini PR Review (Advanced)\n\non:\n  pull_request:\n    types: [opened, synchronize]\n\npermissions:\n  pull-requests: write\n  contents: read\n\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout\n        uses: actions/checkout@v4\n        with:\n          fetch-depth: 0\n\n      - name: Setup Node.js\n        uses: actions/setup-node@v4\n        with:\n          node-version: '20'\n\n      - name: Install Gemini CLI and jq\n        run: |\n          npm install -g @google/gemini-cli\n          # jq is pre-installed on ubuntu-latest, but just in case:\n          which jq || sudo apt-get install -y jq\n\n      - name: Run structured review\n        id: gemini\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD)\n\n          # Skip review if diff is empty\n          if [ -z \"$DIFF\" ]; then\n            echo \"No changes to review.\"\n            echo \"skip=true\" >> \"$GITHUB_OUTPUT\"\n            exit 0\n          fi\n\n          # Run Gemini with JSON output\n          RESULT=$(echo \"$DIFF\" | gemini -p \"You are a code review bot. Analyze this PR diff and respond with ONLY a valid JSON object (no markdown fences) using this exact schema:\n          {\n            \\\"verdict\\\": \\\"APPROVE\\\" | \\\"REQUEST_CHANGES\\\" | \\\"COMMENT\\\",\n            \\\"summary\\\": \\\"Brief overall summary\\\",\n            \\\"bugs\\\": [{ \\\"file\\\": \\\"path\\\", \\\"line\\\": number, \\\"description\\\": \\\"...\\\" }],\n            \\\"security\\\": [{ \\\"file\\\": \\\"path\\\", \\\"severity\\\": \\\"low|medium|high|critical\\\", \\\"description\\\": \\\"...\\\" }],\n            \\\"performance\\\": [{ \\\"file\\\": \\\"path\\\", \\\"description\\\": \\\"...\\\" }],\n            \\\"suggestions\\\": [{ \\\"file\\\": \\\"path\\\", \\\"description\\\": \\\"...\\\" }]\n          }\n          Return empty arrays if no issues found in a category.\" \\\n            --output-format json)\n\n          # The outer JSON has a .response field containing the model's text.\n          # The model's text itself is a JSON string we need to parse.\n          REVIEW_RESPONSE=$(echo \"$RESULT\" | jq -r '.response')\n\n          # Parse the inner JSON (the actual review)\n          echo \"$REVIEW_RESPONSE\" | jq '.' > /tmp/review.json\n\n          # Extract fields for downstream steps\n          VERDICT=$(cat /tmp/review.json | jq -r '.verdict')\n          SUMMARY=$(cat /tmp/review.json | jq -r '.summary')\n          BUG_COUNT=$(cat /tmp/review.json | jq '.bugs | length')\n          SECURITY_COUNT=$(cat /tmp/review.json | jq '.security | length')\n          CRITICAL_COUNT=$(cat /tmp/review.json | jq '[.security[] | select(.severity == \"critical\")] | length')\n\n          echo \"verdict=$VERDICT\" >> \"$GITHUB_OUTPUT\"\n          echo \"bug_count=$BUG_COUNT\" >> \"$GITHUB_OUTPUT\"\n          echo \"security_count=$SECURITY_COUNT\" >> \"$GITHUB_OUTPUT\"\n          echo \"critical_count=$CRITICAL_COUNT\" >> \"$GITHUB_OUTPUT\"\n          echo \"skip=false\" >> \"$GITHUB_OUTPUT\"\n\n          # Log token usage from the Gemini stats envelope\n          TOKENS=$(echo \"$RESULT\" | jq '.stats.models | to_entries[0].value.tokens.total // 0')\n          LATENCY=$(echo \"$RESULT\" | jq '.stats.models | to_entries[0].value.api.totalLatencyMs // 0')\n          echo \"Tokens used: $TOKENS, Latency: ${LATENCY}ms\"\n\n      - name: Format and post review comment\n        if: steps.gemini.outputs.skip != 'true'\n        uses: actions/github-script@v7\n        with:\n          script: |\n            const fs = require('fs');\n            const review = JSON.parse(fs.readFileSync('/tmp/review.json', 'utf8'));\n\n            let body = `## Gemini Code Review\\n\\n`;\n            body += `**Verdict:** ${review.verdict}\\n\\n`;\n            body += `**Summary:** ${review.summary}\\n\\n`;\n\n            if (review.bugs.length > 0) {\n              body += `### Bugs (${review.bugs.length})\\n`;\n              for (const bug of review.bugs) {\n                body += `- **${bug.file}** (line ${bug.line}): ${bug.description}\\n`;\n              }\n              body += '\\n';\n            }\n\n            if (review.security.length > 0) {\n              body += `### Security Issues (${review.security.length})\\n`;\n              for (const issue of review.security) {\n                const emoji = issue.severity === 'critical' ? '\u00f0\u0178\u201d\u00b4' :\n                              issue.severity === 'high' ? '\u00f0\u0178\u0178\u00a0' :\n                              issue.severity === 'medium' ? '\u00f0\u0178\u0178\u00a1' : '\u00f0\u0178\u0178\u00a2';\n                body += `- ${emoji} **[${issue.severity.toUpperCase()}]** ${issue.file}: ${issue.description}\\n`;\n              }\n              body += '\\n';\n            }\n\n            if (review.performance.length > 0) {\n              body += `### Performance (${review.performance.length})\\n`;\n              for (const perf of review.performance) {\n                body += `- **${perf.file}**: ${perf.description}\\n`;\n              }\n              body += '\\n';\n            }\n\n            if (review.suggestions.length > 0) {\n              body += `### Suggestions (${review.suggestions.length})\\n`;\n              for (const sug of review.suggestions) {\n                body += `- **${sug.file}**: ${sug.description}\\n`;\n              }\n              body += '\\n';\n            }\n\n            body += `---\\n*Automated review by Gemini CLI*`;\n\n            await github.rest.issues.createComment({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              issue_number: context.issue.number,\n              body\n            });\n\n      - name: Fail on critical security issues\n        if: steps.gemini.outputs.critical_count != '0' && steps.gemini.outputs.skip != 'true'\n        run: |\n          echo \"::error::Gemini found ${{ steps.gemini.outputs.critical_count }} critical security issue(s). Review required.\"\n          exit 1\n```\n\n---\n\n## Understanding the JSON Output\n\nGemini CLI's `--output-format json` wraps the model's response in an envelope with this schema:\n\n```json\n{\n  \"response\": \"The model's text response (string)\",\n  \"stats\": {\n    \"models\": {\n      \"gemini-2.5-pro\": {\n        \"api\": {\n          \"totalRequests\": 2,\n          \"totalErrors\": 0,\n          \"totalLatencyMs\": 5053\n        },\n        \"tokens\": {\n          \"prompt\": 24939,\n          \"candidates\": 20,\n          \"total\": 25113,\n          \"cached\": 21263,\n          \"thoughts\": 154,\n          \"tool\": 0\n        }\n      }\n    },\n    \"tools\": {\n      \"totalCalls\": 1,\n      \"totalSuccess\": 1,\n      \"totalFail\": 0,\n      \"totalDurationMs\": 1881\n    },\n    \"files\": {\n      \"totalLinesAdded\": 5,\n      \"totalLinesRemoved\": 2\n    }\n  }\n}\n```\n\nTo parse this in your pipeline:\n\n```bash\n# Get the model's text response\necho \"$RESULT\" | jq -r '.response'\n\n# Get total tokens used\necho \"$RESULT\" | jq '.stats.models | to_entries[0].value.tokens.total'\n\n# Get API latency\necho \"$RESULT\" | jq '.stats.models | to_entries[0].value.api.totalLatencyMs'\n\n# Check for API errors\necho \"$RESULT\" | jq '.stats.models | to_entries[0].value.api.totalErrors'\n```\n\n---\n\n## Exit Codes for CI Logic\n\nGemini CLI returns meaningful exit codes you can use for conditional logic:\n\n| Exit Code | Meaning |\n|-----------|---------|\n| 0 | Success |\n| 1 | General error |\n| 2 | Authentication error (bad or missing API key) |\n| 3 | Rate limit / quota exceeded |\n\nExample usage:\n\n```bash\ngemini -p \"Review this code\" --output-format json > /tmp/result.json 2>/dev/null\nEXIT_CODE=$?\n\nif [ $EXIT_CODE -eq 2 ]; then\n  echo \"::error::Gemini API key is invalid or missing. Check your GEMINI_API_KEY secret.\"\n  exit 1\nelif [ $EXIT_CODE -eq 3 ]; then\n  echo \"::warning::Gemini rate limit reached. Skipping review.\"\n  exit 0\nelif [ $EXIT_CODE -ne 0 ]; then\n  echo \"::warning::Gemini CLI failed with exit code $EXIT_CODE. Skipping review.\"\n  exit 0\nfi\n```\n\n---\n\n## Tips and Best Practices\n\n1. **Use `--output-format json` (not `stream-json`)** for CI. The `json` format gives you a single complete JSON object after all processing is done. The `stream-json` format emits newline-delimited JSON events in real-time, which is harder to parse in a script.\n\n2. **Authentication**: Set `GEMINI_API_KEY` as an environment variable. Gemini CLI picks it up automatically. No `settings.json` or `.env` file is needed in CI.\n\n3. **Model selection**: Add `-m gemini-2.5-flash` for faster, cheaper reviews or `-m gemini-2.5-pro` for deeper analysis:\n   ```bash\n   gemini -m gemini-2.5-flash -p \"Review this diff\" --output-format json\n   ```\n\n4. **Large diffs**: Gemini models support up to 1M tokens of context. For very large PRs, consider filtering the diff to only changed files that matter:\n   ```bash\n   git diff origin/main...HEAD -- '*.py' '*.js' '*.ts' | gemini -p \"Review\" --output-format json\n   ```\n\n5. **File references with `@` syntax**: You can reference specific files directly in your prompt:\n   ```bash\n   gemini -p \"Review @./src/auth.py and @./src/middleware.py for security issues\" --output-format json\n   ```\n\n6. **Do NOT use `--yolo` for reviews**. The `--yolo` flag auto-accepts all tool actions (file writes, shell commands). For a read-only review, you do not need it. Only use `--yolo` if you want Gemini to make actual code changes.\n\n7. **Rate limits on the free tier**: 60 requests/minute, 1,000 requests/day. For high-volume repos, consider using a paid API key or adding concurrency limits to your workflow.\n\n8. **Debug logging**: If reviews are producing unexpected output, add `--debug-log` to capture full API traces:\n   ```bash\n   gemini -p \"Review this code\" --output-format json --debug-log ./gemini-debug.jsonl\n   ```\n   Then upload `gemini-debug.jsonl` as a workflow artifact for inspection.\n\n---\n\n## Minimal Quick-Start Example\n\nIf you just want the simplest possible setup to get started:\n\n```yaml\nname: Gemini Review\non: [pull_request]\npermissions:\n  pull-requests: write\n  contents: read\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v4\n        with:\n          fetch-depth: 0\n      - run: npm install -g @google/gemini-cli\n      - name: Review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          REVIEW=$(git diff origin/${{ github.base_ref }}...HEAD | gemini -p \"Review these changes for bugs and issues\" --output-format json | jq -r '.response')\n          echo \"$REVIEW\" >> $GITHUB_STEP_SUMMARY\n```\n\nThis pipes the PR diff into Gemini CLI, extracts the review text from the JSON output with `jq`, and writes it to the GitHub Actions step summary where it is visible directly on the workflow run page.\n"}], "eval-1-cicd-setup-without_skill": [{"name": "response.md", "type": "text", "content": "# Setting Up Gemini CLI for Automated PR Reviews in GitHub Actions\n\nThis guide covers how to configure Gemini CLI in a GitHub Actions CI/CD pipeline to automatically review pull requests, using an API key from GitHub Secrets and structured JSON output for parsing.\n\n## Prerequisites\n\n- A Google AI API key (from [Google AI Studio](https://aistudio.google.com/apikey))\n- The API key stored as a GitHub repository secret (e.g., `GEMINI_API_KEY`)\n- A GitHub repository with pull request workflows\n\n## Step 1: Store Your API Key in GitHub Secrets\n\n1. Go to your repository on GitHub.\n2. Navigate to **Settings > Secrets and variables > Actions**.\n3. Click **New repository secret**.\n4. Name it `GEMINI_API_KEY` and paste your API key as the value.\n\n## Step 2: Create the GitHub Actions Workflow\n\nCreate a file at `.github/workflows/gemini-pr-review.yml`:\n\n```yaml\nname: Gemini PR Review\n\non:\n  pull_request:\n    types: [opened, synchronize, reopened]\n\npermissions:\n  contents: read\n  pull-requests: write\n\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@v4\n        with:\n          fetch-depth: 0\n\n      - name: Set up Node.js\n        uses: actions/setup-node@v4\n        with:\n          node-version: '20'\n\n      - name: Install Gemini CLI\n        run: npm install -g @anthropic-ai/gemini-cli || npm install -g gemini-cli\n\n      - name: Get PR diff\n        id: diff\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD)\n          # Write diff to a file to avoid shell escaping issues\n          echo \"$DIFF\" > /tmp/pr_diff.txt\n\n      - name: Get changed files list\n        id: files\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)\n          echo \"$FILES\" > /tmp/changed_files.txt\n\n      - name: Run Gemini CLI review\n        id: gemini-review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          DIFF=$(cat /tmp/pr_diff.txt)\n          FILES=$(cat /tmp/changed_files.txt)\n\n          PROMPT=$(cat <<'PROMPT_EOF'\n          You are a senior code reviewer. Analyze the following pull request diff and provide a structured review.\n\n          Changed files:\n          FILE_LIST_PLACEHOLDER\n\n          Diff:\n          DIFF_PLACEHOLDER\n\n          Respond ONLY with valid JSON in this exact format (no markdown fencing, no extra text):\n          {\n            \"summary\": \"Brief summary of the changes\",\n            \"risk_level\": \"low|medium|high\",\n            \"issues\": [\n              {\n                \"file\": \"path/to/file\",\n                \"line\": 42,\n                \"severity\": \"error|warning|suggestion\",\n                \"message\": \"Description of the issue\",\n                \"suggestion\": \"How to fix it\"\n              }\n            ],\n            \"positive_aspects\": [\"List of things done well\"],\n            \"approval_recommendation\": \"approve|request_changes|comment\"\n          }\n          PROMPT_EOF\n          )\n\n          # Replace placeholders\n          PROMPT=\"${PROMPT/FILE_LIST_PLACEHOLDER/$FILES}\"\n          PROMPT=\"${PROMPT/DIFF_PLACEHOLDER/$DIFF}\"\n\n          # Run Gemini CLI with the prompt\n          # Using the --json flag if available, or parsing output directly\n          RESPONSE=$(echo \"$PROMPT\" | gemini \\\n            --model gemini-2.0-flash \\\n            --api-key \"$GEMINI_API_KEY\" \\\n            2>/dev/null) || true\n\n          # If gemini CLI is not available, fall back to direct API call\n          if [ -z \"$RESPONSE\" ]; then\n            RESPONSE=$(curl -s \"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${GEMINI_API_KEY}\" \\\n              -H 'Content-Type: application/json' \\\n              -d \"$(jq -n --arg prompt \"$PROMPT\" '{\n                \"contents\": [{\"parts\": [{\"text\": $prompt}]}],\n                \"generationConfig\": {\n                  \"responseMimeType\": \"application/json\",\n                  \"temperature\": 0.2\n                }\n              }')\" | jq -r '.candidates[0].content.parts[0].text')\n          fi\n\n          echo \"$RESPONSE\" > /tmp/review_result.json\n\n          # Validate JSON\n          if jq empty /tmp/review_result.json 2>/dev/null; then\n            echo \"valid_json=true\" >> $GITHUB_OUTPUT\n          else\n            echo \"valid_json=false\" >> $GITHUB_OUTPUT\n            echo \"Raw response was not valid JSON, attempting extraction...\"\n            # Try to extract JSON from markdown fenced blocks\n            sed -n '/^```json/,/^```$/p' /tmp/review_result.json | sed '1d;$d' > /tmp/review_cleaned.json\n            if jq empty /tmp/review_cleaned.json 2>/dev/null; then\n              mv /tmp/review_cleaned.json /tmp/review_result.json\n              echo \"valid_json=true\" >> $GITHUB_OUTPUT\n            fi\n          fi\n\n      - name: Parse and post review\n        if: success()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: |\n          REVIEW_FILE=\"/tmp/review_result.json\"\n\n          if ! jq empty \"$REVIEW_FILE\" 2>/dev/null; then\n            echo \"Failed to get valid JSON review. Posting raw output as comment.\"\n            gh pr comment ${{ github.event.pull_request.number }} \\\n              --body \"## Gemini Review\n\n          > Could not parse structured output. Raw response attached.\n\n          \\`\\`\\`\n          $(cat \"$REVIEW_FILE\")\n          \\`\\`\\`\"\n            exit 0\n          fi\n\n          # Extract fields from JSON\n          SUMMARY=$(jq -r '.summary' \"$REVIEW_FILE\")\n          RISK=$(jq -r '.risk_level' \"$REVIEW_FILE\")\n          APPROVAL=$(jq -r '.approval_recommendation' \"$REVIEW_FILE\")\n          POSITIVES=$(jq -r '.positive_aspects // [] | map(\"- \" + .) | join(\"\\n\")' \"$REVIEW_FILE\")\n          ISSUE_COUNT=$(jq -r '.issues | length' \"$REVIEW_FILE\")\n\n          # Build issue table\n          ISSUES_MD=\"\"\n          if [ \"$ISSUE_COUNT\" -gt 0 ]; then\n            ISSUES_MD=\"### Issues Found ($ISSUE_COUNT)\n\n          | Severity | File | Line | Message | Suggestion |\n          |----------|------|------|---------|------------|\n          \"\n            ISSUES_MD+=$(jq -r '.issues[] | \"| \\(.severity) | `\\(.file)` | \\(.line) | \\(.message) | \\(.suggestion) |\"' \"$REVIEW_FILE\")\n          else\n            ISSUES_MD=\"### No issues found\"\n          fi\n\n          # Build risk badge\n          case \"$RISK\" in\n            low)    RISK_BADGE=\"![Risk: Low](https://img.shields.io/badge/Risk-Low-green)\" ;;\n            medium) RISK_BADGE=\"![Risk: Medium](https://img.shields.io/badge/Risk-Medium-yellow)\" ;;\n            high)   RISK_BADGE=\"![Risk: High](https://img.shields.io/badge/Risk-High-red)\" ;;\n            *)      RISK_BADGE=\"Risk: $RISK\" ;;\n          esac\n\n          # Compose comment\n          COMMENT=$(cat <<EOF\n          ## Gemini Automated PR Review\n\n          $RISK_BADGE\n\n          ### Summary\n          $SUMMARY\n\n          $ISSUES_MD\n\n          ### Positive Aspects\n          $POSITIVES\n\n          ---\n          **Recommendation:** \\`$APPROVAL\\`\n          EOF\n          )\n\n          # Post comment to PR\n          gh pr comment ${{ github.event.pull_request.number }} --body \"$COMMENT\"\n```\n\n## Step 3: Direct API Approach with Structured JSON Output (Recommended)\n\nIf the Gemini CLI does not reliably produce JSON, use the Gemini REST API directly with `responseMimeType` set to `application/json`. This forces the model to return valid JSON. Here is a standalone workflow that uses `curl` and `jq`:\n\n```yaml\nname: Gemini PR Review (API Direct)\n\non:\n  pull_request:\n    types: [opened, synchronize, reopened]\n\npermissions:\n  contents: read\n  pull-requests: write\n\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout\n        uses: actions/checkout@v4\n        with:\n          fetch-depth: 0\n\n      - name: Generate review via Gemini API\n        id: review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD | head -c 30000)\n          FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)\n\n          # Build the prompt\n          PROMPT=\"You are a code reviewer. Review this PR diff. Changed files: ${FILES}. Diff: ${DIFF}. Return a JSON object with keys: summary (string), risk_level (low/medium/high), issues (array of {file, line, severity, message, suggestion}), positive_aspects (array of strings), approval_recommendation (approve/request_changes/comment).\"\n\n          # Call Gemini API with structured output\n          RESPONSE=$(curl -sf \\\n            \"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${GEMINI_API_KEY}\" \\\n            -H 'Content-Type: application/json' \\\n            -d \"$(jq -n --arg prompt \"$PROMPT\" '{\n              contents: [{parts: [{text: $prompt}]}],\n              generationConfig: {\n                responseMimeType: \"application/json\",\n                responseSchema: {\n                  type: \"object\",\n                  properties: {\n                    summary: {type: \"string\"},\n                    risk_level: {type: \"string\", enum: [\"low\", \"medium\", \"high\"]},\n                    issues: {\n                      type: \"array\",\n                      items: {\n                        type: \"object\",\n                        properties: {\n                          file: {type: \"string\"},\n                          line: {type: \"integer\"},\n                          severity: {type: \"string\", enum: [\"error\", \"warning\", \"suggestion\"]},\n                          message: {type: \"string\"},\n                          suggestion: {type: \"string\"}\n                        },\n                        required: [\"file\", \"severity\", \"message\"]\n                      }\n                    },\n                    positive_aspects: {type: \"array\", items: {type: \"string\"}},\n                    approval_recommendation: {type: \"string\", enum: [\"approve\", \"request_changes\", \"comment\"]}\n                  },\n                  required: [\"summary\", \"risk_level\", \"issues\", \"approval_recommendation\"]\n                },\n                temperature: 0.2\n              }\n            }')\")\n\n          # Extract the text content from the API response\n          REVIEW=$(echo \"$RESPONSE\" | jq -r '.candidates[0].content.parts[0].text')\n          echo \"$REVIEW\" > /tmp/review.json\n\n          # Validate\n          jq empty /tmp/review.json\n\n      - name: Post review comment\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: |\n          REVIEW_FILE=\"/tmp/review.json\"\n\n          SUMMARY=$(jq -r '.summary' \"$REVIEW_FILE\")\n          RISK=$(jq -r '.risk_level' \"$REVIEW_FILE\")\n          APPROVAL=$(jq -r '.approval_recommendation' \"$REVIEW_FILE\")\n\n          # Format issues as markdown\n          ISSUES=$(jq -r '\n            if (.issues | length) > 0 then\n              \"| Severity | File | Line | Issue | Fix |\\n|---|---|---|---|---|\\n\" +\n              (.issues[] | \"| \\(.severity) | `\\(.file)` | \\(.line // \"N/A\") | \\(.message) | \\(.suggestion // \"N/A\") |\")\n            else\n              \"_No issues found._\"\n            end\n          ' \"$REVIEW_FILE\")\n\n          POSITIVES=$(jq -r '(.positive_aspects // []) | map(\"- \" + .) | join(\"\\n\")' \"$REVIEW_FILE\")\n\n          BODY=$(cat <<EOF\n          ## Automated PR Review (Gemini)\n\n          **Risk Level:** \\`${RISK}\\` | **Recommendation:** \\`${APPROVAL}\\`\n\n          ### Summary\n          ${SUMMARY}\n\n          ### Issues\n          ${ISSUES}\n\n          ### What looks good\n          ${POSITIVES}\n          EOF\n          )\n\n          gh pr comment \"${{ github.event.pull_request.number }}\" --body \"$BODY\"\n```\n\n## Step 4: Using Gemini CLI with `--json` Flag (If Supported)\n\nSome versions of the Gemini CLI support a `--json` output flag. If your version does, the invocation simplifies to:\n\n```bash\ngemini --model gemini-2.0-flash \\\n  --api-key \"$GEMINI_API_KEY\" \\\n  --json \\\n  --prompt \"Review this code diff and return structured JSON: $DIFF\"\n```\n\nCheck your installed version's help output (`gemini --help`) to confirm available flags.\n\n## Step 5: Parsing the JSON Output in Downstream Steps\n\nOnce you have valid JSON in a file, you can use it in subsequent pipeline steps:\n\n```yaml\n      - name: Fail on high-risk reviews\n        run: |\n          RISK=$(jq -r '.risk_level' /tmp/review.json)\n          ERRORS=$(jq '[.issues[] | select(.severity == \"error\")] | length' /tmp/review.json)\n\n          echo \"Risk level: $RISK\"\n          echo \"Error count: $ERRORS\"\n\n          if [ \"$RISK\" = \"high\" ] || [ \"$ERRORS\" -gt 0 ]; then\n            echo \"::error::PR review found high risk or errors. Please address the issues.\"\n            exit 1\n          fi\n```\n\nYou can also use it in a matrix strategy or pass data between jobs:\n\n```yaml\n      - name: Export review data\n        id: export\n        run: |\n          echo \"risk=$(jq -r '.risk_level' /tmp/review.json)\" >> $GITHUB_OUTPUT\n          echo \"approval=$(jq -r '.approval_recommendation' /tmp/review.json)\" >> $GITHUB_OUTPUT\n          echo \"issue_count=$(jq '.issues | length' /tmp/review.json)\" >> $GITHUB_OUTPUT\n```\n\nThen reference in later steps or jobs:\n\n```yaml\n      - name: Auto-approve if clean\n        if: steps.export.outputs.approval == 'approve' && steps.export.outputs.issue_count == '0'\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: |\n          gh pr review \"${{ github.event.pull_request.number }}\" --approve --body \"Automated review passed with no issues.\"\n```\n\n## Key Points\n\n1. **`responseMimeType: \"application/json\"`** in the Gemini API `generationConfig` forces the model to return valid JSON. This is the most reliable way to get structured output.\n\n2. **`responseSchema`** lets you define the exact JSON schema you expect. The model will conform to it. This eliminates the need for post-processing or regex extraction.\n\n3. **Diff size limits**: Git diffs can be very large. Truncate them (e.g., `head -c 30000`) to stay within token limits. For large PRs, consider reviewing changed files individually.\n\n4. **Error handling**: Always validate the JSON output with `jq empty` before trying to parse it. Have a fallback path that posts raw output as a comment.\n\n5. **Rate limits**: The Gemini API has rate limits. For high-traffic repos, add retry logic or use the paid tier.\n\n6. **Security**: The `GEMINI_API_KEY` secret is never exposed in logs because GitHub Actions automatically masks secrets in output. The `GITHUB_TOKEN` is provided automatically by Actions with the permissions declared in the workflow.\n\n## Troubleshooting\n\n- **\"jq: parse error\"**: The model returned non-JSON text. Use `responseMimeType: \"application/json\"` to fix this.\n- **Empty response**: Check that `GEMINI_API_KEY` is set correctly in secrets and that the model name is valid.\n- **Diff too large**: Truncate the diff or split the review across multiple API calls, one per file.\n- **Permission denied on PR comment**: Ensure `permissions: pull-requests: write` is set in the workflow.\n"}], "eval-2-mcp-server-config-with_skill": [{"name": "response.md", "type": "text", "content": "# Connecting a Custom Python MCP Server to Gemini CLI\n\nYou need to configure your server in `settings.json`, pass the API key via the `env` field, and use `includeTools` to restrict which tools are exposed. Here is the full walkthrough.\n\n---\n\n## 1. Configure the MCP Server in settings.json\n\nEdit (or create) `~/.gemini/settings.json` for a global configuration, or `<your-project>/.gemini/settings.json` for project-scoped configuration. Add your server under the `mcpServers` key:\n\n```json\n{\n  \"mcpServers\": {\n    \"myPythonServer\": {\n      \"command\": \"python\",\n      \"args\": [\"-m\", \"my_mcp_server\"],\n      \"cwd\": \"./mcp_tools\",\n      \"env\": {\n        \"MY_API_KEY\": \"$MY_API_KEY\"\n      },\n      \"timeout\": 15000,\n      \"trust\": false,\n      \"includeTools\": [\"search_records\", \"get_status\"]\n    }\n  }\n}\n```\n\n### What each field does\n\n| Field | Type | Description |\n|---|---|---|\n| `command` | string | The executable to run. Use `\"python\"` (or `\"python3\"` depending on your system). |\n| `args` | string[] | Arguments passed to the command. `[\"-m\", \"my_mcp_server\"]` runs your package as a module. You can also use a script path like `[\"./server.py\"]`. |\n| `cwd` | string | Working directory for the server process. Optional -- set this if your server needs to run from a specific directory. |\n| `env` | object | Environment variables passed to the server process. Use the `\"$VAR_NAME\"` syntax to reference variables from your shell environment. |\n| `timeout` | number | Connection timeout in milliseconds. Default is 10000 (10 seconds). Increase if your server takes longer to start. |\n| `trust` | boolean | When `true`, skips all tool confirmation dialogs. Keep this `false` unless you fully control and trust the server. |\n| `includeTools` | string[] | **Whitelist** -- only these tools will be exposed to Gemini. If set, all other tools from the server are hidden. |\n| `excludeTools` | string[] | **Blacklist** -- these specific tools will be hidden. All others remain available. |\n\n**Important:** `includeTools` and `excludeTools` are mutually exclusive approaches. Use one or the other:\n- Use `includeTools` when your server exposes many tools but you only want a few available.\n- Use `excludeTools` when you want most tools but need to block specific dangerous ones.\n\n---\n\n## 2. Pass the API Key Securely\n\nNever hardcode secrets directly in `settings.json`. Instead, use the `\"$VAR_NAME\"` pattern to reference environment variables from your shell:\n\n```json\n\"env\": {\n  \"MY_API_KEY\": \"$MY_API_KEY\"\n}\n```\n\nThen set the variable in your shell before running Gemini:\n\n```bash\nexport MY_API_KEY=\"sk-your-actual-api-key-here\"\ngemini\n```\n\nAlternatively, you can store the variable in a `.env` file that Gemini loads automatically:\n\n- `~/.gemini/.env` -- global env file\n- `./.gemini/.env` -- project-level env file\n\nExample `.gemini/.env`:\n\n```\nMY_API_KEY=sk-your-actual-api-key-here\n```\n\n**Security note:** Gemini CLI automatically redacts sensitive env vars (names containing `TOKEN`, `SECRET`, `KEY`, etc.) from MCP processes unless you explicitly list them in the `env` field. So you must include the variable in `env` for it to reach your server.\n\n---\n\n## 3. Full Working Example\n\nSuppose you have a Python MCP server at `~/mcp-servers/analytics_server.py` that exposes five tools (`query_data`, `generate_report`, `delete_records`, `export_csv`, `run_migration`), and you only want `query_data` and `generate_report` available. The server needs a `ANALYTICS_API_KEY`.\n\n### settings.json\n\n```json\n{\n  \"mcpServers\": {\n    \"analytics\": {\n      \"command\": \"python3\",\n      \"args\": [\"analytics_server.py\"],\n      \"cwd\": \"/home/user/mcp-servers\",\n      \"env\": {\n        \"ANALYTICS_API_KEY\": \"$ANALYTICS_API_KEY\"\n      },\n      \"timeout\": 15000,\n      \"trust\": false,\n      \"includeTools\": [\"query_data\", \"generate_report\"]\n    }\n  }\n}\n```\n\n### Shell setup\n\n```bash\nexport ANALYTICS_API_KEY=\"your-key-here\"\ngemini\n```\n\n### Using the server in a Gemini session\n\nOnce inside the interactive session, you can verify the server connected:\n\n```\n/mcp list\n/mcp status\n```\n\nThen invoke the tools by addressing the server:\n\n```\n> @analytics Query monthly active users for Q4 2025\n> @analytics Generate a report on user retention trends\n```\n\n---\n\n## 4. Alternative: Use excludeTools Instead\n\nIf you want all tools available except specific dangerous ones:\n\n```json\n{\n  \"mcpServers\": {\n    \"analytics\": {\n      \"command\": \"python3\",\n      \"args\": [\"analytics_server.py\"],\n      \"cwd\": \"/home/user/mcp-servers\",\n      \"env\": {\n        \"ANALYTICS_API_KEY\": \"$ANALYTICS_API_KEY\"\n      },\n      \"timeout\": 15000,\n      \"trust\": false,\n      \"excludeTools\": [\"delete_records\", \"run_migration\"]\n    }\n  }\n}\n```\n\nThis exposes everything except `delete_records` and `run_migration`.\n\n---\n\n## 5. Adding the Server via CLI (Alternative to Manual Editing)\n\nYou can also add MCP servers from the command line:\n\n```bash\ngemini mcp add analytics --command \"python3 analytics_server.py\"\n```\n\nHowever, the CLI `add` command provides less control over fields like `env`, `includeTools`, and `cwd`. For full configuration, editing `settings.json` directly is recommended.\n\nTo manage servers later:\n\n```bash\ngemini mcp list           # list all configured servers\ngemini mcp remove analytics  # remove a server\n```\n\nOr from within an interactive session:\n\n```\n/mcp list\n/mcp enable analytics\n/mcp disable analytics\n```\n\n---\n\n## 6. Minimal Python MCP Server Skeleton\n\nFor reference, here is the basic structure of a Python MCP server that your configuration would launch:\n\n```python\n# analytics_server.py\nimport os\nimport json\nimport sys\n\n# Read the API key passed through the env config\napi_key = os.environ.get(\"ANALYTICS_API_KEY\")\nif not api_key:\n    print(\"Error: ANALYTICS_API_KEY not set\", file=sys.stderr)\n    sys.exit(1)\n\n# Your MCP server implementation here\n# The server communicates over stdio using the MCP protocol\n# Tools like query_data, generate_report, etc. are registered\n# and handled according to the MCP specification.\n```\n\n---\n\n## Summary\n\n1. Add your server to `mcpServers` in `~/.gemini/settings.json` (or project-level).\n2. Pass the API key using `\"env\": { \"MY_API_KEY\": \"$MY_API_KEY\" }` -- never hardcode it.\n3. Restrict tools with `\"includeTools\": [\"tool1\", \"tool2\"]` (whitelist) or `\"excludeTools\": [\"bad_tool\"]` (blacklist).\n4. Set the actual environment variable in your shell or in a `.gemini/.env` file.\n5. Verify the connection with `/mcp list` and `/mcp status` inside an interactive session.\n"}], "eval-2-mcp-server-config-without_skill": [{"name": "response.md", "type": "text", "content": "# Connecting a Custom Python MCP Server to Gemini CLI\n\n## Overview\n\nGemini CLI supports MCP (Model Context Protocol) servers through its settings configuration. You can connect a custom Python MCP server, pass environment variables (like API keys), and restrict which tools are exposed.\n\n## 1. Build Your Python MCP Server\n\nFirst, make sure your Python MCP server uses the official MCP SDK. Here is a minimal example:\n\n```python\n# my_mcp_server.py\nimport os\nfrom mcp.server.fastmcp import FastMCP\n\n# Read the API key from environment\nAPI_KEY = os.environ.get(\"MY_API_KEY\")\nif not API_KEY:\n    raise ValueError(\"MY_API_KEY environment variable is required\")\n\nmcp = FastMCP(\"my-custom-server\")\n\n@mcp.tool()\ndef fetch_data(query: str) -> str:\n    \"\"\"Fetch data from the external API using the configured API key.\"\"\"\n    # Use API_KEY here for your requests\n    return f\"Results for '{query}' using key ending in ...{API_KEY[-4:]}\"\n\n@mcp.tool()\ndef analyze_data(data: str) -> str:\n    \"\"\"Analyze the provided data.\"\"\"\n    return f\"Analysis of: {data}\"\n\n@mcp.tool()\ndef internal_debug(info: str) -> str:\n    \"\"\"Internal debug tool - you may want to hide this from Gemini.\"\"\"\n    return f\"Debug: {info}\"\n\nif __name__ == \"__main__\":\n    mcp.run()\n```\n\nInstall the dependency:\n\n```bash\npip install mcp\n```\n\n## 2. Configure Gemini CLI to Use Your MCP Server\n\nGemini CLI reads MCP server configuration from a `settings.json` file. There are two levels:\n\n- **Global (user-level):** `~/.gemini/settings.json`\n- **Project-level:** `.gemini/settings.json` in your project root\n\n### Basic Configuration\n\nCreate or edit `~/.gemini/settings.json` (or `.gemini/settings.json` in your project):\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"cwd\": \"/absolute/path/to/your/server/directory\",\n      \"env\": {\n        \"MY_API_KEY\": \"your-api-key-value-here\"\n      }\n    }\n  }\n}\n```\n\n### Configuration Fields Explained\n\n| Field | Description |\n|-------|-------------|\n| `command` | The executable to run (e.g., `python`, `python3`, `node`, `npx`) |\n| `args` | Array of arguments passed to the command |\n| `cwd` | Working directory for the server process (optional but recommended) |\n| `env` | Environment variables passed to the server process |\n\n### Using a Virtual Environment\n\nIf your MCP server has its own virtual environment:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"/path/to/venv/bin/python\",\n      \"args\": [\"/absolute/path/to/my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"your-api-key-value-here\"\n      }\n    }\n  }\n}\n```\n\nOn Windows, use the Windows path to the venv Python:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"C:\\\\path\\\\to\\\\venv\\\\Scripts\\\\python.exe\",\n      \"args\": [\"C:\\\\path\\\\to\\\\my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"your-api-key-value-here\"\n      }\n    }\n  }\n}\n```\n\n## 3. Passing the API Key Securely\n\nThere are several approaches to handle the API key:\n\n### Option A: Inline in settings.json (simplest, but key is in plaintext)\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"sk-abc123your-actual-key\"\n      }\n    }\n  }\n}\n```\n\n### Option B: Reference a shell variable at launch time\n\nSet the environment variable in your shell profile (`.bashrc`, `.zshrc`, etc.):\n\n```bash\nexport MY_API_KEY=\"sk-abc123your-actual-key\"\n```\n\nThen in `settings.json`, you can omit the `env` block entirely if the variable is already in your shell environment. The child process inherits the parent environment. However, to be explicit:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"${MY_API_KEY}\"\n      }\n    }\n  }\n}\n```\n\n> **Note:** Whether `${MY_API_KEY}` variable interpolation is supported depends on the Gemini CLI version. If it is not supported, you may need to use a wrapper script approach (see Option C).\n\n### Option C: Wrapper script (most flexible)\n\nCreate a wrapper script `run_server.sh`:\n\n```bash\n#!/bin/bash\n# run_server.sh\n# Load API key from a secure location\nsource ~/.secrets/my_api_keys.env\nexport MY_API_KEY\nexec python /path/to/my_mcp_server.py \"$@\"\n```\n\nThen configure:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"bash\",\n      \"args\": [\"/path/to/run_server.sh\"]\n    }\n  }\n}\n```\n\n## 4. Restricting Which Tools Are Exposed\n\nThere are two approaches to restricting tools: server-side and client-side.\n\n### Approach A: Server-Side Restriction (Recommended)\n\nOnly define the tools you want exposed in your MCP server. Simply remove or comment out any tools you do not want Gemini to access:\n\n```python\n# my_mcp_server.py - only expose the tools you want\nfrom mcp.server.fastmcp import FastMCP\nimport os\n\nAPI_KEY = os.environ.get(\"MY_API_KEY\")\nmcp = FastMCP(\"my-custom-server\")\n\n@mcp.tool()\ndef fetch_data(query: str) -> str:\n    \"\"\"Fetch data from the external API.\"\"\"\n    return f\"Results for '{query}'\"\n\n@mcp.tool()\ndef analyze_data(data: str) -> str:\n    \"\"\"Analyze the provided data.\"\"\"\n    return f\"Analysis of: {data}\"\n\n# internal_debug is NOT registered - Gemini will never see it\n\nif __name__ == \"__main__\":\n    mcp.run()\n```\n\n### Approach B: Conditional Tool Registration\n\nUse an environment variable or config file to control which tools are registered:\n\n```python\n# my_mcp_server.py - conditionally register tools\nfrom mcp.server.fastmcp import FastMCP\nimport os\nimport json\n\nAPI_KEY = os.environ.get(\"MY_API_KEY\")\nALLOWED_TOOLS = os.environ.get(\"ALLOWED_TOOLS\", \"\").split(\",\")\n\nmcp = FastMCP(\"my-custom-server\")\n\ndef register_if_allowed(tool_name):\n    \"\"\"Decorator factory that only registers a tool if it's in the allowed list.\"\"\"\n    def decorator(func):\n        if not ALLOWED_TOOLS or not ALLOWED_TOOLS[0]:\n            # No restriction set, register all tools\n            return mcp.tool()(func)\n        elif tool_name in ALLOWED_TOOLS:\n            return mcp.tool()(func)\n        return func\n    return decorator\n\n@register_if_allowed(\"fetch_data\")\ndef fetch_data(query: str) -> str:\n    \"\"\"Fetch data from the external API.\"\"\"\n    return f\"Results for '{query}'\"\n\n@register_if_allowed(\"analyze_data\")\ndef analyze_data(data: str) -> str:\n    \"\"\"Analyze the provided data.\"\"\"\n    return f\"Analysis of: {data}\"\n\n@register_if_allowed(\"internal_debug\")\ndef internal_debug(info: str) -> str:\n    \"\"\"Internal debug tool.\"\"\"\n    return f\"Debug: {info}\"\n\nif __name__ == \"__main__\":\n    mcp.run()\n```\n\nThen in your `settings.json`, pass the allowed tools list:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"your-api-key-here\",\n        \"ALLOWED_TOOLS\": \"fetch_data,analyze_data\"\n      }\n    }\n  }\n}\n```\n\nThis way, `internal_debug` is never registered and Gemini cannot call it.\n\n### Approach C: Gemini CLI Tool Filtering (if supported)\n\nSome MCP clients support an `allowedTools` or `toolFilter` field in the server configuration. If Gemini CLI supports this, the configuration would look like:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"your-api-key-here\"\n      },\n      \"toolFilter\": {\n        \"allow\": [\"fetch_data\", \"analyze_data\"]\n      }\n    }\n  }\n}\n```\n\n> **Note:** The exact field name and syntax for client-side tool filtering varies by MCP client implementation. Check the Gemini CLI documentation for the exact syntax. If Gemini CLI does not support this field, use the server-side approaches (A or B) described above.\n\n## 5. Complete Working Example\n\n### Directory Structure\n\n```\nmy-mcp-project/\n  my_mcp_server.py\n  requirements.txt\n  .gemini/\n    settings.json\n```\n\n### requirements.txt\n\n```\nmcp>=1.0.0\n```\n\n### my_mcp_server.py\n\n```python\nimport os\nfrom mcp.server.fastmcp import FastMCP\n\nAPI_KEY = os.environ.get(\"MY_API_KEY\")\nif not API_KEY:\n    raise ValueError(\"MY_API_KEY environment variable must be set\")\n\nALLOWED_TOOLS = [t.strip() for t in os.environ.get(\"ALLOWED_TOOLS\", \"\").split(\",\") if t.strip()]\n\nmcp = FastMCP(\"my-custom-server\")\n\n\ndef expose(tool_name):\n    \"\"\"Only register the tool if it passes the allowlist filter.\"\"\"\n    def decorator(func):\n        if not ALLOWED_TOOLS or tool_name in ALLOWED_TOOLS:\n            return mcp.tool()(func)\n        return func\n    return decorator\n\n\n@expose(\"fetch_data\")\ndef fetch_data(query: str) -> str:\n    \"\"\"Fetch data from the external API using the configured API key.\"\"\"\n    # Your real implementation here\n    return f\"Fetched results for '{query}'\"\n\n\n@expose(\"analyze_data\")\ndef analyze_data(data: str) -> str:\n    \"\"\"Analyze the provided data and return insights.\"\"\"\n    return f\"Analysis complete for: {data}\"\n\n\n@expose(\"dangerous_reset\")\ndef dangerous_reset() -> str:\n    \"\"\"Reset all data - restricted tool.\"\"\"\n    return \"Data reset complete\"\n\n\nif __name__ == \"__main__\":\n    mcp.run()\n```\n\n### .gemini/settings.json\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"cwd\": \"/absolute/path/to/my-mcp-project\",\n      \"env\": {\n        \"MY_API_KEY\": \"sk-your-api-key-here\",\n        \"ALLOWED_TOOLS\": \"fetch_data,analyze_data\"\n      }\n    }\n  }\n}\n```\n\nWith this configuration:\n- Gemini CLI launches your Python MCP server as a subprocess\n- The `MY_API_KEY` environment variable is passed securely to the server process\n- Only `fetch_data` and `analyze_data` are registered as tools; `dangerous_reset` is excluded\n- Gemini can discover and invoke the allowed tools during conversation\n\n## 6. Testing Your Server\n\nBefore connecting to Gemini CLI, test your server standalone:\n\n```bash\n# Test that the server starts correctly\nMY_API_KEY=\"test-key\" python my_mcp_server.py\n\n# Or with tool restrictions\nMY_API_KEY=\"test-key\" ALLOWED_TOOLS=\"fetch_data\" python my_mcp_server.py\n```\n\nYou can also use the MCP Inspector tool to verify your server works:\n\n```bash\nnpx @modelcontextprotocol/inspector python my_mcp_server.py\n```\n\nThen launch Gemini CLI in your project directory. It will automatically pick up the `.gemini/settings.json` and start your MCP server.\n\n## 7. Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Server not found | Use absolute paths in `command` and `args` |\n| API key not available | Verify the `env` block in settings.json; check with a print/log statement |\n| Tools not showing up | Confirm the tool functions are decorated with `@mcp.tool()` and not filtered out |\n| Permission errors | Ensure the Python executable and script have correct permissions |\n| Server crashes on start | Run the server manually from the terminal first to see error output |\n| Wrong Python version | Use the full path to the correct Python interpreter in `command` |\n"}], "eval-3-custom-slash-command-with_skill": [{"name": "response.md", "type": "text", "content": "# Creating a Global Custom Slash Command for AI Commit Messages in Gemini CLI\n\nYou can create a custom slash command by writing a `.toml` file in the global commands directory. Since you want it available in all your projects, place it under `~/.gemini/commands/`.\n\n## Step 1: Create the Commands Directory\n\n```bash\nmkdir -p ~/.gemini/commands\n```\n\n## Step 2: Create the Slash Command TOML File\n\nCreate the file `~/.gemini/commands/commit.toml`:\n\n```toml\ndescription = \"Generate a commit message from staged changes\"\nprompt = \"\"\"\nWrite a concise conventional commit message for the following staged changes.\n\nUse the format: <type>(<scope>): <description>\n\nWhere type is one of: feat, fix, docs, style, refactor, test, chore\n\nOnly output the commit message itself, nothing else.\n\nStaged diff:\n!{git diff --cached}\n\"\"\"\n```\n\n### How This Works\n\n- The `prompt` field defines the instruction sent to Gemini when the command is invoked.\n- `!{git diff --cached}` is a **shell execution block** -- Gemini CLI runs the shell command inside `!{...}` and inlines its output into the prompt before sending it to the model.\n- `description` is optional but shows up when you run `/help` in an interactive session.\n\n## Step 3: Use the Command\n\nStart Gemini CLI in any project that has staged git changes:\n\n```bash\ncd your-project\ngit add -A   # or stage specific files\ngemini\n```\n\nThen in the interactive session, type:\n\n```\n/commit\n```\n\nGemini will execute `git diff --cached`, include the diff in the prompt, and return a conventional commit message.\n\n## Namespaced Alternative\n\nIf you prefer a namespaced command like `/git:commit`, use a subdirectory:\n\n```bash\nmkdir -p ~/.gemini/commands/git\n```\n\nCreate `~/.gemini/commands/git/commit.toml`:\n\n```toml\ndescription = \"Generate a commit message from staged changes\"\nprompt = \"\"\"\nWrite a concise conventional commit message for the following staged changes.\n\nUse the format: <type>(<scope>): <description>\n\nWhere type is one of: feat, fix, docs, style, refactor, test, chore\n\nOnly output the commit message itself, nothing else.\n\nStaged diff:\n!{git diff --cached}\n\"\"\"\n```\n\nThen invoke it with:\n\n```\n/git:commit\n```\n\n## Using It with Arguments\n\nYou can also make the command accept arguments with `{{args}}` to allow the user to pass additional context:\n\n```toml\ndescription = \"Generate a commit message from staged changes with optional context\"\nprompt = \"\"\"\nWrite a concise conventional commit message for the following staged changes.\n\nUse the format: <type>(<scope>): <description>\n\nWhere type is one of: feat, fix, docs, style, refactor, test, chore\n\nAdditional context from the user: {{args}}\n\nOnly output the commit message itself, nothing else.\n\nStaged diff:\n!{git diff --cached}\n\"\"\"\n```\n\nThis lets you provide extra hints:\n\n```\n/commit focus on the auth refactoring\n```\n\n## Fully Automated Shell Script (Bonus)\n\nIf you want to go beyond interactive mode and automate the entire commit flow from your shell, you can create a shell script. Note that custom slash commands currently do **not** work in headless mode (`-p` flag), so for automation you need to embed the prompt directly:\n\n```bash\n#!/bin/bash\n# Save as ~/bin/ai-commit or anywhere on your PATH\n\nDIFF=$(git diff --cached)\n\nif [ -z \"$DIFF\" ]; then\n  echo \"Error: Nothing staged. Run 'git add' first.\"\n  exit 1\nfi\n\nMSG=$(echo \"$DIFF\" | gemini -p \"Write a concise conventional commit message (type(scope): description format, max 72 chars). Only output the message, nothing else.\" --output-format json | jq -r '.response')\n\nif [ -z \"$MSG\" ]; then\n  echo \"Error: Failed to generate commit message.\"\n  exit 1\nfi\n\necho \"Proposed commit message:\"\necho \"  $MSG\"\necho \"\"\nread -p \"Commit with this message? [y/N] \" confirm\n\nif [[ \"$confirm\" =~ ^[Yy]$ ]]; then\n  git commit -m \"$MSG\"\n  echo \"Committed.\"\nelse\n  echo \"Aborted.\"\nfi\n```\n\nMake it executable:\n\n```bash\nchmod +x ~/bin/ai-commit\n```\n\nThen use it from any project:\n\n```bash\ngit add -A\nai-commit\n```\n\n## Summary of File Locations\n\n| Path | Scope | Invocation |\n|---|---|---|\n| `~/.gemini/commands/commit.toml` | Global (all projects) | `/commit` |\n| `~/.gemini/commands/git/commit.toml` | Global, namespaced | `/git:commit` |\n| `<project>/.gemini/commands/commit.toml` | Project only | `/commit` |\n\nThe global location (`~/.gemini/commands/`) is what makes the command available across all your projects without any per-project setup.\n"}], "eval-3-custom-slash-command-without_skill": [{"name": "response.md", "type": "text", "content": "# Creating a Custom Slash Command in Gemini CLI for Commit Messages\n\n## Overview\n\nGemini CLI supports custom slash commands through its extensions system. To make a command available across all your projects, you place it in the global configuration directory at `~/.gemini/extensions/`. Extensions are defined using `gemini_extension.json` manifest files that describe the command and its behavior.\n\n## Step-by-Step Setup\n\n### 1. Create the Global Extensions Directory\n\n```bash\nmkdir -p ~/.gemini/extensions/commit-message\n```\n\n### 2. Create the Shell Script\n\nCreate `~/.gemini/extensions/commit-message/generate-commit-msg.sh`:\n\n```bash\n#!/bin/bash\n# generate-commit-msg.sh\n# Gets the staged diff and outputs it for the AI to process\n\nDIFF=$(git diff --cached)\n\nif [ -z \"$DIFF\" ]; then\n  echo \"ERROR: No staged changes found. Stage your changes with 'git add' first.\"\n  exit 1\nfi\n\necho \"Here is the staged git diff:\"\necho \"\"\necho '```diff'\necho \"$DIFF\"\necho '```'\necho \"\"\necho \"Please write a concise, conventional commit message for these changes.\"\necho \"Follow the Conventional Commits format: type(scope): description\"\necho \"\"\necho \"Types: feat, fix, docs, style, refactor, perf, test, build, ci, chore\"\necho \"\"\necho \"Provide:\"\necho \"1. A single-line subject (max 72 chars)\"\necho \"2. A blank line\"\necho \"3. An optional body explaining the 'why' behind the changes\"\n```\n\nMake it executable:\n\n```bash\nchmod +x ~/.gemini/extensions/commit-message/generate-commit-msg.sh\n```\n\n### 3. Create the Extension Manifest\n\nCreate `~/.gemini/extensions/commit-message/gemini_extension.json`:\n\n```json\n{\n  \"name\": \"commit-message\",\n  \"description\": \"Generate a commit message from staged changes using git diff --cached\",\n  \"version\": \"1.0.0\",\n  \"commands\": [\n    {\n      \"name\": \"commit-msg\",\n      \"description\": \"Analyze staged git changes and generate a conventional commit message\",\n      \"action\": {\n        \"type\": \"shell\",\n        \"command\": \"~/.gemini/extensions/commit-message/generate-commit-msg.sh\"\n      }\n    }\n  ]\n}\n```\n\n### 4. Usage\n\nOnce configured, open Gemini CLI in any project and type:\n\n```\n/commit-msg\n```\n\nThe command will:\n1. Run `git diff --cached` to capture your staged changes\n2. Pass the diff output into the conversation context\n3. Ask Gemini to generate a conventional commit message based on the diff\n\n## Alternative Approach: Using GEMINI.md with a Prompt-Based Workflow\n\nIf extensions are not behaving as expected in your version of Gemini CLI, you can achieve a similar result using a global `GEMINI.md` instruction file combined with a simple shell alias.\n\n### 1. Create a Global GEMINI.md\n\nCreate or edit `~/.gemini/GEMINI.md`:\n\n```markdown\n# Custom Commands\n\n## Commit Message Generation\n\nWhen I say \"generate commit message\" or provide a git diff, follow these rules:\n\n1. Analyze the diff carefully\n2. Write a commit message in Conventional Commits format\n3. Use the format: type(scope): short description\n4. Include a body if the changes are complex\n5. Keep the subject line under 72 characters\n6. Use imperative mood (\"add feature\" not \"added feature\")\n```\n\n### 2. Create a Shell Helper Script\n\nAdd to your `~/.bashrc` or `~/.zshrc`:\n\n```bash\n# Gemini CLI commit message helper\ngcm() {\n  local diff\n  diff=$(git diff --cached)\n\n  if [ -z \"$diff\" ]; then\n    echo \"No staged changes. Use 'git add' to stage files first.\"\n    return 1\n  fi\n\n  echo \"Staged diff:\"\n  echo '```diff'\n  echo \"$diff\"\n  echo '```'\n  echo \"\"\n  echo \"Generate a conventional commit message for the above changes.\"\n}\n```\n\nThen in Gemini CLI, you can run:\n\n```\n@run gcm\n```\n\nOr pipe the output directly:\n\n```bash\n# From your regular terminal, pipe into gemini\ngcm | gemini\n```\n\n## Alternative Approach: Inline Tool Use\n\nGemini CLI allows you to invoke shell commands inline. You can simply type within a Gemini CLI session:\n\n```\nPlease run `git diff --cached` and write me a commit message based on the output.\n```\n\nSince Gemini CLI has the ability to execute shell commands (with your approval), it will run the diff, read the output, and generate a commit message. This requires no configuration at all, but it is a manual process each time rather than a single slash command.\n\n## Applying the Generated Commit Message\n\nOnce Gemini generates a commit message you like, you can apply it directly:\n\n```bash\ngit commit -m \"feat(auth): add OAuth2 token refresh logic\n\nImplement automatic token refresh when access tokens expire.\nUses refresh token rotation for improved security.\"\n```\n\nOr ask Gemini CLI to run the commit for you:\n\n```\nPlease commit with that message.\n```\n\n## Directory Structure Summary\n\n```\n~/.gemini/\n  GEMINI.md                          # Global instructions (optional)\n  extensions/\n    commit-message/\n      gemini_extension.json          # Extension manifest\n      generate-commit-msg.sh         # Shell script for the command\n```\n\n## Tips\n\n- **Scope control**: Placing extensions in `~/.gemini/extensions/` makes them global. For project-specific commands, place them in `.gemini/extensions/` within the project root.\n- **Customizing the prompt**: Edit the echo statements at the end of `generate-commit-msg.sh` to adjust the instructions given to the AI (e.g., require a specific commit style, add ticket number references, etc.).\n- **Large diffs**: If the staged diff is very large, consider truncating it or summarizing file-level changes to stay within context limits. You can modify the script to include `--stat` output alongside a truncated diff.\n- **Security**: The shell script only reads data (via `git diff --cached`). It does not modify your repository or make commits on its own.\n\n## Troubleshooting\n\n- **\"Command not found\"**: Make sure the shell script is executable (`chmod +x`) and the path in `gemini_extension.json` is correct.\n- **\"No staged changes\"**: Run `git add <files>` before invoking the command.\n- **Extension not loading**: Verify the JSON manifest is valid (`cat ~/.gemini/extensions/commit-message/gemini_extension.json | python3 -m json.tool`) and restart Gemini CLI.\n- **Path issues on Windows**: Use forward slashes or the `%USERPROFILE%` equivalent in paths. The extensions directory would be at `%USERPROFILE%\\.gemini\\extensions\\`.\n"}]}, "benchmark": {"metadata": {"skill_name": "gemini-cli", "skill_path": "src/skills/gemini-cli/", "executor_model": "claude-opus-4-6", "analyzer_model": "claude-opus-4-6", "timestamp": "2026-03-11T18:15:00Z", "evals_run": [1, 2, 3], "runs_per_configuration": 3}, "runs": [{"eval_id": 1, "eval_name": "cicd-setup", "configuration": "with_skill", "run_number": 1, "result": {"pass_rate": 1.0, "passed": 6, "failed": 0, "total": 6, "time_seconds": 53.6, "tokens": 64216}, "expectations": [{"text": "Uses -p or --prompt flag for headless/non-interactive mode", "passed": true, "evidence": "Line 53: gemini -p with --output-format json"}, {"text": "Includes --output-format json flag for structured output", "passed": true, "evidence": "Line 62: --output-format json flag used correctly"}, {"text": "Shows how to set GEMINI_API_KEY from GitHub secrets", "passed": true, "evidence": "Line 42: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}"}, {"text": "Provides a GitHub Actions workflow YAML snippet", "passed": true, "evidence": "Complete workflow YAML with checkout, node setup, install, review, and comment steps"}, {"text": "Shows how to parse JSON response (e.g., with jq .response)", "passed": true, "evidence": "Line 65: jq -r '.response' and line 68: token stats extraction"}, {"text": "Does not invent non-existent CLI flags or options", "passed": true, "evidence": "Only uses verified flags: -p and --output-format json. Exit codes match corrected values."}], "notes": []}, {"eval_id": 1, "eval_name": "cicd-setup", "configuration": "without_skill", "run_number": 1, "result": {"pass_rate": 0.6667, "passed": 4, "failed": 2, "total": 6, "time_seconds": 68.0, "tokens": 59322}, "expectations": [{"text": "Uses -p or --prompt flag for headless/non-interactive mode", "passed": true, "evidence": "Line 96: gemini -p"}, {"text": "Includes --output-format json flag for structured output", "passed": false, "evidence": "Does not use --output-format json. Line 95 comment mentions '--json flag if available' which is not a real flag."}, {"text": "Shows how to set GEMINI_API_KEY from GitHub secrets", "passed": true, "evidence": "Line 62: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}"}, {"text": "Provides a GitHub Actions workflow YAML snippet", "passed": true, "evidence": "Complete workflow YAML provided"}, {"text": "Shows how to parse JSON response (e.g., with jq .response)", "passed": true, "evidence": "Uses jq for JSON validation and parsing"}, {"text": "Does not invent non-existent CLI flags or options", "passed": false, "evidence": "Line 95: references '--json flag if available' which is not a real Gemini CLI flag"}], "notes": []}, {"eval_id": 2, "eval_name": "mcp-server-config", "configuration": "with_skill", "run_number": 1, "result": {"pass_rate": 1.0, "passed": 6, "failed": 0, "total": 6, "time_seconds": 47.3, "tokens": 66522}, "expectations": [{"text": "Points to ~/.gemini/settings.json or project-level settings.json", "passed": true, "evidence": "Opens with ~/.gemini/settings.json (global) or <project>/.gemini/settings.json (project-scoped)"}, {"text": "Shows mcpServers config with command, args fields", "passed": true, "evidence": "Full JSON config with mcpServers, command, args, env, timeout, trust, includeTools"}, {"text": "Shows $VAR pattern for environment variable references in env field", "passed": true, "evidence": "Shows \"API_KEY\": \"$MY_API_KEY\" pattern"}, {"text": "Explains includeTools and/or excludeTools for restricting exposed tools", "passed": true, "evidence": "Dedicated section for both includeTools and excludeTools with examples"}, {"text": "Mentions security considerations (trust field, not hardcoding keys, etc.)", "passed": true, "evidence": "Explains trust field, warns never to hardcode secrets, mentions env var redaction"}, {"text": "Uses correct Python command (python/python3) in the command field", "passed": true, "evidence": "Uses \"command\": \"python\" consistently"}], "notes": []}, {"eval_id": 2, "eval_name": "mcp-server-config", "configuration": "without_skill", "run_number": 1, "result": {"pass_rate": 1.0, "passed": 6, "failed": 0, "total": 6, "time_seconds": 86.7, "tokens": 61417}, "expectations": [{"text": "Points to ~/.gemini/settings.json or project-level settings.json", "passed": true, "evidence": "States ~/.gemini/settings.json (global) and .gemini/settings.json (project-level)"}, {"text": "Shows mcpServers config with command, args fields", "passed": true, "evidence": "Full JSON config with mcpServers, command, args, env"}, {"text": "Shows $VAR pattern for environment variable references in env field", "passed": true, "evidence": "Shows both $MY_API_KEY and ${ANOTHER_ENV_VAR} syntax"}, {"text": "Explains includeTools and/or excludeTools for restricting exposed tools", "passed": true, "evidence": "Dedicated sections for includeTools and excludeTools with examples"}, {"text": "Mentions security considerations (trust field, not hardcoding keys, etc.)", "passed": true, "evidence": "Mentions trust field, automatic redaction, env variable security"}, {"text": "Uses correct Python command (python/python3) in the command field", "passed": true, "evidence": "Uses \"command\": \"python\" consistently"}], "notes": []}, {"eval_id": 3, "eval_name": "custom-slash-command", "configuration": "with_skill", "run_number": 1, "result": {"pass_rate": 1.0, "passed": 5, "failed": 0, "total": 5, "time_seconds": 43.7, "tokens": 63047}, "expectations": [{"text": "Points to ~/.gemini/commands/ for global user-scoped commands", "passed": true, "evidence": "States ~/.gemini/commands/commit.toml for global commands"}, {"text": "Shows .toml file format with prompt field", "passed": true, "evidence": "Complete .toml example with description and prompt fields"}, {"text": "Uses !{git diff --cached} shell execution syntax in the prompt", "passed": true, "evidence": "Uses !{git diff --cached} and explains the syntax"}, {"text": "Warns that custom slash commands don't work in headless/non-interactive mode", "passed": true, "evidence": "States 'Custom slash commands work in interactive mode only. They are not available in headless mode (-p flag).'"}, {"text": "Provides a complete, copy-pasteable .toml file example", "passed": true, "evidence": "Complete commit.toml ready to copy-paste"}], "notes": []}, {"eval_id": 3, "eval_name": "custom-slash-command", "configuration": "without_skill", "run_number": 1, "result": {"pass_rate": 0.8, "passed": 4, "failed": 1, "total": 5, "time_seconds": 73.6, "tokens": 59908}, "expectations": [{"text": "Points to ~/.gemini/commands/ for global user-scoped commands", "passed": true, "evidence": "States ~/.gemini/commands/ for global commands"}, {"text": "Shows .toml file format with prompt field", "passed": true, "evidence": "Complete .toml example with description and prompt fields"}, {"text": "Uses !{git diff --cached} shell execution syntax in the prompt", "passed": true, "evidence": "Uses !{git diff --cached} in the prompt field"}, {"text": "Warns that custom slash commands don't work in headless/non-interactive mode", "passed": false, "evidence": "No mention of headless mode limitations anywhere in the response"}, {"text": "Provides a complete, copy-pasteable .toml file example", "passed": true, "evidence": "Complete commit.toml file provided"}], "notes": []}], "run_summary": {"with_skill": {"pass_rate": {"mean": 1.0, "stddev": 0.0, "min": 1.0, "max": 1.0}, "time_seconds": {"mean": 48.2, "stddev": 5.0, "min": 43.7, "max": 53.6}, "tokens": {"mean": 64595, "stddev": 1741, "min": 63047, "max": 66522}}, "without_skill": {"pass_rate": {"mean": 0.822, "stddev": 0.167, "min": 0.667, "max": 1.0}, "time_seconds": {"mean": 76.1, "stddev": 9.7, "min": 68.0, "max": 86.7}, "tokens": {"mean": 60216, "stddev": 1372, "min": 59322, "max": 61417}}, "delta": {"pass_rate": "+0.18", "time_seconds": "-27.9", "tokens": "+4379"}}, "notes": ["With-skill achieves 100% pass rate across all 3 evals (17/17 assertions) vs 82.2% baseline (14/17)", "Eval 2 (MCP server config) is non-discriminating \u00e2\u20ac\u201d both configs pass all 6 assertions. This eval could be made harder.", "With-skill is ~28s faster on average despite using ~4k more tokens, suggesting more focused responses", "Baseline failures are accuracy-related: hallucinated --json flag (eval 1) and missing headless limitation warning (eval 3)", "The skill's value is strongest on accuracy assertions \u00e2\u20ac\u201d it prevents hallucination of non-existent flags and ensures important caveats are mentioned"]}};
-    // ---- State ----
-    let feedbackMap = {};  // run_id -> feedback text
-    let currentIndex = 0;
-    let visitedRuns = new Set();
-    // ---- Init ----
-    async function init() {
-      // Load saved feedback from server — but only if this isn't a fresh
-      // iteration (indicated by previous_feedback being present). When
-      // previous feedback exists, the feedback.json on disk is stale from
-      // the prior iteration and should not pre-fill the textareas.
-      const hasPrevious = Object.keys(EMBEDDED_DATA.previous_feedback || {}).length > 0
-        || Object.keys(EMBEDDED_DATA.previous_outputs || {}).length > 0;
-      if (!hasPrevious) {
-        try {
-          const resp = await fetch("/api/feedback");
-          const data = await resp.json();
-          if (data.reviews) {
-            for (const r of data.reviews) feedbackMap[r.run_id] = r.feedback;
-          }
-        } catch { /* first run, no feedback yet */ }
-      }
-      document.getElementById("skill-name").textContent = EMBEDDED_DATA.skill_name;
-      showRun(0);
-      // Wire up feedback auto-save
-      const textarea = document.getElementById("feedback");
-      let saveTimeout = null;
-      textarea.addEventListener("input", () => {
-        clearTimeout(saveTimeout);
-        document.getElementById("feedback-status").textContent = "";
-        saveTimeout = setTimeout(() => saveCurrentFeedback(), 800);
-      });
-    }
-    // ---- Navigation ----
-    function navigate(delta) {
-      const newIndex = currentIndex + delta;
-      if (newIndex >= 0 && newIndex < EMBEDDED_DATA.runs.length) {
-        saveCurrentFeedback();
-        showRun(newIndex);
-      }
-    }
-    function updateNavButtons() {
-      document.getElementById("prev-btn").disabled = currentIndex === 0;
-      document.getElementById("next-btn").disabled =
-        currentIndex === EMBEDDED_DATA.runs.length - 1;
-    }
-    // ---- Show a run ----
-    function showRun(index) {
-      currentIndex = index;
-      const run = EMBEDDED_DATA.runs[index];
-      // Progress
-      document.getElementById("progress").textContent =
-        `${index + 1} of ${EMBEDDED_DATA.runs.length}`;
-      // Prompt
-      document.getElementById("prompt-text").textContent = run.prompt;
-      // Config badge
-      const badge = document.getElementById("config-badge");
-      const configMatch = run.id.match(/(with_skill|without_skill|new_skill|old_skill)/);
-      if (configMatch) {
-        const config = configMatch[1];
-        const isBaseline = config === "without_skill" || config === "old_skill";
-        badge.textContent = config.replace(/_/g, " ");
-        badge.className = "config-badge " + (isBaseline ? "config-baseline" : "config-primary");
-        badge.style.display = "inline-block";
-      } else {
-        badge.style.display = "none";
-      }
-      // Outputs
-      renderOutputs(run);
-      // Previous outputs
-      renderPrevOutputs(run);
-      // Grades
-      renderGrades(run);
-      // Previous feedback
-      const prevFb = (EMBEDDED_DATA.previous_feedback || {})[run.id];
-      const prevEl = document.getElementById("prev-feedback");
-      if (prevFb) {
-        document.getElementById("prev-feedback-text").textContent = prevFb;
-        prevEl.style.display = "block";
-      } else {
-        prevEl.style.display = "none";
-      }
-      // Feedback
-      document.getElementById("feedback").value = feedbackMap[run.id] || "";
-      document.getElementById("feedback-status").textContent = "";
-      updateNavButtons();
-      // Track visited runs and promote done button when all visited
-      visitedRuns.add(index);
-      const doneBtn = document.getElementById("done-btn");
-      if (visitedRuns.size >= EMBEDDED_DATA.runs.length) {
-        doneBtn.classList.add("ready");
-      }
-      // Scroll main content to top
-      document.querySelector(".main").scrollTop = 0;
-    }
-    // ---- Render outputs ----
-    function renderOutputs(run) {
-      const container = document.getElementById("outputs-body");
-      container.innerHTML = "";
-      const outputs = run.outputs || [];
-      if (outputs.length === 0) {
-        container.innerHTML = '<div class="empty-state">No output files</div>';
-        return;
-      }
-      for (const file of outputs) {
-        const fileDiv = document.createElement("div");
-        fileDiv.className = "output-file";
-        // Always show file header with download link
-        const header = document.createElement("div");
-        header.className = "output-file-header";
-        const nameSpan = document.createElement("span");
-        nameSpan.textContent = file.name;
-        header.appendChild(nameSpan);
-        const dlBtn = document.createElement("a");
-        dlBtn.className = "dl-btn";
-        dlBtn.textContent = "Download";
-        dlBtn.download = file.name;
-        dlBtn.href = getDownloadUri(file);
-        header.appendChild(dlBtn);
-        fileDiv.appendChild(header);
-        const content = document.createElement("div");
-        content.className = "output-file-content";
-        if (file.type === "text") {
-          const pre = document.createElement("pre");
-          pre.textContent = file.content;
-          content.appendChild(pre);
-        } else if (file.type === "image") {
-          const img = document.createElement("img");
-          img.src = file.data_uri;
-          img.alt = file.name;
-          content.appendChild(img);
-        } else if (file.type === "pdf") {
-          const iframe = document.createElement("iframe");
-          iframe.src = file.data_uri;
-          content.appendChild(iframe);
-        } else if (file.type === "xlsx") {
-          renderXlsx(content, file.data_b64);
-        } else if (file.type === "binary") {
-          const a = document.createElement("a");
-          a.className = "download-link";
-          a.href = file.data_uri;
-          a.download = file.name;
-          a.textContent = "Download " + file.name;
-          content.appendChild(a);
-        } else if (file.type === "error") {
-          const pre = document.createElement("pre");
-          pre.textContent = file.content;
-          pre.style.color = "var(--red)";
-          content.appendChild(pre);
-        }
-        fileDiv.appendChild(content);
-        container.appendChild(fileDiv);
-      }
-    }
-    // ---- XLSX rendering via SheetJS ----
-    function renderXlsx(container, b64Data) {
-      try {
-        const raw = Uint8Array.from(atob(b64Data), c => c.charCodeAt(0));
-        const wb = XLSX.read(raw, { type: "array" });
-        for (let i = 0; i < wb.SheetNames.length; i++) {
-          const sheetName = wb.SheetNames[i];
-          const ws = wb.Sheets[sheetName];
-          if (wb.SheetNames.length > 1) {
-            const sheetLabel = document.createElement("div");
-            sheetLabel.style.cssText =
-              "font-weight:600; font-size:0.8rem; color:#b0aea5; margin-top:0.5rem; margin-bottom:0.25rem;";
-            sheetLabel.textContent = "Sheet: " + sheetName;
-            container.appendChild(sheetLabel);
-          }
-          const htmlStr = XLSX.utils.sheet_to_html(ws, { editable: false });
-          const wrapper = document.createElement("div");
-          wrapper.innerHTML = htmlStr;
-          container.appendChild(wrapper);
-        }
-      } catch (err) {
-        container.textContent = "Error rendering spreadsheet: " + err.message;
-      }
-    }
-    // ---- Grades ----
-    function renderGrades(run) {
-      const section = document.getElementById("grades-section");
-      const content = document.getElementById("grades-content");
-      if (!run.grading) {
-        section.style.display = "none";
-        return;
-      }
-      const grading = run.grading;
-      section.style.display = "block";
-      // Reset to collapsed
-      content.classList.remove("open");
-      document.getElementById("grades-arrow").classList.remove("open");
-      const summary = grading.summary || {};
-      const expectations = grading.expectations || [];
-      let html = '<div style="padding: 1rem;">';
-      // Summary line
-      const passRate = summary.pass_rate != null
-        ? Math.round(summary.pass_rate * 100) + "%"
-        : "?";
-      const badgeClass = summary.pass_rate >= 0.8 ? "grade-pass" : summary.pass_rate >= 0.5 ? "" : "grade-fail";
-      html += '<div class="grades-summary">';
-      html += '<span class="grade-badge ' + badgeClass + '">' + passRate + '</span>';
-      html += '<span>' + (summary.passed || 0) + ' passed, ' + (summary.failed || 0) + ' failed of ' + (summary.total || 0) + '</span>';
-      html += '</div>';
-      // Assertions list
-      html += '<ul class="assertion-list">';
-      for (const exp of expectations) {
-        const statusClass = exp.passed ? "pass" : "fail";
-        const statusIcon = exp.passed ? "\u2713" : "\u2717";
-        html += '<li class="assertion-item">';
-        html += '<span class="assertion-status ' + statusClass + '">' + statusIcon + '</span>';
-        html += '<span>' + escapeHtml(exp.text) + '</span>';
-        if (exp.evidence) {
-          html += '<div class="assertion-evidence">' + escapeHtml(exp.evidence) + '</div>';
-        }
-        html += '</li>';
-      }
-      html += '</ul>';
-      html += '</div>';
-      content.innerHTML = html;
-    }
-    function toggleGrades() {
-      const content = document.getElementById("grades-content");
-      const arrow = document.getElementById("grades-arrow");
-      content.classList.toggle("open");
-      arrow.classList.toggle("open");
-    }
-    // ---- Previous outputs (collapsible) ----
-    function renderPrevOutputs(run) {
-      const section = document.getElementById("prev-outputs-section");
-      const content = document.getElementById("prev-outputs-content");
-      const prevOutputs = (EMBEDDED_DATA.previous_outputs || {})[run.id];
-      if (!prevOutputs || prevOutputs.length === 0) {
-        section.style.display = "none";
-        return;
-      }
-      section.style.display = "block";
-      // Reset to collapsed
-      content.classList.remove("open");
-      document.getElementById("prev-outputs-arrow").classList.remove("open");
-      // Render the files into the content area
-      content.innerHTML = "";
-      const wrapper = document.createElement("div");
-      wrapper.style.padding = "1rem";
-      for (const file of prevOutputs) {
-        const fileDiv = document.createElement("div");
-        fileDiv.className = "output-file";
-        const header = document.createElement("div");
-        header.className = "output-file-header";
-        const nameSpan = document.createElement("span");
-        nameSpan.textContent = file.name;
-        header.appendChild(nameSpan);
-        const dlBtn = document.createElement("a");
-        dlBtn.className = "dl-btn";
-        dlBtn.textContent = "Download";
-        dlBtn.download = file.name;
-        dlBtn.href = getDownloadUri(file);
-        header.appendChild(dlBtn);
-        fileDiv.appendChild(header);
-        const fc = document.createElement("div");
-        fc.className = "output-file-content";
-        if (file.type === "text") {
-          const pre = document.createElement("pre");
-          pre.textContent = file.content;
-          fc.appendChild(pre);
-        } else if (file.type === "image") {
-          const img = document.createElement("img");
-          img.src = file.data_uri;
-          img.alt = file.name;
-          fc.appendChild(img);
-        } else if (file.type === "pdf") {
-          const iframe = document.createElement("iframe");
-          iframe.src = file.data_uri;
-          fc.appendChild(iframe);
-        } else if (file.type === "xlsx") {
-          renderXlsx(fc, file.data_b64);
-        } else if (file.type === "binary") {
-          const a = document.createElement("a");
-          a.className = "download-link";
-          a.href = file.data_uri;
-          a.download = file.name;
-          a.textContent = "Download " + file.name;
-          fc.appendChild(a);
-        }
-        fileDiv.appendChild(fc);
-        wrapper.appendChild(fileDiv);
-      }
-      content.appendChild(wrapper);
-    }
-    function togglePrevOutputs() {
-      const content = document.getElementById("prev-outputs-content");
-      const arrow = document.getElementById("prev-outputs-arrow");
-      content.classList.toggle("open");
-      arrow.classList.toggle("open");
-    }
-    // ---- Feedback (saved to server -> feedback.json) ----
-    function saveCurrentFeedback() {
-      const run = EMBEDDED_DATA.runs[currentIndex];
-      const text = document.getElementById("feedback").value;
-      if (text.trim() === "") {
-        delete feedbackMap[run.id];
-      } else {
-        feedbackMap[run.id] = text;
-      }
-      // Build reviews array from map
-      const reviews = [];
-      for (const [run_id, feedback] of Object.entries(feedbackMap)) {
-        if (feedback.trim()) {
-          reviews.push({ run_id, feedback, timestamp: new Date().toISOString() });
-        }
-      }
-      fetch("/api/feedback", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ reviews, status: "in_progress" }),
-      }).then(() => {
-        document.getElementById("feedback-status").textContent = "Saved";
-      }).catch(() => {
-        // Static mode or server unavailable — no-op on auto-save,
-        // feedback will be downloaded on final submit
-        document.getElementById("feedback-status").textContent = "Will download on submit";
-      });
-    }
-    // ---- Done ----
-    function showDoneDialog() {
-      // Save current textarea to feedbackMap (but don't POST yet)
-      const run = EMBEDDED_DATA.runs[currentIndex];
-      const text = document.getElementById("feedback").value;
-      if (text.trim() === "") {
-        delete feedbackMap[run.id];
-      } else {
-        feedbackMap[run.id] = text;
-      }
-      // POST once with status: complete — include ALL runs so the model
-      // can distinguish "no feedback" (looks good) from "not reviewed"
-      const reviews = [];
-      const ts = new Date().toISOString();
-      for (const r of EMBEDDED_DATA.runs) {
-        reviews.push({ run_id: r.id, feedback: feedbackMap[r.id] || "", timestamp: ts });
-      }
-      const payload = JSON.stringify({ reviews, status: "complete" }, null, 2);
-      fetch("/api/feedback", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: payload,
-      }).then(() => {
-        document.getElementById("done-overlay").classList.add("visible");
-      }).catch(() => {
-        // Server not available (static mode) — download as file
-        const blob = new Blob([payload], { type: "application/json" });
-        const url = URL.createObjectURL(blob);
-        const a = document.createElement("a");
-        a.href = url;
-        a.download = "feedback.json";
-        a.click();
-        URL.revokeObjectURL(url);
-        document.getElementById("done-overlay").classList.add("visible");
-      });
-    }
-    function closeDoneDialog() {
-      // Reset status back to in_progress
-      saveCurrentFeedback();
-      document.getElementById("done-overlay").classList.remove("visible");
-    }
-    // ---- Toast ----
-    function showToast(message) {
-      const toast = document.getElementById("toast");
-      toast.textContent = message;
-      toast.classList.add("visible");
-      setTimeout(() => toast.classList.remove("visible"), 2000);
-    }
-    // ---- Keyboard nav ----
-    document.addEventListener("keydown", (e) => {
-      // Don't capture when typing in textarea
-      if (e.target.tagName === "TEXTAREA") return;
-      if (e.key === "ArrowLeft" || e.key === "ArrowUp") {
-        e.preventDefault();
-        navigate(-1);
-      } else if (e.key === "ArrowRight" || e.key === "ArrowDown") {
-        e.preventDefault();
-        navigate(1);
-      }
-    });
-    // ---- Util ----
-    function getDownloadUri(file) {
-      if (file.data_uri) return file.data_uri;
-      if (file.data_b64) return "data:application/octet-stream;base64," + file.data_b64;
-      if (file.type === "text") return "data:text/plain;charset=utf-8," + encodeURIComponent(file.content);
-      return "#";
-    }
-    function escapeHtml(text) {
-      const div = document.createElement("div");
-      div.textContent = text;
-      return div.innerHTML;
-    }
-    // ---- View switching ----
-    function switchView(view) {
-      document.querySelectorAll(".view-tab").forEach(t => t.classList.remove("active"));
-      document.querySelectorAll(".view-panel").forEach(p => p.classList.remove("active"));
-      document.querySelector(`[onclick="switchView('${view}')"]`).classList.add("active");
-      document.getElementById("panel-" + view).classList.add("active");
-    }
-    // ---- Benchmark rendering ----
-    function renderBenchmark() {
-      const data = EMBEDDED_DATA.benchmark;
-      if (!data) return;
-      // Show the tabs
-      document.getElementById("view-tabs").style.display = "flex";
-      const container = document.getElementById("benchmark-content");
-      const summary = data.run_summary || {};
-      const metadata = data.metadata || {};
-      const notes = data.notes || [];
-      let html = "";
-      // Header
-      html += "<h2 style='font-family: Poppins, sans-serif; margin-bottom: 0.5rem;'>Benchmark Results</h2>";
-      html += "<p style='color: var(--text-muted); font-size: 0.875rem; margin-bottom: 1.25rem;'>";
-      if (metadata.skill_name) html += "<strong>" + escapeHtml(metadata.skill_name) + "</strong> &mdash; ";
-      if (metadata.timestamp) html += metadata.timestamp + " &mdash; ";
-      if (metadata.evals_run) html += "Evals: " + metadata.evals_run.join(", ") + " &mdash; ";
-      html += (metadata.runs_per_configuration || "?") + " runs per configuration";
-      html += "</p>";
-      // Summary table
-      html += '<table class="benchmark-table">';
-      function fmtStat(stat, pct) {
-        if (!stat) return "—";
-        const suffix = pct ? "%" : "";
-        const m = pct ? (stat.mean * 100).toFixed(0) : stat.mean.toFixed(1);
-        const s = pct ? (stat.stddev * 100).toFixed(0) : stat.stddev.toFixed(1);
-        return m + suffix + " ± " + s + suffix;
-      }
-      function deltaClass(val) {
-        if (!val) return "";
-        const n = parseFloat(val);
-        if (n > 0) return "benchmark-delta-positive";
-        if (n < 0) return "benchmark-delta-negative";
-        return "";
-      }
-      // Discover config names dynamically (everything except "delta")
-      const configs = Object.keys(summary).filter(k => k !== "delta");
-      const configA = configs[0] || "config_a";
-      const configB = configs[1] || "config_b";
-      const labelA = configA.replace(/_/g, " ").replace(/\b\w/g, c => c.toUpperCase());
-      const labelB = configB.replace(/_/g, " ").replace(/\b\w/g, c => c.toUpperCase());
-      const a = summary[configA] || {};
-      const b = summary[configB] || {};
-      const delta = summary.delta || {};
-      html += "<thead><tr><th>Metric</th><th>" + escapeHtml(labelA) + "</th><th>" + escapeHtml(labelB) + "</th><th>Delta</th></tr></thead>";
-      html += "<tbody>";
-      html += "<tr><td><strong>Pass Rate</strong></td>";
-      html += "<td>" + fmtStat(a.pass_rate, true) + "</td>";
-      html += "<td>" + fmtStat(b.pass_rate, true) + "</td>";
-      html += '<td class="' + deltaClass(delta.pass_rate) + '">' + (delta.pass_rate || "—") + "</td></tr>";
-      // Time (only show row if data exists)
-      if (a.time_seconds || b.time_seconds) {
-        html += "<tr><td><strong>Time (s)</strong></td>";
-        html += "<td>" + fmtStat(a.time_seconds, false) + "</td>";
-        html += "<td>" + fmtStat(b.time_seconds, false) + "</td>";
-        html += '<td class="' + deltaClass(delta.time_seconds) + '">' + (delta.time_seconds ? delta.time_seconds + "s" : "—") + "</td></tr>";
-      }
-      // Tokens (only show row if data exists)
-      if (a.tokens || b.tokens) {
-        html += "<tr><td><strong>Tokens</strong></td>";
-        html += "<td>" + fmtStat(a.tokens, false) + "</td>";
-        html += "<td>" + fmtStat(b.tokens, false) + "</td>";
-        html += '<td class="' + deltaClass(delta.tokens) + '">' + (delta.tokens || "—") + "</td></tr>";
-      }
-      html += "</tbody></table>";
-      // Per-eval breakdown (if runs data available)
-      const runs = data.runs || [];
-      if (runs.length > 0) {
-        const evalIds = [...new Set(runs.map(r => r.eval_id))].sort((a, b) => a - b);
-        html += "<h3 style='font-family: Poppins, sans-serif; margin-bottom: 0.75rem;'>Per-Eval Breakdown</h3>";
-        const hasTime = runs.some(r => r.result && r.result.time_seconds != null);
-        const hasErrors = runs.some(r => r.result && r.result.errors > 0);
-        for (const evalId of evalIds) {
-          const evalRuns = runs.filter(r => r.eval_id === evalId);
-          const evalName = evalRuns[0] && evalRuns[0].eval_name ? evalRuns[0].eval_name : "Eval " + evalId;
-          html += "<h4 style='font-family: Poppins, sans-serif; margin: 1rem 0 0.5rem; color: var(--text);'>" + escapeHtml(evalName) + "</h4>";
-          html += '<table class="benchmark-table">';
-          html += "<thead><tr><th>Config</th><th>Run</th><th>Pass Rate</th>";
-          if (hasTime) html += "<th>Time (s)</th>";
-          if (hasErrors) html += "<th>Crashes During Execution</th>";
-          html += "</tr></thead>";
-          html += "<tbody>";
-          // Group by config and render with average rows
-          const configGroups = [...new Set(evalRuns.map(r => r.configuration))];
-          for (let ci = 0; ci < configGroups.length; ci++) {
-            const config = configGroups[ci];
-            const configRuns = evalRuns.filter(r => r.configuration === config);
-            if (configRuns.length === 0) continue;
-            const rowClass = ci === 0 ? "benchmark-row-with" : "benchmark-row-without";
-            const configLabel = config.replace(/_/g, " ").replace(/\b\w/g, c => c.toUpperCase());
-            for (const run of configRuns) {
-              const r = run.result || {};
-              const prClass = r.pass_rate >= 0.8 ? "benchmark-delta-positive" : r.pass_rate < 0.5 ? "benchmark-delta-negative" : "";
-              html += '<tr class="' + rowClass + '">';
-              html += "<td>" + configLabel + "</td>";
-              html += "<td>" + run.run_number + "</td>";
-              html += '<td class="' + prClass + '">' + ((r.pass_rate || 0) * 100).toFixed(0) + "% (" + (r.passed || 0) + "/" + (r.total || 0) + ")</td>";
-              if (hasTime) html += "<td>" + (r.time_seconds != null ? r.time_seconds.toFixed(1) : "—") + "</td>";
-              if (hasErrors) html += "<td>" + (r.errors || 0) + "</td>";
-              html += "</tr>";
-            }
-            // Average row
-            const rates = configRuns.map(r => (r.result || {}).pass_rate || 0);
-            const avgRate = rates.reduce((a, b) => a + b, 0) / rates.length;
-            const avgPrClass = avgRate >= 0.8 ? "benchmark-delta-positive" : avgRate < 0.5 ? "benchmark-delta-negative" : "";
-            html += '<tr class="benchmark-row-avg ' + rowClass + '">';
-            html += "<td>" + configLabel + "</td>";
-            html += "<td>Avg</td>";
-            html += '<td class="' + avgPrClass + '">' + (avgRate * 100).toFixed(0) + "%</td>";
-            if (hasTime) {
-              const times = configRuns.map(r => (r.result || {}).time_seconds).filter(t => t != null);
-              html += "<td>" + (times.length ? (times.reduce((a, b) => a + b, 0) / times.length).toFixed(1) : "—") + "</td>";
-            }
-            if (hasErrors) html += "<td></td>";
-            html += "</tr>";
-          }
-          html += "</tbody></table>";
-          // Per-assertion detail for this eval
-          const runsWithExpectations = {};
-          for (const config of configGroups) {
-            runsWithExpectations[config] = evalRuns.filter(r => r.configuration === config && r.expectations && r.expectations.length > 0);
-          }
-          const hasAnyExpectations = Object.values(runsWithExpectations).some(runs => runs.length > 0);
-          if (hasAnyExpectations) {
-            // Collect all unique assertion texts across all configs
-            const allAssertions = [];
-            const seen = new Set();
-            for (const config of configGroups) {
-              for (const run of runsWithExpectations[config]) {
-                for (const exp of (run.expectations || [])) {
-                  if (!seen.has(exp.text)) {
-                    seen.add(exp.text);
-                    allAssertions.push(exp.text);
-                  }
-                }
-              }
-            }
-            html += '<table class="benchmark-table" style="margin-top: 0.5rem;">';
-            html += "<thead><tr><th>Assertion</th>";
-            for (const config of configGroups) {
-              const label = config.replace(/_/g, " ").replace(/\b\w/g, c => c.toUpperCase());
-              html += "<th>" + escapeHtml(label) + "</th>";
-            }
-            html += "</tr></thead><tbody>";
-            for (const assertionText of allAssertions) {
-              html += "<tr><td>" + escapeHtml(assertionText) + "</td>";
-              for (const config of configGroups) {
-                html += "<td>";
-                for (const run of runsWithExpectations[config]) {
-                  const exp = (run.expectations || []).find(e => e.text === assertionText);
-                  if (exp) {
-                    const cls = exp.passed ? "benchmark-delta-positive" : "benchmark-delta-negative";
-                    const icon = exp.passed ? "\u2713" : "\u2717";
-                    html += '<span class="' + cls + '" title="Run ' + run.run_number + ': ' + escapeHtml(exp.evidence || "") + '">' + icon + "</span> ";
-                  } else {
-                    html += "— ";
-                  }
-                }
-                html += "</td>";
-              }
-              html += "</tr>";
-            }
-            html += "</tbody></table>";
-          }
-        }
-      }
-      // Notes
-      if (notes.length > 0) {
-        html += '<div class="benchmark-notes">';
-        html += "<h3>Analysis Notes</h3>";
-        html += "<ul>";
-        for (const note of notes) {
-          html += "<li>" + escapeHtml(note) + "</li>";
-        }
-        html += "</ul></div>";
-      }
-      container.innerHTML = html;
-    }
-    // ---- Start ----
-    init();
-    renderBenchmark();
-  </script>
-</body>
-</html>